Section
{
admin;
- base;
+ cli-mono;
comm;
+ database;
debian-installer;
+ debug;
devel;
doc;
editors;
embedded;
electronics;
+ fonts;
games;
gnome;
graphics;
+ gnu-r;
+ gnustep;
hamradio;
+ haskell;
+ httpd;
interpreters;
+ java;
kde;
+ kernel;
libdevel;
libs;
+ lisp;
+ localization;
mail;
math;
misc;
net;
news;
+ ocaml;
oldlibs;
otherosfs;
perl;
+ php;
python;
+ ruby;
science;
shells;
sound;
text;
utils;
web;
+ vcs;
+ video;
x11;
+ xfce;
+ zope;
};
Priority
Section
{
admin;
- base;
+ cli-mono;
comm;
+ database;
debian-installer;
+ debug;
devel;
doc;
editors;
- electronics;
embedded;
+ electronics;
+ fonts;
games;
gnome;
graphics;
+ gnu-r;
+ gnustep;
hamradio;
+ haskell;
+ httpd;
interpreters;
+ java;
kde;
+ kernel;
libdevel;
libs;
+ lisp;
+ localization;
mail;
math;
misc;
net;
news;
+ ocaml;
oldlibs;
otherosfs;
perl;
+ php;
python;
+ ruby;
science;
shells;
sound;
text;
utils;
web;
+ vcs;
+ video;
x11;
- non-US;
+ xfce;
+ zope;
};
Priority
tree "dists/proposed-updates"
{
FileList "/srv/ftp.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
- SourceFileList "/srv/ftp.debian.org/database/dists/proposed_updates_$(SECTION)_source.list";
+ SourceFileList "/srv/ftp.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
Sections "main contrib non-free";
Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source";
BinOverride "override.lenny.$(SECTION)";
FileList "/srv/ftp.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
SourceFileList "/srv/ftp.debian.org/database/dists/unstable_$(SECTION)_source.list";
Sections "main contrib non-free";
- Architectures "alpha amd64 armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc source";
+ Architectures "alpha amd64 armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 source";
BinOverride "override.sid.$(SECTION)";
ExtraOverride "override.sid.extra.$(SECTION)";
SrcOverride "override.sid.$(SECTION).src";
FileList "/srv/ftp.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
SourceFileList "/srv/ftp.debian.org/database/dists/experimental_$(SECTION)_source.list";
Sections "main contrib non-free";
- Architectures "alpha amd64 armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc source";
+ Architectures "alpha amd64 armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 source";
BinOverride "override.sid.$(SECTION)";
SrcOverride "override.sid.$(SECTION).src";
};
log "$*"
fi
}
+
+# used by cron.dinstall *and* cron.unchecked.
+function make_buildd_dir () {
+ cd $configdir
+ apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
+
+ cd ${incoming}
+ rm -f buildd/Release*
+ apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="Debian" -o APT::FTPArchive::Release::Label="Debian" -o APT::FTPArchive::Release::Description="buildd incoming" -o APT::FTPArchive::Release::Architectures="${archs}" release buildd > Release
+ gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o Release.gpg Release
+ mv Release* buildd/.
+
+ cd ${incoming}
+ mkdir -p tree/${STAMP}
+ cp -al ${incoming}/buildd/. tree/${STAMP}/
+ ln -sfT tree/${STAMP} ${incoming}/builddweb
+ find ./tree -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
+}
rm -f ${LOCK_ACCEPTED}
}
+# If we error out this one is called, *FOLLOWED* by cleanup above
+function onerror() {
+ ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
+ cat "$LOGFILE" | mail -s "ATTENTION ATTENTION! dinstall error at ${ERRDATE} (Be quiet, Brain, or I'll stab you with a Q-tip)" cron@ftp-master.debian.org
+}
+
########################################################################
# the actual dinstall functions follow #
########################################################################
# Create the postgres dump files
function pgdump_pre() {
log "Creating pre-daily-cron-job backup of projectb database..."
- pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+ pg_dump projectb > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S)
}
function pgdump_post() {
echo "drop database projectb" | psql -p 5433 template1
cat currentall | psql -p 5433 template1
createdb -p 5433 -T template0 projectb
- fgrep -v '\connect' `cat current` | psql -p 5433 projectb
+ fgrep -v '\connect' current | psql -p 5433 projectb
}
# Updating various files
function dakcleanup() {
log "Cleanup old packages/files"
- dak clean-suites
+ dak clean-suites -m 10000
dak clean-queues
}
apt-ftparchive generate apt.conf.buildd
}
+function buildd_dir() {
+ # Rebuilt the buildd dir to avoid long times of 403
+ log "Regenerating the buildd incoming dir"
+ make_buildd_dir
+}
+
function scripts() {
log "Running various scripts from $scriptsdir"
cd $scriptsdir
}
function merkel2() {
- # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
+ # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
log "Trigger merkels projectb sync"
- ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+ ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1
+}
+
+function merkel3() {
+ # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached
+ log "Trigger merkels dd accessible parts sync"
+ ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
}
function runparts() {
function compress() {
log "Compress old psql backups"
cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +1 |
+ find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
+
+ find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin 720 |
while read dumpname; do
echo "Compressing $dumpname"
bzip2 -9v "$dumpname"
done
+ find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin 720 |
+ while read dumpname; do
+ echo "Compressing $dumpname"
+ bzip2 -9v "$dumpname"
+ done
+ finddup -l -d $base/backup
}
function logstats() {
echo ${NOW} > "${dbdir}/dinstallstart"
}
+function maillogfile() {
+ cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
+}
+
function renamelogfile() {
- if [ -f "${dbdir}/dinstallstart" ]; then
- RENAMETO=$(cat "${dbdir}/dinstallstart")
- mv "$LOGFILE" "$logdir/dinstall_${RENAMETO}.log"
- logstats "$logdir/dinstall_${RENAMETO}.log"
- bzip2 -9 "$logdir/dinstall_${RENAMETO}.log"
- else
- error "Problem, I don't know when dinstall started, unable to do log statistics."
- NOW=`date "+%Y.%m.%d-%H:%M:%S"`
- mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
- bzip2 -9 "$logdir/dinstall_${NOW}.log"
- fi
+ if [ -f "${dbdir}/dinstallstart" ]; then
+ NOW=$(cat "${dbdir}/dinstallstart")
+ maillogfile
+ mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
+ logstats "$logdir/dinstall_${NOW}.log"
+ bzip2 -9 "$logdir/dinstall_${NOW}.log"
+ else
+ error "Problem, I don't know when dinstall started, unable to do log statistics."
+ NOW=`date "+%Y.%m.%d-%H:%M:%S"`
+ maillogfile
+ mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
+ bzip2 -9 "$logdir/dinstall_${NOW}.log"
+ fi
}
+function testingsourcelist() {
+ dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
+}
########################################################################
########################################################################
# it has to cd first!
cd ${configdir}
- if [ -f "${LOCK_STOP}" ]; then
- log "${LOCK_STOP} exists, exiting immediately"
- exit 42
- fi
+ if [ -f "${LOCK_STOP}" ]; then
+ log "${LOCK_STOP} exists, exiting immediately"
+ exit 42
+ fi
if [ "${ERR}" = "false" ]; then
set +e
ts "${TIME}"
fi
- if [ -f "${LOCK_STOP}" ]; then
- log "${LOCK_STOP} exists, exiting immediately"
- exit 42
- fi
+ if [ -f "${LOCK_STOP}" ]; then
+ log "${LOCK_STOP} exists, exiting immediately"
+ exit 42
+ fi
}
########################################################################
LOCK_STOP="$lockdir/archive.stop"
lockfile -l 3600 "${LOCK_DAILY}"
-trap cleanup EXIT ERR TERM HUP INT QUIT
+trap onerror ERR
+trap cleanup EXIT TERM HUP INT QUIT
touch "${LOCK_BRITNEY}"
)
stage $GO
+GO=(
+ FUNC="buildd_dir"
+ TIME="buildd_dir"
+ ARGS=""
+ ERR="false"
+)
+stage $GO
+
GO=(
FUNC="cruft"
TIME="cruft"
)
stage $GO
-GO=(
- FUNC="pgdakdev"
- TIME="dak-dev db"
- ARGS=""
- ERR="false"
-)
-stage $GO
-
GO=(
FUNC="expire"
TIME="expire_dumps"
FUNC="bts"
TIME=""
ARGS=""
- ERR=""
+ ERR="false"
)
stage $GO
)
stage $GO
-ulimit -m 90000 -d 90000 -s 10000 -v 200000
-
GO=(
FUNC="runparts"
TIME="run-parts"
)
stage $GO
+GO=(
+ FUNC="testingsourcelist"
+ TIME=""
+ ARGS=""
+ ERR="false"
+)
+stage $GO
+
rm -f ${LOCK_BRITNEY}
+GO=(
+ FUNC="pgdakdev"
+ TIME="dak-dev db"
+ ARGS=""
+ ERR="false"
+)
+stage $GO
+
GO=(
FUNC="aptftpcleanup"
TIME="apt-ftparchive cleanup"
)
stage $GO
+GO=(
+ FUNC="merkel3"
+ TIME="merkel ddaccessible sync"
+ ARGS=""
+ ERR="false"
+)
+stage $GO
+
GO=(
FUNC="compress"
TIME="compress"
exec > "$logdir/afterdinstall.log" 2>&1
-cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
-
GO=(
FUNC="renamelogfile"
TIME=""
for m in mail bxamail; do
if [ -f $m ]; then
mv $m ${m}-$DATE
+ touch ${m}
+ chown dak:ftpteam ${m}
+ chmod 660 ${m}
sleep 20
gzip -9 ${m}-$DATE
chgrp $ftpgroup ${m}-$DATE.gz
export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
. $SCRIPTVARS
+# common functions are "outsourced"
+. "${configdir}/common"
+
LOCKDAILY=""
LOCKFILE="$lockdir/unchecked.lock"
NOTICE="$lockdir/daily.lock"
cat override.sid.$i.src >> override.sid.all3.src
fi
done
- cd $configdir
- apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
-
- cd ${incoming}
- rm -f buildd/Release*
- apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="Debian" -o APT::FTPArchive::Release::Label="Debian" -o APT::FTPArchive::Release::Description="buildd incoming" -o APT::FTPArchive::Release::Architectures="${archs}" release buildd > Release
- gpg --secret-keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 6070D3A1 --detach-sign -o Release.gpg Release
- mv Release* buildd/.
-
- cd ${incoming}
- mkdir -p tree/${STAMP}
- cp -al ${incoming}/buildd/. tree/${STAMP}/
- ln -sfT tree/${STAMP} ${incoming}/builddweb
- find ./tree -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
+ make_buildd_dir
. $configdir/cron.buildd
fi
Dinstall
{
GPGKeyring {
- "/srv/keyring.debian.org/keyrings/debian-keyring.gpg";
+ "/srv/keyring.debian.org/keyrings/debian-keyring.gpg";
"/srv/keyring.debian.org/keyrings/debian-keyring.pgp";
"/srv/ftp.debian.org/keyrings/debian-maintainers.gpg";
};
CodeName "etch";
OverrideCodeName "etch";
Priority "5";
- Untouchable "1";
ChangeLogBase "dists/oldstable/";
UdebComponents
{
CodeName "lenny";
OverrideCodeName "lenny";
Priority "5";
- Untouchable "1";
ChangeLogBase "dists/stable/";
UdebComponents
{
s390 "IBM S/390";
sh "Hitatchi SuperH";
sparc "Sun SPARC/UltraSPARC";
+ kfreebsd-i386 "GNU/kFreeBSD i386";
+ kfreebsd-amd64 "GNU/kFreeBSD amd64";
};
Archive
Section
{
admin;
- base;
+ cli-mono;
comm;
+ database;
debian-installer;
+ debug;
devel;
doc;
editors;
embedded;
electronics;
+ fonts;
games;
gnome;
graphics;
+ gnu-r;
+ gnustep;
hamradio;
+ haskell;
+ httpd;
interpreters;
+ java;
kde;
+ kernel;
libdevel;
libs;
+ lisp;
+ localization;
mail;
math;
misc;
net;
news;
+ ocaml;
oldlibs;
otherosfs;
perl;
+ php;
python;
+ ruby;
science;
shells;
sound;
text;
utils;
web;
+ vcs;
+ video;
x11;
+ xfce;
+ zope;
};
Priority
critical;
};
};
+
+Contents
+{
+ Header "contents";
+ Root "/srv/ftp.debian.org/test/";
+}
ftpdir=$base/ftp
webdir=$base/web
indices=$ftpdir/indices
-archs="alpha amd64 arm armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc"
+archs="alpha amd64 arm armel hppa hurd-i386 i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 "
scriptdir=$base/scripts
masterdir=$base/dak/
controls = ""
bc = BugClassifier()
- for bug in bc.unclassified_bugs():
- controls += bc.classify_bug(bug)
-
- return controls
+ try:
+ for bug in bc.unclassified_bugs():
+ controls += bc.classify_bug(bug)
+
+ return controls
+ except:
+ log.error("couldn't retrieve bugs from soap interface: %s" % sys.exc_info()[0])
+ return None
def send_email(commands, simulate=False):
global Cnf
# Delete files from the pool
query = "SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)
if max_delete is not None:
- query += " LIMIT %d" % maximum
- sys.stdout.write("Limiting removals to %d" % Cnf["Clean-Suites::Options::Maximum"])
+ query += " LIMIT %d" % max_delete
+ sys.stdout.write("Limiting removals to %d\n" % max_delete)
q=projectB.query(query)
for i in q.getresult():
--- /dev/null
+#!/usr/bin/env python
+"""
+Create all the contents files
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2008, 2009 Michael Casadevall <mcasadevall@debian.org>
+@copyright: 2009 Mike O'Connor <stew@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+################################################################################
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <Ganneff> there is the idea to slowly replace contents files
+# <Ganneff> with a new generation of such files.
+# <Ganneff> having more info.
+
+# <Ganneff> of course that wont help for now where we need to generate them :)
+
+################################################################################
+
+import sys
+import os
+import logging
+import math
+import gzip
+import apt_pkg
+from daklib import utils
+from daklib.binary import Binary
+from daklib.config import Config
+from daklib.dbconn import DBConn
+################################################################################
+
+def usage (exit_code=0):
+ print """Usage: dak contents [options] command [arguments]
+
+COMMANDS
+ generate
+ generate Contents-$arch.gz files
+
+ bootstrap
+ scan the debs in the existing pool and load contents in the the database
+
+ cruft
+ remove files/paths which are no longer referenced by a binary
+
+OPTIONS
+ -h, --help
+ show this help and exit
+
+ -v, --verbose
+ show verbose information messages
+
+ -q, --quiet
+ supress all output but errors
+
+ -s, --suite={stable,testing,unstable,...}
+ only operate on a single suite
+
+ -a, --arch={i386,amd64}
+ only operate on a single architecture
+"""
+ sys.exit(exit_code)
+
+################################################################################
+
+# where in dak.conf all of our configuration will be stowed
+
+options_prefix = "Contents"
+options_prefix = "%s::Options" % options_prefix
+
+log = logging.getLogger()
+
+################################################################################
+
+# get all the arches delivered for a given suite
+# this should probably exist somehere common
+arches_q = """PREPARE arches_q(int) as
+ SELECT s.architecture, a.arch_string
+ FROM suite_architectures s
+ JOIN architecture a ON (s.architecture=a.id)
+ WHERE suite = $1"""
+
+# find me the .deb for a given binary id
+debs_q = """PREPARE debs_q(int, int) as
+ SELECT b.id, f.filename FROM bin_assoc_by_arch baa
+ JOIN binaries b ON baa.bin=b.id
+ JOIN files f ON b.file=f.id
+ WHERE suite = $1
+ AND arch = $2"""
+
+# ask if we already have contents associated with this binary
+olddeb_q = """PREPARE olddeb_q(int) as
+ SELECT 1 FROM content_associations
+ WHERE binary_pkg = $1
+ LIMIT 1"""
+
+# find me all of the contents for a given .deb
+contents_q = """PREPARE contents_q(int,int,int,int) as
+ SELECT (p.path||'/'||n.file) AS fn,
+ comma_separated_list(s.section||'/'||b.package)
+ FROM content_associations c
+ JOIN content_file_paths p ON (c.filepath=p.id)
+ JOIN content_file_names n ON (c.filename=n.id)
+ JOIN binaries b ON (b.id=c.binary_pkg)
+ JOIN bin_associations ba ON (b.id=ba.bin)
+ JOIN override o ON (o.package=b.package)
+ JOIN section s ON (s.id=o.section)
+ WHERE (b.architecture = $1 OR b.architecture = $2)
+ AND ba.suite = $3
+ AND o.suite = $3
+ AND b.type = 'deb'
+ AND o.type = $4
+ GROUP BY fn
+ ORDER BY fn"""
+
+udeb_contents_q = """PREPARE udeb_contents_q(int,int,int,int,int) as
+ SELECT (p.path||'/'||n.file) as fn,
+ comma_separated_list(s.section||'/'||b.package)
+ FROM content_associations c
+ JOIN content_file_paths p ON (c.filepath=p.id)
+ JOIN content_file_names n ON (c.filename=n.id)
+ JOIN binaries b ON (b.id=c.binary_pkg)
+ JOIN bin_associations ba ON (b.id=ba.bin)
+ JOIN override o ON (o.package=b.package)
+ JOIN section s ON (s.id=o.section)
+ WHERE (b.architecture = $1 OR b.architecture = $2)
+ AND s.id = $3
+ AND ba.suite = $4
+ AND o.suite = $4
+ AND b.type = 'udeb'
+ AND o.type = $5
+ GROUP BY fn
+ ORDER BY fn"""
+
+
+# clear out all of the temporarily stored content associations
+# this should be run only after p-a has run. after a p-a
+# run we should have either accepted or rejected every package
+# so there should no longer be anything in the queue
+remove_pending_contents_cruft_q = """DELETE FROM pending_content_associations"""
+
+# delete any filenames we are storing which have no binary associated with them
+remove_filename_cruft_q = """DELETE FROM content_file_names
+ WHERE id IN (SELECT cfn.id FROM content_file_names cfn
+ LEFT JOIN content_associations ca
+ ON ca.filename=cfn.id
+ WHERE ca.id IS NULL)"""
+
+# delete any paths we are storing which have no binary associated with them
+remove_filepath_cruft_q = """DELETE FROM content_file_paths
+ WHERE id IN (SELECT cfn.id FROM content_file_paths cfn
+ LEFT JOIN content_associations ca
+ ON ca.filepath=cfn.id
+ WHERE ca.id IS NULL)"""
+class Contents(object):
+ """
+ Class capable of generating Contents-$arch.gz files
+
+ Usage GenerateContents().generateContents( ["main","contrib","non-free"] )
+ """
+
+ def __init__(self):
+ self.header = None
+
+ def reject(self, message):
+ log.error("E: %s" % message)
+
+ def _getHeader(self):
+ """
+ Internal method to return the header for Contents.gz files
+
+ This is boilerplate which explains the contents of the file and how
+ it can be used.
+ """
+ if self.header == None:
+ if Config().has_key("Contents::Header"):
+ try:
+ h = open(os.path.join( Config()["Dir::Templates"],
+ Config()["Contents::Header"] ), "r")
+ self.header = h.read()
+ h.close()
+ except:
+ log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"],
+ traceback.format_exc() ))
+ self.header = False
+ else:
+ self.header = False
+
+ return self.header
+
+ # goal column for section column
+ _goal_column = 54
+
+ def _write_content_file(self, cursor, filename):
+ """
+ Internal method for writing all the results to a given file.
+ The cursor should have a result set generated from a query already.
+ """
+ filepath = Config()["Contents::Root"] + filename
+ filedir = os.path.dirname(filepath)
+ if not os.path.isdir(filedir):
+ os.makedirs(filedir)
+ f = gzip.open(filepath, "w")
+ try:
+ header = self._getHeader()
+
+ if header:
+ f.write(header)
+
+ while True:
+ contents = cursor.fetchone()
+ if not contents:
+ return
+
+ num_tabs = max(1,
+ int(math.ceil((self._goal_column - len(contents[0])-1) / 8)))
+ f.write(contents[0] + ( '\t' * num_tabs ) + contents[-1] + "\n")
+
+ finally:
+ f.close()
+
+ def cruft(self):
+ """
+ remove files/paths from the DB which are no longer referenced
+ by binaries and clean the temporary table
+ """
+ cursor = DBConn().cursor();
+ cursor.execute( "BEGIN WORK" )
+ cursor.execute( remove_pending_contents_cruft_q )
+ cursor.execute( remove_filename_cruft_q )
+ cursor.execute( remove_filepath_cruft_q )
+ cursor.execute( "COMMIT" )
+
+
+ def bootstrap(self):
+ """
+ scan the existing debs in the pool to populate the contents database tables
+ """
+ pooldir = Config()[ 'Dir::Pool' ]
+
+ cursor = DBConn().cursor();
+ DBConn().prepare("debs_q",debs_q)
+ DBConn().prepare("olddeb_q",olddeb_q)
+ DBConn().prepare("arches_q",arches_q)
+
+ suites = self._suites()
+ for suite in [i.lower() for i in suites]:
+ suite_id = DBConn().get_suite_id(suite)
+
+ arch_list = self._arches(cursor, suite_id)
+ arch_all_id = DBConn().get_architecture_id("all")
+ for arch_id in arch_list:
+ cursor.execute( "EXECUTE debs_q(%d, %d)" % ( suite_id, arch_id[0] ) )
+
+ count = 0
+ while True:
+ deb = cursor.fetchone()
+ if not deb:
+ break
+ count += 1
+ cursor1 = DBConn().cursor();
+ cursor1.execute( "EXECUTE olddeb_q(%d)" % (deb[0] ) )
+ old = cursor1.fetchone()
+ if old:
+ log.debug( "already imported: %s" % (deb[1]) )
+ else:
+ log.debug( "scanning: %s" % (deb[1]) )
+ debfile = os.path.join( pooldir, deb[1] )
+ if os.path.exists( debfile ):
+ Binary(debfile, self.reject).scan_package(deb[0],True)
+ else:
+ log.error("missing .deb: %s" % deb[1])
+
+ def generate(self):
+ """
+ Generate Contents-$arch.gz files for every available arch in each given suite.
+ """
+ cursor = DBConn().cursor();
+
+ DBConn().prepare( "arches_q", arches_q )
+ DBConn().prepare( "contents_q", contents_q )
+ DBConn().prepare( "udeb_contents_q", udeb_contents_q )
+
+ debtype_id=DBConn().get_override_type_id("deb")
+ udebtype_id=DBConn().get_override_type_id("udeb")
+
+ suites = self._suites()
+
+ # Get our suites, and the architectures
+ for suite in [i.lower() for i in suites]:
+ suite_id = DBConn().get_suite_id(suite)
+ arch_list = self._arches(cursor, suite_id)
+
+ arch_all_id = DBConn().get_architecture_id("all")
+
+ for arch_id in arch_list:
+ cursor.execute("EXECUTE contents_q(%d,%d,%d,%d)" % (arch_id[0], arch_all_id, suite_id, debtype_id))
+ self._write_content_file(cursor, "dists/%s/Contents-%s.gz" % (suite, arch_id[1]))
+
+ # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
+ # This is HORRIBLY debian specific :-/
+ for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"),
+ ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
+
+ for arch_id in arch_list:
+ section_id = DBConn().get_section_id(section) # all udebs should be here)
+ if section_id != -1:
+ cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d,%d,%d)" % (arch_id[0], arch_all_id, section_id, suite_id, udebtype_id))
+
+ self._write_content_file(cursor, fn_pattern % (suite, arch_id[1]))
+
+
+################################################################################
+
+ def _suites(self):
+ """
+ return a list of suites to operate on
+ """
+ if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
+ suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
+ else:
+ suites = Config().SubTree("Suite").List()
+
+ return suites
+
+ def _arches(self, cursor, suite):
+ """
+ return a list of archs to operate on
+ """
+ arch_list = [ ]
+ if Config().has_key( "%s::%s" %(options_prefix,"Arch")):
+ archs = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Arch")])
+ for arch_name in archs:
+ arch_list.append((DBConn().get_architecture_id(arch_name), arch_name))
+ else:
+ cursor.execute("EXECUTE arches_q(%d)" % (suite))
+ while True:
+ r = cursor.fetchone()
+ if not r:
+ break
+
+ if r[1] != "source" and r[1] != "all":
+ arch_list.append((r[0], r[1]))
+
+ return arch_list
+
+################################################################################
+
+def main():
+ cnf = Config()
+
+ arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
+ ('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"),
+ ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
+ ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
+ ('a',"arch", "%s::%s" % (options_prefix,"Arch"),"HasArg"),
+ ]
+
+ commands = {'generate' : Contents.generate,
+ 'bootstrap' : Contents.bootstrap,
+ 'cruft' : Contents.cruft,
+ }
+
+ args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments,sys.argv)
+
+ if (len(args) < 1) or not commands.has_key(args[0]):
+ usage()
+
+ if cnf.has_key("%s::%s" % (options_prefix,"Help")):
+ usage()
+
+ level=logging.INFO
+ if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
+ level=logging.ERROR
+
+ elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
+ level=logging.DEBUG
+
+
+ logging.basicConfig( level=level,
+ format='%(asctime)s %(levelname)s %(message)s',
+ stream = sys.stderr )
+
+ commands[args[0]](Contents())
+
+if __name__ == '__main__':
+ main()
if action == "list":
list_overrides(suite, component, otype)
else:
- if Cnf.has_key("Suite::%s::Untouchable" % suite) and Cnf["Suite::%s::Untouchable" % suite] != 0:
+ if database.get_suite_untouchable(suite):
utils.fubar("%s: suite is untouchable" % suite)
noaction = 0
################################################################################
-import sys, imp
-import daklib.utils, daklib.extensions
+import sys
+import imp
+import daklib.utils
+import daklib.extensions
################################################################################
"Generate package <-> file mapping"),
("generate-releases",
"Generate Release files"),
+ ("contents",
+ "Generate content files"),
("generate-index-diffs",
"Generate .diff/Index files"),
("clean-suites",
#!/usr/bin/env python
-
"""
Database Update Script - Get suite_architectures table use sane values
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2009 Joerg Jaspert <joerg@debian.org>
@license: GNU General Public License version 2 or later
-
"""
# This program is free software; you can redistribute it and/or modify
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Debian Archive Kit Database Update Script
+Copyright © 2008 Michael Casadevall <mcasadevall@debian.org>
+Copyright © 2008 Roger Leigh <rleigh@debian.org>
+
+Debian Archive Kit Database Update Script 2
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <tomv_w> really, if we want to screw ourselves, let's find a better way.
+# <Ganneff> rm -rf /srv/ftp.debian.org
+
+################################################################################
+
+import psycopg2
+import time
+
+################################################################################
+
+def do_update(self):
+ print "Adding content fields to database"
+
+ try:
+ c = self.db.cursor()
+ c.execute("""CREATE TABLE content_file_paths (
+ id serial primary key not null,
+ path text unique not null
+ )""")
+
+ c.execute("""CREATE TABLE content_file_names (
+ id serial primary key not null,
+ file text unique not null
+ )""")
+
+ c.execute("""CREATE TABLE content_associations (
+ id serial not null,
+ binary_pkg int4 not null references binaries(id) on delete cascade,
+ filepath int4 not null references content_file_paths(id) on delete cascade,
+ filename int4 not null references content_file_names(id) on delete cascade
+ );""")
+
+ c.execute("""CREATE TABLE pending_content_associations (
+ id serial not null,
+ package text not null,
+ version debversion not null,
+ filepath int4 not null references content_file_paths(id) on delete cascade,
+ filename int4 not null references content_file_names(id) on delete cascade
+ );""")
+
+ c.execute("""CREATE FUNCTION comma_concat(text, text) RETURNS text
+ AS $_$select case
+ WHEN $2 is null or $2 = '' THEN $1
+ WHEN $1 is null or $1 = '' THEN $2
+ ELSE $1 || ',' || $2
+ END$_$
+ LANGUAGE sql""")
+
+ c.execute("""CREATE AGGREGATE comma_separated_list (
+ BASETYPE = text,
+ SFUNC = comma_concat,
+ STYPE = text,
+ INITCOND = ''
+ );""")
+
+ c.execute( "CREATE INDEX content_assocaitions_binary ON content_associations(binary_pkg)" )
+
+ c.execute("UPDATE config SET value = '6' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to appy debversion updates, rollback issued. Error message : %s" % (str(msg))
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Debian Archive Kit Database Update Script
+Copyright © 2008 Michael Casadevall <mcasadevall@debian.org>
+Copyright © 2009 Joerg Jaspert <joerg@debian.org>
+
+Debian Archive Kit Database Update Script 7
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# * Ganneff ponders how to best write the text to -devel. (need to tell em in
+# case they find more bugs). "We fixed the fucking idiotic broken implementation
+# to be less so" is probably not the nicest, even if perfect valid, way to say so
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+from daklib.utils import get_conf
+
+################################################################################
+
+def do_update(self):
+ print "Moving some of the suite config into the DB"
+ Cnf = get_conf()
+
+ try:
+ c = self.db.cursor()
+
+ c.execute("ALTER TABLE suite ADD COLUMN untouchable BOOLEAN NOT NULL DEFAULT FALSE;")
+ query = "UPDATE suite SET untouchable = TRUE WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ untouchable = Cnf.Find("Suite::%s::Untouchable" % (suite))
+ if not untouchable:
+ continue
+ print "[Untouchable] Processing suite %s" % (suite)
+ suite = suite.lower()
+ c.execute(query, [suite])
+
+
+ c.execute("ALTER TABLE suite ADD COLUMN announce text NOT NULL DEFAULT 'debian-devel-changes@lists.debian.org';")
+ query = "UPDATE suite SET announce = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ announce_list = Cnf.Find("Suite::%s::Announce" % (suite))
+ print "[Announce] Processing suite %s" % (suite)
+ suite = suite.lower()
+ c.execute(query, [announce_list, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN codename text;")
+ query = "UPDATE suite SET codename = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ codename = Cnf.Find("Suite::%s::CodeName" % (suite))
+ print "[Codename] Processing suite %s" % (suite)
+ suite = suite.lower()
+ c.execute(query, [codename, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN overridecodename text;")
+ query = "UPDATE suite SET overridecodename = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ codename = Cnf.Find("Suite::%s::OverrideCodeName" % (suite))
+ print "[OverrideCodeName] Processing suite %s" % (suite)
+ suite = suite.lower()
+ c.execute(query, [codename, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN validtime integer NOT NULL DEFAULT 604800;")
+ query = "UPDATE suite SET validtime = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ validtime = Cnf.Find("Suite::%s::ValidTime" % (suite))
+ print "[ValidTime] Processing suite %s" % (suite)
+ if not validtime:
+ validtime = 0
+ suite = suite.lower()
+ c.execute(query, [validtime, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN priority integer NOT NULL DEFAULT 0;")
+ query = "UPDATE suite SET priority = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ priority = Cnf.Find("Suite::%s::Priority" % (suite))
+ print "[Priority] Processing suite %s" % (suite)
+ if not priority:
+ priority = 0
+ suite = suite.lower()
+ c.execute(query, [priority, suite])
+
+
+ c.execute("ALTER TABLE suite ADD COLUMN notautomatic BOOLEAN NOT NULL DEFAULT FALSE;")
+ query = "UPDATE suite SET notautomatic = TRUE WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ notautomatic = Cnf.Find("Suite::%s::NotAutomatic" % (suite))
+ print "[NotAutomatic] Processing suite %s" % (suite)
+ if not notautomatic:
+ continue
+ suite = suite.lower()
+ c.execute(query, [suite])
+
+ c.execute("UPDATE config SET value = '7' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to appy suite config updates, rollback issued. Error message : %s" % (str(msg))
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Debian Archive Kit Database Update Script
+Copyright © 2008 Michael Casadevall <mcasadevall@debian.org>
+Copyright © 2009 Joerg Jaspert <joerg@debian.org>
+
+Debian Archive Kit Database Update Script 8
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# * Ganneff ponders how to best write the text to -devel. (need to tell em in
+# case they find more bugs). "We fixed the fucking idiotic broken implementation
+# to be less so" is probably not the nicest, even if perfect valid, way to say so
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+from daklib.utils import get_conf
+
+################################################################################
+
+def do_update(self):
+ print "Moving some more of the suite config into the DB"
+ Cnf = get_conf()
+
+ try:
+ c = self.db.cursor()
+
+ c.execute("ALTER TABLE suite ADD COLUMN copychanges TEXT;")
+ query = "UPDATE suite SET copychanges = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ copychanges = Cnf.Find("Suite::%s::CopyChanges" % (suite))
+ print "[CopyChanges] Processing suite %s" % (suite)
+ if not copychanges:
+ continue
+ suite = suite.lower()
+ c.execute(query, [copychanges, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN copydotdak TEXT;")
+ query = "UPDATE suite SET copydotdak = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ copydotdak = Cnf.Find("Suite::%s::CopyDotDak" % (suite))
+ print "[CopyDotDak] Processing suite %s" % (suite)
+ if not copydotdak:
+ continue
+ suite = suite.lower()
+ c.execute(query, [copydotdak, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN commentsdir TEXT;")
+ query = "UPDATE suite SET commentsdir = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ commentsdir = Cnf.Find("Suite::%s::CommentsDir" % (suite))
+ print "[CommentsDir] Processing suite %s" % (suite)
+ if not commentsdir:
+ continue
+ suite = suite.lower()
+ c.execute(query, [commentsdir, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN overridesuite TEXT;")
+ query = "UPDATE suite SET overridesuite = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ overridesuite = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
+ print "[OverrideSuite] Processing suite %s" % (suite)
+ if not overridesuite:
+ continue
+ suite = suite.lower()
+ c.execute(query, [overridesuite, suite])
+
+ c.execute("ALTER TABLE suite ADD COLUMN changelogbase TEXT;")
+ query = "UPDATE suite SET changelogbase = %s WHERE suite_name = %s" #: Update query
+ for suite in Cnf.SubTree("Suite").List():
+ changelogbase = Cnf.Find("Suite::%s::ChangeLogBase" % (suite))
+ print "[ChangeLogBase] Processing suite %s" % (suite)
+ if not changelogbase:
+ continue
+ suite = suite.lower()
+ c.execute(query, [changelogbase, suite])
+
+ c.execute("UPDATE config SET value = '8' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg))
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Debian Archive Kit Database Update Script
+Copyright © 2008 Michael Casadevall <mcasadevall@debian.org>
+Copyright © 2009 Mike O'Connor <stew@debian.org>
+
+Debian Archive Kit Database Update Script 8
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# * Ganneff ponders how to best write the text to -devel. (need to tell em in
+# case they find more bugs). "We fixed the fucking idiotic broken implementation
+# to be less so" is probably not the nicest, even if perfect valid, way to say so
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+from daklib.utils import get_conf
+
+################################################################################
+
+def do_update(self):
+ print "pending_contents should distinguish by arch"
+ Cnf = get_conf()
+
+ try:
+ c = self.db.cursor()
+
+ c.execute("DELETE FROM pending_content_associations")
+ c.execute("""ALTER TABLE pending_content_associations
+ ADD COLUMN architecture integer NOT NULL""")
+ c.execute("""ALTER TABLE ONLY pending_content_associations
+ ADD CONSTRAINT pending_content_assiciations_arch
+ FOREIGN KEY (architecture)
+ REFERENCES architecture(id)
+ ON DELETE CASCADE""")
+ c.execute("UPDATE config SET value = '9' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg))
#!/usr/bin/env python
-""" Script to automate some parts of checking NEW packages """
-# Copyright (C) 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
+"""
+Script to automate some parts of checking NEW packages
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-import errno, os, pg, re, sys, md5
+import errno
+import os
+import pg
+import re
+import sys
+import md5
import apt_pkg, apt_inst
from daklib import database
from daklib import utils
database.init(Cnf, projectB)
printed_copyrights = {}
+package_relations = {} #: Store relations of packages for later output
# default is to not output html.
use_html = 0
comma_count += 1
return result
-def output_deb_info(suite, filename):
+def output_package_relations ():
+ """
+ Output the package relations, if there is more than one package checked in this run.
+ """
+
+ if len(package_relations) < 2:
+ # Only list something if we have more than one binary to compare
+ return
+
+ to_print = ""
+ for package in package_relations:
+ for relation in package_relations[package]:
+ to_print += "%-15s: (%s) %s\n" % (package, relation, package_relations[package][relation])
+
+ package_relations.clear()
+ foldable_output("Package relations", "relations", to_print)
+
+def output_deb_info(suite, filename, packagename):
(control, control_keys, section, depends, recommends, arch, maintainer) = read_control(filename)
if control == '':
return formatted_text("no control info")
to_print = ""
+ if not package_relations.has_key(packagename):
+ package_relations[packagename] = {}
for key in control_keys :
if key == 'Depends':
field_value = create_depends_string(suite, depends)
+ package_relations[packagename][key] = field_value
elif key == 'Recommends':
field_value = create_depends_string(suite, recommends)
+ package_relations[packagename][key] = field_value
elif key == 'Section':
field_value = section
elif key == 'Architecture':
foldable_output("control file for %s" % (filename), "binary-%s-control"%packagename,
- output_deb_info(suite, deb_filename), norow=True)
+ output_deb_info(suite, deb_filename, packagename), norow=True)
if is_a_udeb:
foldable_output("skipping lintian check for udeb", "binary-%s-lintian"%packagename,
else:
utils.fubar("Unrecognised file type: '%s'." % (f))
finally:
+ output_package_relations()
if not Options["Html-Output"]:
# Reset stdout here so future less invocations aren't FUBAR
less_fd.close()
print "Processing: " + suite
SuiteBlock = Cnf.SubTree("Suite::" + suite)
- if SuiteBlock.has_key("Untouchable"):
+ if database.get_suite_untouchable(suite):
print "Skipping: " + suite + " (untouchable)"
continue
def print_sha256_files (tree, files):
print_md5sha_files (tree, files, apt_pkg.sha256sum)
+def write_release_file (relpath, suite, component, origin, label, arch, version="", suite_suffix="", notautomatic=""):
+ try:
+ if os.access(relpath, os.F_OK):
+ if os.stat(relpath).st_nlink > 1:
+ os.unlink(relpath)
+ release = open(relpath, "w")
+ except IOError:
+ utils.fubar("Couldn't write to " + relpath)
+
+ release.write("Archive: %s\n" % (suite))
+ if version != "":
+ release.write("Version: %s\n" % (version))
+
+ if suite_suffix:
+ release.write("Component: %s/%s\n" % (suite_suffix,component))
+ else:
+ release.write("Component: %s\n" % (component))
+
+ release.write("Origin: %s\n" % (origin))
+ release.write("Label: %s\n" % (label))
+ if notautomatic != "":
+ release.write("NotAutomatic: %s\n" % (notautomatic))
+ release.write("Architecture: %s\n" % (arch))
+ release.close()
+
################################################################################
def main ():
print "Processing: " + suite
SuiteBlock = Cnf.SubTree("Suite::" + suite)
- if SuiteBlock.has_key("Untouchable") and not Options["Force-Touch"]:
+ if database.get_suite_untouchable(suite) and not Options["Force-Touch"]:
print "Skipping: " + suite + " (untouchable)"
continue
else:
rel = "%s/binary-%s/Release" % (sec, arch)
relpath = Cnf["Dir::Root"]+tree+"/"+rel
-
- try:
- if os.access(relpath, os.F_OK):
- if os.stat(relpath).st_nlink > 1:
- os.unlink(relpath)
- release = open(relpath, "w")
- #release = open(longsuite.replace("/","_") + "_" + arch + "_" + sec + "_Release", "w")
- except IOError:
- utils.fubar("Couldn't write to " + relpath)
-
- release.write("Archive: %s\n" % (suite))
- if version != "":
- release.write("Version: %s\n" % (version))
- if suite_suffix:
- release.write("Component: %s/%s\n" % (suite_suffix,sec))
- else:
- release.write("Component: %s\n" % (sec))
- release.write("Origin: %s\n" % (origin))
- release.write("Label: %s\n" % (label))
- if notautomatic != "":
- release.write("NotAutomatic: %s\n" % (notautomatic))
- release.write("Architecture: %s\n" % (arch))
- release.close()
+ write_release_file(relpath, suite, sec, origin, label, arch, version, suite_suffix, notautomatic)
files.append(rel)
if AptCnf.has_key("tree::%s/main" % (tree)):
for arch in AptCnf["tree::%s/%s::Architectures" % (tree,dis)].split():
if arch != "source": # always true
+ rel = "%s/%s/binary-%s/Release" % (dis, sec, arch)
+ relpath = Cnf["Dir::Root"]+tree+"/"+rel
+ write_release_file(relpath, suite, dis, origin, label, arch, version, suite_suffix, notautomatic)
+ files.append(rel)
for cfile in compressnames("tree::%s/%s" % (tree,dis),
"Packages",
"%s/%s/binary-%s/Packages" % (dis, sec, arch)):
################################################################################
-import pg, sys
+import psycopg2, sys
import apt_pkg
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None
-projectB = None
+from daklib import utils
+from daklib.dbconn import DBConn
+from daklib.config import Config
################################################################################
################################################################################
def sql_get (config, key):
- """Return the value of config[key] in quotes or NULL if it doesn't exist."""
+ """Return the value of config[key] or None if it doesn't exist."""
- if config.has_key(key):
- return "'%s'" % (config[key])
- else:
- return "NULL"
+ try:
+ return config[key]
+ except KeyError:
+ return None
################################################################################
-def do_archive():
- """Initalize the archive table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM archive")
- for name in Cnf.SubTree("Archive").List():
- archive_config = Cnf.SubTree("Archive::%s" % (name))
- origin_server = sql_get(archive_config, "OriginServer")
- description = sql_get(archive_config, "Description")
- projectB.query("INSERT INTO archive (name, origin_server, description) "
- "VALUES ('%s', %s, %s)"
- % (name, origin_server, description))
- projectB.query("COMMIT WORK")
-
-def do_architecture():
- """Initalize the architecture table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM architecture")
- for arch in Cnf.SubTree("Architectures").List():
- description = Cnf["Architectures::%s" % (arch)]
- projectB.query("INSERT INTO architecture (arch_string, description) "
- "VALUES ('%s', '%s')" % (arch, description))
- projectB.query("COMMIT WORK")
-
-def do_component():
- """Initalize the component table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM component")
- for name in Cnf.SubTree("Component").List():
- component_config = Cnf.SubTree("Component::%s" % (name))
- description = sql_get(component_config, "Description")
- if component_config.get("MeetsDFSG").lower() == "true":
- meets_dfsg = "true"
- else:
- meets_dfsg = "false"
- projectB.query("INSERT INTO component (name, description, meets_dfsg) "
- "VALUES ('%s', %s, %s)"
- % (name, description, meets_dfsg))
- projectB.query("COMMIT WORK")
-
-def do_location():
- """Initalize the location table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM location")
- for location in Cnf.SubTree("Location").List():
- location_config = Cnf.SubTree("Location::%s" % (location))
- archive_id = database.get_archive_id(location_config["Archive"])
- if archive_id == -1:
- utils.fubar("Archive '%s' for location '%s' not found."
- % (location_config["Archive"], location))
- location_type = location_config.get("type")
- if location_type == "pool":
- for component in Cnf.SubTree("Component").List():
- component_id = database.get_component_id(component)
- projectB.query("INSERT INTO location (path, component, "
- "archive, type) VALUES ('%s', %d, %d, '%s')"
- % (location, component_id, archive_id,
- location_type))
- else:
- utils.fubar("E: type '%s' not recognised in location %s."
- % (location_type, location))
- projectB.query("COMMIT WORK")
-
-def do_suite():
- """Initalize the suite table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM suite")
- for suite in Cnf.SubTree("Suite").List():
- suite_config = Cnf.SubTree("Suite::%s" %(suite))
- version = sql_get(suite_config, "Version")
- origin = sql_get(suite_config, "Origin")
- description = sql_get(suite_config, "Description")
- projectB.query("INSERT INTO suite (suite_name, version, origin, "
- "description) VALUES ('%s', %s, %s, %s)"
- % (suite.lower(), version, origin, description))
- for architecture in Cnf.SubTree("Architectures").List():
- architecture_id = database.get_architecture_id (architecture)
- if architecture_id < 0:
- utils.fubar("architecture '%s' not found in architecture"
- " table for suite %s."
- % (architecture, suite))
- projectB.query("INSERT INTO suite_architectures (suite, "
- "architecture) VALUES (currval('suite_id_seq'), %d)"
- % (architecture_id))
- projectB.query("COMMIT WORK")
-
-def do_override_type():
- """Initalize the override_type table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM override_type")
- for override_type in Cnf.ValueList("OverrideType"):
- projectB.query("INSERT INTO override_type (type) VALUES ('%s')"
- % (override_type))
- projectB.query("COMMIT WORK")
-
-def do_priority():
- """Initialize the priority table."""
-
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM priority")
- for priority in Cnf.SubTree("Priority").List():
- projectB.query("INSERT INTO priority (priority, level) VALUES "
- "('%s', %s)"
- % (priority, Cnf["Priority::%s" % (priority)]))
- projectB.query("COMMIT WORK")
-
-def do_section():
- """Initalize the section table."""
- projectB.query("BEGIN WORK")
- projectB.query("DELETE FROM section")
- for component in Cnf.SubTree("Component").List():
- if Cnf["Control-Overrides::ComponentPosition"] == "prefix":
- suffix = ""
- if component != "main":
- prefix = component + '/'
- else:
- prefix = ""
- else:
- prefix = ""
- if component != "main":
- suffix = '/' + component
+class InitDB(object):
+ def __init__(self, Cnf, projectB):
+ self.Cnf = Cnf
+ self.projectB = projectB
+
+ def do_archive(self):
+ """initalize the archive table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM archive")
+ archive_add = "INSERT INTO archive (name, origin_server, description) VALUES (%s, %s, %s)"
+ for name in self.Cnf.SubTree("Archive").List():
+ archive_config = self.Cnf.SubTree("Archive::%s" % (name))
+ origin_server = sql_get(archive_config, "OriginServer")
+ description = sql_get(archive_config, "Description")
+ c.execute(archive_add, [name, origin_server, description])
+ self.projectB.commit()
+
+ def do_architecture(self):
+ """Initalize the architecture table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM architecture")
+ arch_add = "INSERT INTO architecture (arch_string, description) VALUES (%s, %s)"
+ for arch in self.Cnf.SubTree("Architectures").List():
+ description = self.Cnf["Architectures::%s" % (arch)]
+ c.execute(arch_add, [arch, description])
+ self.projectB.commit()
+
+ def do_component(self):
+ """Initalize the component table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM component")
+
+ comp_add = "INSERT INTO component (name, description, meets_dfsg) " + \
+ "VALUES (%s, %s, %s)"
+
+ for name in self.Cnf.SubTree("Component").List():
+ component_config = self.Cnf.SubTree("Component::%s" % (name))
+ description = sql_get(component_config, "Description")
+ meets_dfsg = (component_config.get("MeetsDFSG").lower() == "true")
+ c.execute(comp_add, [name, description, meets_dfsg])
+
+ self.projectB.commit()
+
+ def do_location(self):
+ """Initalize the location table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM location")
+
+ loc_add = "INSERT INTO location (path, component, archive, type) " + \
+ "VALUES (%s, %s, %s, %s)"
+
+ for location in self.Cnf.SubTree("Location").List():
+ location_config = self.Cnf.SubTree("Location::%s" % (location))
+ archive_id = self.projectB.get_archive_id(location_config["Archive"])
+ if archive_id == -1:
+ utils.fubar("Archive '%s' for location '%s' not found."
+ % (location_config["Archive"], location))
+ location_type = location_config.get("type")
+ if location_type == "pool":
+ for component in self.Cnf.SubTree("Component").List():
+ component_id = self.projectB.get_component_id(component)
+ c.execute(loc_add, [location, component_id, archive_id, location_type])
else:
+ utils.fubar("E: type '%s' not recognised in location %s."
+ % (location_type, location))
+
+ self.projectB.commit()
+
+ def do_suite(self):
+ """Initalize the suite table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM suite")
+
+ suite_add = "INSERT INTO suite (suite_name, version, origin, description) " + \
+ "VALUES (%s, %s, %s, %s)"
+
+ sa_add = "INSERT INTO suite_architectures (suite, architecture) " + \
+ "VALUES (currval('suite_id_seq'), %s)"
+
+ for suite in self.Cnf.SubTree("Suite").List():
+ suite_config = self.Cnf.SubTree("Suite::%s" %(suite))
+ version = sql_get(suite_config, "Version")
+ origin = sql_get(suite_config, "Origin")
+ description = sql_get(suite_config, "Description")
+ c.execute(suite_add, [suite.lower(), version, origin, description])
+ for architecture in self.Cnf.SubTree("Architectures").List():
+ architecture_id = self.projectB.get_architecture_id (architecture)
+ if architecture_id < 0:
+ utils.fubar("architecture '%s' not found in architecture"
+ " table for suite %s."
+ % (architecture, suite))
+ c.execute(sa_add, [architecture_id])
+
+ self.projectB.commit()
+
+ def do_override_type(self):
+ """Initalize the override_type table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM override_type")
+
+ over_add = "INSERT INTO override_type (type) VALUES (%s)"
+
+ for override_type in self.Cnf.ValueList("OverrideType"):
+ c.execute(over_add, [override_type])
+
+ self.projectB.commit()
+
+ def do_priority(self):
+ """Initialize the priority table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM priority")
+
+ prio_add = "INSERT INTO priority (priority, level) VALUES (%s, %s)"
+
+ for priority in self.Cnf.SubTree("Priority").List():
+ c.execute(prio_add, [priority, self.Cnf["Priority::%s" % (priority)]])
+
+ self.projectB.commit()
+
+ def do_section(self):
+ """Initalize the section table."""
+
+ c = self.projectB.cursor()
+ c.execute("DELETE FROM section")
+
+ sect_add = "INSERT INTO section (section) VALUES (%s)"
+
+ for component in self.Cnf.SubTree("Component").List():
+ if self.Cnf["Control-Overrides::ComponentPosition"] == "prefix":
suffix = ""
- for section in Cnf.ValueList("Section"):
- projectB.query("INSERT INTO section (section) VALUES "
- "('%s%s%s')" % (prefix, section, suffix))
- projectB.query("COMMIT WORK")
+ if component != "main":
+ prefix = component + '/'
+ else:
+ prefix = ""
+ else:
+ prefix = ""
+ if component != "main":
+ suffix = '/' + component
+ else:
+ suffix = ""
+ for section in self.Cnf.ValueList("Section"):
+ c.execute(sect_add, [prefix + section + suffix])
+
+ self.projectB.commit()
+
+ def do_all(self):
+ self.do_archive()
+ self.do_architecture()
+ self.do_component()
+ self.do_location()
+ self.do_suite()
+ self.do_override_type()
+ self.do_priority()
+ self.do_section()
################################################################################
def main ():
"""Sync dak.conf configuartion file and the SQL database"""
- global Cnf, projectB
-
Cnf = utils.get_conf()
arguments = [('h', "help", "Init-DB::Options::Help")]
for i in [ "help" ]:
utils.warn("dak init-db takes no arguments.")
usage(exit_code=1)
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"],
- int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
- do_archive()
- do_architecture()
- do_component()
- do_location()
- do_suite()
- do_override_type()
- do_priority()
- do_section()
+ # Just let connection failures be reported to the user
+ projectB = DBConn()
+ Cnf = Config()
+
+ InitDB(Cnf, projectB).do_all()
################################################################################
database.init(Cnf, projectB)
for suite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
- if Cnf.has_key("Suite::%s::Untouchable" % suite) and Cnf["Suite::%s::Untouchable" % suite] != 0:
+ if database.get_suite_untouchable(suite):
continue
suite = suite.lower()
delete_version = version[0]
delete_id = packages[delete_unique_id]["sourceid"]
delete_arch = packages[delete_unique_id]["arch"]
- if not Cnf.Find("Suite::%s::Untouchable" % (suite)) or Options["Force"]:
+ if not database.get_suite_untouchable(suite) or Options["Force"]:
if Options["No-Delete"]:
print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch)
else:
###############################################################################
-import errno, fcntl, os, sys, time, re
-import apt_pkg
+import errno
+import fcntl
+import os
+import sys
+import time
+import re
+import apt_pkg, commands
from daklib import database
from daklib import logging
from daklib import queue
else:
os.unlink(self.log_filename)
+
###############################################################################
+
def reject (str, prefix="Rejected: "):
global reject_message
if str:
suite_id = database.get_suite_id(suite)
projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id))
+ if not database.copy_temporary_contents(package, version, architecture, newfile, reject):
+ print "REJECT\n" + reject_message,
+ projectB.query("ROLLBACK")
+ raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (package, newfile )
+
+
orig_tar_id = Upload.pkg.orig_tar_id
orig_tar_location = Upload.pkg.orig_tar_location
utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest)
for dest in copy_dot_dak.keys():
utils.copy(Upload.pkg.changes_file[:-8]+".dak", dest)
-
projectB.query("COMMIT WORK")
# Move the .changes into the 'done' directory
elif ftype == "dsc":
examine_package.check_dsc(changes['distribution'], f)
finally:
+ examine_package.output_package_relations()
sys.stdout = stdout_fd
except IOError, e:
if e.errno == errno.EPIPE:
return
Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
Upload.dump_vars(queue_dir)
- move_to_dir(queue_dir)
+ move_to_dir(queue_dir, perms=0664)
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
def _accept():
################################################################################
-import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback
-import apt_inst, apt_pkg
-from daklib import database
+import commands
+import errno
+import fcntl
+import os
+import re
+import shutil
+import stat
+import sys
+import time
+import traceback
+import tarfile
+import apt_inst
+import apt_pkg
+from debian_bundle import deb822
+from daklib.dbconn import DBConn
+from daklib.binary import Binary
from daklib import logging
from daklib import queue
from daklib import utils
(source, dest) = args[1:3]
if changes["distribution"].has_key(source):
for arch in changes["architecture"].keys():
- if arch not in database.get_suite_architectures(source):
+ if arch not in DBConn().get_suite_architectures(source):
reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),"")
del changes["distribution"][source]
changes["distribution"][dest] = 1
################################################################################
-def check_deb_ar(filename):
- """
- Sanity check the ar of a .deb, i.e. that there is:
-
- 1. debian-binary
- 2. control.tar.gz
- 3. data.tar.gz or data.tar.bz2
-
- in that order, and nothing else.
- """
- cmd = "ar t %s" % (filename)
- (result, output) = commands.getstatusoutput(cmd)
- if result != 0:
- reject("%s: 'ar t' invocation failed." % (filename))
- reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
- chunks = output.split('\n')
- if len(chunks) != 3:
- reject("%s: found %d chunks, expected 3." % (filename, len(chunks)))
- if chunks[0] != "debian-binary":
- reject("%s: first chunk is '%s', expected 'debian-binary'." % (filename, chunks[0]))
- if chunks[1] != "control.tar.gz":
- reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (filename, chunks[1]))
- if chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
- reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (filename, chunks[2]))
-
-################################################################################
-
def check_files():
global reprocess
has_binaries = 0
has_source = 0
+ cursor = DBConn().cursor()
+ # Check for packages that have moved from one component to another
+ # STU: this should probably be changed to not join on architecture, suite tables but instead to used their cached name->id mappings from DBConn
+ DBConn().prepare("moved_pkg_q", """
+ PREPARE moved_pkg_q(text,text,text) AS
+ SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
+ component c, architecture a, files f
+ WHERE b.package = $1 AND s.suite_name = $2
+ AND (a.arch_string = $3 OR a.arch_string = 'all')
+ AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
+ AND f.location = l.id
+ AND l.component = c.id
+ AND b.file = f.id""")
+
for f in file_keys:
# Ensure the file does not already exist in one of the accepted directories
for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable")
architecture = control.Find("Architecture")
upload_suite = changes["distribution"].keys()[0]
- if architecture not in database.get_suite_architectures(default_suite) and architecture not in database.get_suite_architectures(upload_suite):
+ if architecture not in DBConn().get_suite_architectures(default_suite) and architecture not in DBConn().get_suite_architectures(upload_suite):
reject("Unknown architecture '%s'." % (architecture))
# Ensure the architecture of the .deb is one of the ones
# Check the version and for file overwrites
reject(Upload.check_binary_against_db(f),"")
- check_deb_ar(f)
+ Binary(f, reject).scan_package()
# Checks for a source package...
else:
# Validate the component
component = files[f]["component"]
- component_id = database.get_component_id(component)
+ component_id = DBConn().get_component_id(component)
if component_id == -1:
reject("file '%s' has unknown component '%s'." % (f, component))
continue
# Determine the location
location = Cnf["Dir::Pool"]
- location_id = database.get_location_id (location, component, archive)
+ location_id = DBConn().get_location_id(location, component, archive)
if location_id == -1:
reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
files[f]["location id"] = location_id
# Check the md5sum & size against existing files (if any)
files[f]["pool name"] = utils.poolify (changes["source"], files[f]["component"])
- files_id = database.get_files_id(files[f]["pool name"] + f, files[f]["size"], files[f]["md5sum"], files[f]["location id"])
+ files_id = DBConn().get_files_id(files[f]["pool name"] + f, files[f]["size"], files[f]["md5sum"], files[f]["location id"])
if files_id == -1:
reject("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
elif files_id == -2:
files[f]["files id"] = files_id
# Check for packages that have moved from one component to another
- q = Upload.projectB.query("""
-SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
- component c, architecture a, files f
- WHERE b.package = '%s' AND s.suite_name = '%s'
- AND (a.arch_string = '%s' OR a.arch_string = 'all')
- AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
- AND f.location = l.id AND l.component = c.id AND b.file = f.id"""
- % (files[f]["package"], suite,
- files[f]["architecture"]))
- ql = q.getresult()
+ files[f]['suite'] = suite
+ cursor.execute("""EXECUTE moved_pkg_q( %(package)s, %(suite)s, %(architecture)s )""", ( files[f] ) )
+ ql = cursor.fetchone()
if ql:
files[f]["othercomponents"] = ql[0][0]
or pkg.orig_tar_gz == -1:
return
- # Create a temporary directory to extract the source into
- if Options["No-Action"]:
- tmpdir = tempfile.mkdtemp()
- else:
- # We're in queue/holding and can create a random directory.
- tmpdir = "%s" % (os.getpid())
- os.mkdir(tmpdir)
+ tmpdir = utils.temp_dirname()
# Move into the temporary directory
cwd = os.getcwd()
################################################################################
def lookup_uid_from_fingerprint(fpr):
- q = Upload.projectB.query("SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
- qs = q.getresult()
- if len(qs) == 0:
- return (None, None, None)
+ """
+ Return the uid,name,isdm for a given gpg fingerprint
+
+ @ptype fpr: string
+ @param fpr: a 40 byte GPG fingerprint
+
+ @return (uid, name, isdm)
+ """
+ cursor = DBConn().cursor()
+ cursor.execute( "SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
+ qs = cursor.fetchone()
+ if qs:
+ return qs
else:
- return qs[0]
+ return (None, None, None)
def check_signed_by_key():
"""Ensure the .changes is signed by an authorized uploader."""
if not sponsored and not may_nmu:
source_ids = []
- q = Upload.projectB.query("SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.dm_upload_allowed = 'yes'" % (changes["source"]))
+ cursor.execute( "SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = %(source)s AND s.dm_upload_allowed = 'yes'", changes )
highest_sid, highest_version = None, None
should_reject = True
- for si in q.getresult():
+ while True:
+ si = cursor.fetchone()
+ if not si:
+ break
+
if highest_version == None or apt_pkg.VersionCompare(si[1], highest_version) == 1:
highest_sid = si[0]
highest_version = si[1]
if highest_sid == None:
reject("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % changes["source"])
else:
- q = Upload.projectB.query("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT su.maintainer FROM src_uploaders su JOIN source s ON (s.id = su.source) WHERE su.source = %s)" % (highest_sid))
- for m in q.getresult():
+
+ cursor.execute("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT su.maintainer FROM src_uploaders su JOIN source s ON (s.id = su.source) WHERE su.source = %s)" % (highest_sid))
+
+ while True:
+ m = cursor.fetchone()
+ if not m:
+ break
+
(rfc822, rfc2047, name, email) = utils.fix_maintainer(m[0])
if email == uid_email or name == uid_name:
should_reject=False
for b in changes["binary"].keys():
for suite in changes["distribution"].keys():
- suite_id = database.get_suite_id(suite)
- q = Upload.projectB.query("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = '%s' AND ba.suite = %s" % (b, suite_id))
- for s in q.getresult():
+ suite_id = DBConn().get_suite_id(suite)
+
+ cursor.execute("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = %(package)s AND ba.suite = %(suite)s" , {'package':b, 'suite':suite_id} )
+ while True:
+ s = cursor.fetchone()
+ if not s:
+ break
+
if s[0] != changes["source"]:
reject("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
################################################################################
def is_unembargo ():
- q = Upload.projectB.query(
- "SELECT package FROM disembargo WHERE package = '%s' AND version = '%s'" %
- (changes["source"], changes["version"]))
- ql = q.getresult()
- if ql:
+ cursor = DBConn().cursor()
+ cursor.execute( "SELECT package FROM disembargo WHERE package = %(source)s AND version = %(version)s", changes )
+ if cursor.fetchone():
return 1
oldcwd = os.getcwd()
if changes["architecture"].has_key("source"):
if Options["No-Action"]: return 1
- Upload.projectB.query(
- "INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" %
- (changes["source"], changes["version"]))
+ cursor.execute( "INSERT INTO disembargo (package, version) VALUES ('%(package)s', '%(version)s')",
+ changes )
+ cursor.execute( "COMMIT" )
return 1
return 0
return 0
if not changes["architecture"].has_key("source"):
- pusuite = database.get_suite_id("proposed-updates")
- q = Upload.projectB.query(
- "SELECT S.source FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.version = '%s' AND sa.suite = %d" %
- (changes["source"], changes["version"], pusuite))
- ql = q.getresult()
- if ql:
+ pusuite = DBConn().get_suite_id("proposed-updates")
+ cursor = DBConn().cursor()
+ cursor.execute( """SELECT 1 FROM source s
+ JOIN src_associations sa ON (s.id = sa.source)
+ WHERE s.source = %(source)s
+ AND s.version = %(version)s
+ AND sa.suite = %(suite)s""",
+ {'source' : changes['source'],
+ 'version' : changes['version'],
+ 'suite' : pusuite})
+
+ if cursor.fetchone():
# source is already in proposed-updates so no need to hold
return 0
return 0
if not changes["architecture"].has_key("source"):
- pusuite = database.get_suite_id("oldstable-proposed-updates")
- q = Upload.projectB.query(
- "SELECT S.source FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.version = '%s' AND sa.suite = %d" %
- (changes["source"], changes["version"], pusuite))
- ql = q.getresult()
- if ql:
- # source is already in oldstable-proposed-updates so no need to hold
+ pusuite = DBConn().get_suite_id("oldstable-proposed-updates")
+ cursor = DBConn().cursor()
+ cursor.execute( """SELECT 1 FROM source s
+ JOIN src_associations sa ON (s.id = sa.source)
+ WHERE s.source = %(source)s
+ AND s.version = %(version)s
+ AND sa.suite = %(suite)s""",
+ {'source' : changes['source'],
+ 'version' : changes['version'],
+ 'suite' : pusuite})
+ if cursor.fetchone():
return 0
return 1
################################################################################
-import commands, os, pg, re, sys
-import apt_pkg, apt_inst
+import commands
+import os
+import pg
+import re
+import sys
+import apt_pkg
+import apt_inst
from daklib import database
from daklib import utils
from daklib.dak_exceptions import *
Cnf = None
projectB = None
-required_database_schema = 5
+required_database_schema = 9
################################################################################
--- /dev/null
+#!/usr/bin/python
+
+"""
+Functions related debian binary packages
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2009 Mike O'Connor <stew@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <Ganneff> are we going the xorg way?
+# <Ganneff> a dak without a dak.conf?
+# <stew> automatically detect the wrong settings at runtime?
+# <Ganneff> yes!
+# <mhy> well, we'll probably always need dak.conf (how do you get the database setting
+# <mhy> but removing most of the config into the database seems sane
+# <Ganneff> mhy: dont spoil the fun
+# <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
+# <Ganneff> maybe one answers to sql
+# <stew> we will discover projectb via avahi
+# <mhy> you're both sick
+# <mhy> really fucking sick
+
+################################################################################
+
+import os
+import sys
+import shutil
+import tempfile
+import tarfile
+import commands
+import traceback
+import atexit
+from debian_bundle import deb822
+from dbconn import DBConn
+from config import Config
+import logging
+import utils
+
+class Binary(object):
+ def __init__(self, filename, reject=None):
+ """
+ @ptype filename: string
+ @param filename: path of a .deb
+
+ @ptype reject: function
+ @param reject: a function to log reject messages to
+ """
+ self.filename = filename
+ self.tmpdir = None
+ self.chunks = None
+ self.wrapped_reject = reject
+
+ def reject(self, message):
+ """
+ if we were given a reject function, send the reject message,
+ otherwise send it to stderr.
+ """
+ print >> sys.stderr, message
+ if self.wrapped_reject:
+ self.wrapped_reject(message)
+
+ def __del__(self):
+ """
+ make sure we cleanup when we are garbage collected.
+ """
+ self._cleanup()
+
+ def _cleanup(self):
+ """
+ we need to remove the temporary directory, if we created one
+ """
+ if self.tmpdir and os.path.exists(self.tmpdir):
+ shutil.rmtree(self.tmpdir)
+ self.tmpdir = None
+
+ def __scan_ar(self):
+ # get a list of the ar contents
+ if not self.chunks:
+
+ cmd = "ar t %s" % (self.filename)
+ (result, output) = commands.getstatusoutput(cmd)
+ if result != 0:
+ rejected = True
+ print("%s: 'ar t' invocation failed." % (self.filename))
+ self.reject("%s: 'ar t' invocation failed." % (self.filename))
+ self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
+ self.chunks = output.split('\n')
+
+
+
+ def __unpack(self):
+ # Internal function which extracts the contents of the .ar to
+ # a temporary directory
+
+ if not self.tmpdir:
+ tmpdir = utils.temp_dirname()
+ cwd = os.getcwd()
+ try:
+ os.chdir( tmpdir )
+ cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
+ (result, output) = commands.getstatusoutput(cmd)
+ if result != 0:
+ print("%s: '%s' invocation failed." % (self.filename, cmd))
+ self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
+ self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
+ else:
+ self.tmpdir = tmpdir
+ atexit.register( self._cleanup )
+
+ finally:
+ os.chdir( cwd )
+
+ def valid_deb(self, relaxed=False):
+ """
+ Check deb contents making sure the .deb contains:
+ 1. debian-binary
+ 2. control.tar.gz
+ 3. data.tar.gz or data.tar.bz2
+ in that order, and nothing else.
+ """
+ self.__scan_ar()
+ rejected = not self.chunks
+ if relaxed:
+ if len(self.chunks) < 3:
+ rejected = True
+ self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
+ else:
+ if len(self.chunks) != 3:
+ rejected = True
+ self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
+ if self.chunks[0] != "debian-binary":
+ rejected = True
+ self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
+ if not rejected and self.chunks[1] != "control.tar.gz":
+ rejected = True
+ self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
+ if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
+ rejected = True
+ self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
+
+ return not rejected
+
+ def scan_package(self, bootstrap_id=0, relaxed=False):
+ """
+ Unpack the .deb, do sanity checking, and gather info from it.
+
+ Currently information gathering consists of getting the contents list. In
+ the hopefully near future, it should also include gathering info from the
+ control file.
+
+ @ptype bootstrap_id: int
+ @param bootstrap_id: the id of the binary these packages
+ should be associated or zero meaning we are not bootstrapping
+ so insert into a temporary table
+
+ @return True if the deb is valid and contents were imported
+ """
+ result = False
+ rejected = not self.valid_deb(relaxed)
+ if not rejected:
+ self.__unpack()
+
+
+ cwd = os.getcwd()
+ if not rejected and self.tmpdir:
+ try:
+ os.chdir(self.tmpdir)
+ if self.chunks[1] == "control.tar.gz":
+ control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
+ control.extract('./control', self.tmpdir )
+ if self.chunks[2] == "data.tar.gz":
+ data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
+ elif self.chunks[2] == "data.tar.bz2":
+ data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
+
+ if bootstrap_id:
+ result = DBConn().insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ else:
+ pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
+ pkg = pkgs.next()
+ result = DBConn().insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+
+ except:
+ traceback.print_exc()
+
+ os.chdir(cwd)
+ self._cleanup()
+ return result
+
+ def check_utf8_package(self, package):
+ """
+ Unpack the .deb, do sanity checking, and gather info from it.
+
+ Currently information gathering consists of getting the contents list. In
+ the hopefully near future, it should also include gathering info from the
+ control file.
+
+ @ptype bootstrap_id: int
+ @param bootstrap_id: the id of the binary these packages
+ should be associated or zero meaning we are not bootstrapping
+ so insert into a temporary table
+
+ @return True if the deb is valid and contents were imported
+ """
+ rejected = not self.valid_deb(True)
+ self.__unpack()
+
+ if not rejected and self.tmpdir:
+ cwd = os.getcwd()
+ try:
+ os.chdir(self.tmpdir)
+ if self.chunks[1] == "control.tar.gz":
+ control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
+ control.extract('control', self.tmpdir )
+ if self.chunks[2] == "data.tar.gz":
+ data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
+ elif self.chunks[2] == "data.tar.bz2":
+ data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
+
+ for tarinfo in data:
+ try:
+ unicode( tarinfo.name )
+ except:
+ print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
+
+ except:
+ traceback.print_exc()
+ result = False
+
+ os.chdir(cwd)
+
+
--- /dev/null
+#!/usr/bin/env python
+
+"""
+Config access class
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2008 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <NCommander> mhy, how about "Now with 20% more monty python references"
+
+################################################################################
+
+import apt_pkg
+import socket
+
+from singleton import Singleton
+
+################################################################################
+
+default_config = "/etc/dak/dak.conf"
+
+def which_conf_file(Cnf):
+ res = socket.gethostbyaddr(socket.gethostname())
+ if Cnf.get("Config::" + res[0] + "::DakConfig"):
+ return Cnf["Config::" + res[0] + "::DakConfig"]
+ else:
+ return default_config
+
+class Config(Singleton):
+ """
+ A Config object is a singleton containing
+ information about the DAK configuration
+ """
+ def __init__(self, *args, **kwargs):
+ super(Config, self).__init__(*args, **kwargs)
+
+ def _readconf(self):
+ apt_pkg.init()
+
+ self.Cnf = apt_pkg.newConfiguration()
+
+ apt_pkg.ReadConfigFileISC(self.Cnf, default_config)
+
+ # Check whether our dak.conf was the real one or
+ # just a pointer to our main one
+ res = socket.gethostbyaddr(socket.gethostname())
+ conffile = self.Cnf.get("Config::" + res[0] + "::DakConfig")
+ if conffile:
+ apt_pkg.ReadConfigFileISC(self.Cnf, conffile)
+
+ # Rebind some functions
+ # TODO: Clean this up
+ self.get = self.Cnf.get
+ self.SubTree = self.Cnf.SubTree
+ self.ValueList = self.Cnf.ValueList
+
+ def _startup(self, *args, **kwargs):
+ self._readconf()
+
+ def has_key(self, name):
+ return self.Cnf.has_key(name)
+
+ def __getitem__(self, name):
+ return self.Cnf[name]
+
"NoFreeFilenameError": """Exception raised when no alternate filename was found.""",
"TransitionsError": """Exception raised when transitions file can't be parsed.""",
"NoSourceFieldError": """Exception raised - we cant find the source - wtf?""",
+ "MissingContents": """Exception raised - we could not determine contents for this deb""",
"DBUpdateError": """Exception raised - could not update the database""",
"ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded"""
} #: All dak exceptions
import sys
import time
import types
+import utils
+from binary import Binary
################################################################################
maintainer_id_cache = {} #: cache for maintainers
keyring_id_cache = {} #: cache for keyrings
source_id_cache = {} #: cache for sources
+
files_id_cache = {} #: cache for files
maintainer_cache = {} #: cache for maintainer names
fingerprint_id_cache = {} #: cache for fingerprints
queue_id_cache = {} #: cache for queues
uid_id_cache = {} #: cache for uids
suite_version_cache = {} #: cache for suite_versions (packages)
+suite_bin_version_cache = {}
+cache_preloaded = False
################################################################################
@return: the version for I{source} in I{suite}
"""
+
global suite_version_cache
cache_key = "%s_%s" % (source, suite)
return version
+def get_latest_binary_version_id(binary, section, suite, arch):
+ global suite_bin_version_cache
+ cache_key = "%s_%s_%s_%s" % (binary, section, suite, arch)
+ cache_key_all = "%s_%s_%s_%s" % (binary, section, suite, get_architecture_id("all"))
+
+ # Check for the cache hit for its arch, then arch all
+ if suite_bin_version_cache.has_key(cache_key):
+ return suite_bin_version_cache[cache_key]
+ if suite_bin_version_cache.has_key(cache_key_all):
+ return suite_bin_version_cache[cache_key_all]
+ if cache_preloaded == True:
+ return # package does not exist
+
+ q = projectB.query("SELECT DISTINCT b.id FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.package = '%s' AND b.architecture = '%d' AND ba.suite = '%d' AND o.section = '%d'" % (binary, int(arch), int(suite), int(section)))
+
+ if not q.getresult():
+ return False
+
+ highest_bid = q.getresult()[0][0]
+
+ suite_bin_version_cache[cache_key] = highest_bid
+ return highest_bid
+
+def preload_binary_id_cache():
+ global suite_bin_version_cache, cache_preloaded
+
+ # Get suite info
+ q = projectB.query("SELECT id FROM suite")
+ suites = q.getresult()
+
+ # Get arch mappings
+ q = projectB.query("SELECT id FROM architecture")
+ arches = q.getresult()
+
+ for suite in suites:
+ for arch in arches:
+ q = projectB.query("SELECT DISTINCT b.id, b.package, o.section FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.architecture = '%d' AND ba.suite = '%d'" % (int(arch[0]), int(suite[0])))
+
+ for bi in q.getresult():
+ cache_key = "%s_%s_%s_%s" % (bi[1], bi[2], suite[0], arch[0])
+ suite_bin_version_cache[cache_key] = int(bi[0])
+
+ cache_preloaded = True
+
def get_suite_architectures(suite):
"""
Returns list of architectures for C{suite}.
q = projectB.query(sql)
return map(lambda x: x[0], q.getresult())
+def get_suite_untouchable(suite):
+ """
+ Returns true if the C{suite} is untouchable, otherwise false.
+
+ @type suite: string, int
+ @param suite: the suite name or the suite_id
+
+ @rtype: boolean
+ @return: status of suite
+ """
+
+ suite_id = None
+ if type(suite) == str:
+ suite_id = get_suite_id(suite.lower())
+ elif type(suite) == int:
+ suite_id = suite
+ else:
+ return None
+
+ sql = """ SELECT untouchable FROM suite WHERE id='%s' """ % (suite_id)
+
+ q = projectB.query(sql)
+ if q.getresult()[0][0] == "f":
+ return False
+ else:
+ return True
################################################################################
q = projectB.query(sql)
return map(lambda x: x[0], q.getresult())
+
+
+################################################################################
+
+def copy_temporary_contents(package, version, arch, deb, reject):
+ """
+ copy the previously stored contents from the temp table to the permanant one
+
+ during process-unchecked, the deb should have been scanned and the
+ contents stored in pending_content_associations
+ """
+
+ # first see if contents exist:
+
+ arch_id = get_architecture_id (arch)
+
+ exists = projectB.query("""SELECT 1 FROM pending_content_associations
+ WHERE package='%s'
+ AND version='%s'
+ AND architecture=%d LIMIT 1"""
+ % (package, version, arch_id) ).getresult()
+
+ if not exists:
+ # This should NOT happen. We should have added contents
+ # during process-unchecked. if it did, log an error, and send
+ # an email.
+ subst = {
+ "__PACKAGE__": package,
+ "__VERSION__": version,
+ "__ARCH__": arch,
+ "__TO_ADDRESS__": Cnf["Dinstall::MyAdminAddress"],
+ "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"] }
+
+ message = utils.TemplateSubst(subst, Cnf["Dir::Templates"]+"/missing-contents")
+ utils.send_mail( message )
+
+ exists = Binary(deb, reject).scan_package()
+
+ if exists:
+ sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
+ SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations
+ WHERE package='%s'
+ AND version='%s'
+ AND architecture=%d""" % (package, version, arch_id)
+ projectB.query(sql)
+ projectB.query("""DELETE from pending_content_associations
+ WHERE package='%s'
+ AND version='%s'
+ AND architecture=%d""" % (package, version, arch_id))
+
+ return exists
--- /dev/null
+#!/usr/bin/python
+
+""" DB access class
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+@copyright: 2008-2009 Mark Hymers <mhy@debian.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Mike O'Connor <stew@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# < mhy> I need a funny comment
+# < sgran> two peanuts were walking down a dark street
+# < sgran> one was a-salted
+# * mhy looks up the definition of "funny"
+
+################################################################################
+
+import os
+import psycopg2
+import traceback
+
+from singleton import Singleton
+from config import Config
+
+################################################################################
+
+class Cache(object):
+ def __init__(self, hashfunc=None):
+ if hashfunc:
+ self.hashfunc = hashfunc
+ else:
+ self.hashfunc = lambda x: x['value']
+
+ self.data = {}
+
+ def SetValue(self, keys, value):
+ self.data[self.hashfunc(keys)] = value
+
+ def GetValue(self, keys):
+ return self.data.get(self.hashfunc(keys))
+
+################################################################################
+
+class DBConn(Singleton):
+ """
+ database module init.
+ """
+ def __init__(self, *args, **kwargs):
+ super(DBConn, self).__init__(*args, **kwargs)
+
+ def _startup(self, *args, **kwargs):
+ self.__createconn()
+ self.__init_caches()
+
+ ## Connection functions
+ def __createconn(self):
+ cnf = Config()
+ connstr = "dbname=%s" % cnf["DB::Name"]
+ if cnf["DB::Host"]:
+ connstr += " host=%s" % cnf["DB::Host"]
+ if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
+ connstr += " port=%s" % cnf["DB::Port"]
+
+ self.db_con = psycopg2.connect(connstr)
+
+ def reconnect(self):
+ try:
+ self.db_con.close()
+ except psycopg2.InterfaceError:
+ pass
+
+ self.db_con = None
+ self.__createconn()
+
+ ## Cache functions
+ def __init_caches(self):
+ self.caches = {'suite': Cache(),
+ 'section': Cache(),
+ 'priority': Cache(),
+ 'override_type': Cache(),
+ 'architecture': Cache(),
+ 'archive': Cache(),
+ 'component': Cache(),
+ 'content_path_names': Cache(),
+ 'content_file_names': Cache(),
+ 'location': Cache(lambda x: '%s_%s_%s' % (x['location'], x['component'], x['location'])),
+ 'maintainer': {}, # TODO
+ 'keyring': {}, # TODO
+ 'source': Cache(lambda x: '%s_%s_' % (x['source'], x['version'])),
+ 'files': Cache(lambda x: '%s_%s_' % (x['filename'], x['location'])),
+ 'maintainer': {}, # TODO
+ 'fingerprint': {}, # TODO
+ 'queue': {}, # TODO
+ 'uid': {}, # TODO
+ 'suite_version': Cache(lambda x: '%s_%s' % (x['source'], x['suite'])),
+ }
+
+ self.prepared_statements = {}
+
+ def prepare(self,name,statement):
+ if not self.prepared_statements.has_key(name):
+ c = self.cursor()
+ c.execute(statement)
+ self.prepared_statements[name] = statement
+
+ def clear_caches(self):
+ self.__init_caches()
+
+ ## Functions to pass through to the database connector
+ def cursor(self):
+ return self.db_con.cursor()
+
+ def commit(self):
+ return self.db_con.commit()
+
+ ## Get functions
+ def __get_single_id(self, query, values, cachename=None):
+ # This is a bit of a hack but it's an internal function only
+ if cachename is not None:
+ res = self.caches[cachename].GetValue(values)
+ if res:
+ return res
+
+ c = self.db_con.cursor()
+ c.execute(query, values)
+
+ if c.rowcount != 1:
+ return None
+
+ res = c.fetchone()[0]
+
+ if cachename is not None:
+ self.caches[cachename].SetValue(values, res)
+
+ return res
+
+ def __get_id(self, retfield, table, qfield, value):
+ query = "SELECT %s FROM %s WHERE %s = %%(value)s" % (retfield, table, qfield)
+ return self.__get_single_id(query, {'value': value}, cachename=table)
+
+ def get_suite_id(self, suite):
+ """
+ Returns database id for given C{suite}.
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type suite: string
+ @param suite: The name of the suite
+
+ @rtype: int
+ @return: the database id for the given suite
+
+ """
+ return int(self.__get_id('id', 'suite', 'suite_name', suite))
+
+ def get_section_id(self, section):
+ """
+ Returns database id for given C{section}.
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type section: string
+ @param section: The name of the section
+
+ @rtype: int
+ @return: the database id for the given section
+
+ """
+ return self.__get_id('id', 'section', 'section', section)
+
+ def get_priority_id(self, priority):
+ """
+ Returns database id for given C{priority}.
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type priority: string
+ @param priority: The name of the priority
+
+ @rtype: int
+ @return: the database id for the given priority
+
+ """
+ return self.__get_id('id', 'priority', 'priority', priority)
+
+ def get_override_type_id(self, override_type):
+ """
+ Returns database id for given override C{type}.
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type type: string
+ @param type: The name of the override type
+
+ @rtype: int
+ @return: the database id for the given override type
+
+ """
+ return self.__get_id('id', 'override_type', 'type', override_type)
+
+ def get_architecture_id(self, architecture):
+ """
+ Returns database id for given C{architecture}.
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type architecture: string
+ @param architecture: The name of the override type
+
+ @rtype: int
+ @return: the database id for the given architecture
+
+ """
+ return self.__get_id('id', 'architecture', 'arch_string', architecture)
+
+ def get_archive_id(self, archive):
+ """
+ returns database id for given c{archive}.
+ results are kept in a cache during runtime to minimize database queries.
+
+ @type archive: string
+ @param archive: the name of the override type
+
+ @rtype: int
+ @return: the database id for the given archive
+
+ """
+ return self.__get_id('id', 'archive', 'lower(name)', archive)
+
+ def get_component_id(self, component):
+ """
+ Returns database id for given C{component}.
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type component: string
+ @param component: The name of the override type
+
+ @rtype: int
+ @return: the database id for the given component
+
+ """
+ return self.__get_id('id', 'component', 'lower(name)', component)
+
+ def get_location_id(self, location, component, archive):
+ """
+ Returns database id for the location behind the given combination of
+ - B{location} - the path of the location, eg. I{/srv/ftp.debian.org/ftp/pool/}
+ - B{component} - the id of the component as returned by L{get_component_id}
+ - B{archive} - the id of the archive as returned by L{get_archive_id}
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type location: string
+ @param location: the path of the location
+
+ @type component: int
+ @param component: the id of the component
+
+ @type archive: int
+ @param archive: the id of the archive
+
+ @rtype: int
+ @return: the database id for the location
+
+ """
+
+ archive_id = self.get_archive_id(archive)
+
+ if not archive_id:
+ return None
+
+ res = None
+
+ if component:
+ component_id = self.get_component_id(component)
+ if component_id:
+ res = self.__get_single_id("SELECT id FROM location WHERE path=%(location)s AND component=%(component)s AND archive=%(archive)s",
+ {'location': location,
+ 'archive': int(archive_id),
+ 'component': component_id}, cachename='location')
+ else:
+ res = self.__get_single_id("SELECT id FROM location WHERE path=%(location)s AND archive=%(archive)d",
+ {'location': location, 'archive': archive_id, 'component': ''}, cachename='location')
+
+ return res
+
+ def get_source_id(self, source, version):
+ """
+ Returns database id for the combination of C{source} and C{version}
+ - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
+ - B{version}
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type source: string
+ @param source: source package name
+
+ @type version: string
+ @param version: the source version
+
+ @rtype: int
+ @return: the database id for the source
+
+ """
+ return self.__get_single_id("SELECT id FROM source s WHERE s.source=%(source)s AND s.version=%(version)s",
+ {'source': source, 'version': version}, cachename='source')
+
+ def get_suite_version(self, source, suite):
+ """
+ Returns database id for a combination of C{source} and C{suite}.
+
+ - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
+ - B{suite} - a suite name, eg. I{unstable}
+
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type source: string
+ @param source: source package name
+
+ @type suite: string
+ @param suite: the suite name
+
+ @rtype: string
+ @return: the version for I{source} in I{suite}
+
+ """
+ return self.__get_single_id("""
+ SELECT s.version FROM source s, suite su, src_associations sa
+ WHERE sa.source=s.id
+ AND sa.suite=su.id
+ AND su.suite_name=%(suite)s
+ AND s.source=%(source)""", {'suite': suite, 'source': source}, cachename='suite_version')
+
+
+ def get_files_id (self, filename, size, md5sum, location_id):
+ """
+ Returns -1, -2 or the file_id for filename, if its C{size} and C{md5sum} match an
+ existing copy.
+
+ The database is queried using the C{filename} and C{location_id}. If a file does exist
+ at that location, the existing size and md5sum are checked against the provided
+ parameters. A size or checksum mismatch returns -2. If more than one entry is
+ found within the database, a -1 is returned, no result returns None, otherwise
+ the file id.
+
+ Results are kept in a cache during runtime to minimize database queries.
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @type size: int
+ @param size: the size of the file to check against the DB
+
+ @type md5sum: string
+ @param md5sum: the md5sum of the file to check against the DB
+
+ @type location_id: int
+ @param location_id: the id of the location as returned by L{get_location_id}
+
+ @rtype: int / None
+ @return: Various return values are possible:
+ - -2: size/checksum error
+ - -1: more than one file found in database
+ - None: no file found in database
+ - int: file id
+
+ """
+ values = {'filename' : filename,
+ 'location' : location_id}
+
+ res = self.caches['files'].GetValue( values )
+
+ if not res:
+ query = """SELECT id, size, md5sum
+ FROM files
+ WHERE filename = %(filename)s AND location = %(location)s"""
+
+ cursor = self.db_con.cursor()
+ cursor.execute( query, values )
+
+ if cursor.rowcount == 0:
+ res = None
+
+ elif cursor.rowcount != 1:
+ res = -1
+
+ else:
+ row = cursor.fetchone()
+
+ if row[1] != int(size) or row[2] != md5sum:
+ res = -2
+
+ else:
+ self.caches['files'].SetValue(values, row[0])
+ res = row[0]
+
+ return res
+
+
+ def get_or_set_contents_file_id(self, filename):
+ """
+ Returns database id for given filename.
+
+ Results are kept in a cache during runtime to minimize database queries.
+ If no matching file is found, a row is inserted.
+
+ @type filename: string
+ @param filename: The filename
+
+ @rtype: int
+ @return: the database id for the given component
+ """
+ try:
+ values={'value': filename}
+ query = "SELECT id FROM content_file_names WHERE file = %(value)s"
+ id = self.__get_single_id(query, values, cachename='content_file_names')
+ if not id:
+ c = self.db_con.cursor()
+ c.execute( "INSERT INTO content_file_names VALUES (DEFAULT, %(value)s) RETURNING id",
+ values )
+
+ id = c.fetchone()[0]
+ self.caches['content_file_names'].SetValue(values, id)
+
+ return id
+ except:
+ traceback.print_exc()
+ raise
+
+ def get_or_set_contents_path_id(self, path):
+ """
+ Returns database id for given path.
+
+ Results are kept in a cache during runtime to minimize database queries.
+ If no matching file is found, a row is inserted.
+
+ @type path: string
+ @param path: The filename
+
+ @rtype: int
+ @return: the database id for the given component
+ """
+ try:
+ values={'value': path}
+ query = "SELECT id FROM content_file_paths WHERE path = %(value)s"
+ id = self.__get_single_id(query, values, cachename='content_path_names')
+ if not id:
+ c = self.db_con.cursor()
+ c.execute( "INSERT INTO content_file_paths VALUES (DEFAULT, %(value)s) RETURNING id",
+ values )
+
+ id = c.fetchone()[0]
+ self.caches['content_path_names'].SetValue(values, id)
+
+ return id
+ except:
+ traceback.print_exc()
+ raise
+
+ def get_suite_architectures(self, suite):
+ """
+ Returns list of architectures for C{suite}.
+
+ @type suite: string, int
+ @param suite: the suite name or the suite_id
+
+ @rtype: list
+ @return: the list of architectures for I{suite}
+ """
+
+ suite_id = None
+ if type(suite) == str:
+ suite_id = self.get_suite_id(suite)
+ elif type(suite) == int:
+ suite_id = suite
+ else:
+ return None
+
+ c = self.db_con.cursor()
+ c.execute( """SELECT a.arch_string FROM suite_architectures sa
+ JOIN architecture a ON (a.id = sa.architecture)
+ WHERE suite='%s'""" % suite_id )
+
+ return map(lambda x: x[0], c.fetchall())
+
+ def insert_content_paths(self, bin_id, fullpaths):
+ """
+ Make sure given path is associated with given binary id
+
+ @type bin_id: int
+ @param bin_id: the id of the binary
+ @type fullpath: string
+ @param fullpath: the path of the file being associated with the binary
+
+ @return True upon success
+ """
+
+ c = self.db_con.cursor()
+
+ c.execute("BEGIN WORK")
+ try:
+
+ for fullpath in fullpaths:
+ (path, file) = os.path.split(fullpath)
+
+ # Get the necessary IDs ...
+ file_id = self.get_or_set_contents_file_id(file)
+ path_id = self.get_or_set_contents_path_id(path)
+
+ c.execute("""INSERT INTO content_associations
+ (binary_pkg, filepath, filename)
+ VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) )
+
+ c.execute("COMMIT")
+ return True
+ except:
+ traceback.print_exc()
+ c.execute("ROLLBACK")
+ return False
+
+ def insert_pending_content_paths(self, package, fullpaths):
+ """
+ Make sure given paths are temporarily associated with given
+ package
+
+ @type package: dict
+ @param package: the package to associate with should have been read in from the binary control file
+ @type fullpaths: list
+ @param fullpaths: the list of paths of the file being associated with the binary
+
+ @return True upon success
+ """
+
+ c = self.db_con.cursor()
+
+ c.execute("BEGIN WORK")
+ try:
+ arch_id = self.get_architecture_id(package['Architecture'])
+
+ # Remove any already existing recorded files for this package
+ c.execute("""DELETE FROM pending_content_associations
+ WHERE package=%(Package)s
+ AND version=%(Version)s
+ AND architecture=%(ArchID)s""", {'Package': package['Package'],
+ 'Version': package['Version'],
+ 'ArchID': arch_id})
+
+ for fullpath in fullpaths:
+ (path, file) = os.path.split(fullpath)
+
+ if path.startswith( "./" ):
+ path = path[2:]
+ # Get the necessary IDs ...
+ file_id = self.get_or_set_contents_file_id(file)
+ path_id = self.get_or_set_contents_path_id(path)
+
+ c.execute("""INSERT INTO pending_content_associations
+ (package, version, architecture, filepath, filename)
+ VALUES (%%(Package)s, %%(Version)s, '%d', '%d', '%d')"""
+ % (arch_id, path_id, file_id), package )
+
+ c.execute("COMMIT")
+ return True
+ except:
+ traceback.print_exc()
+ c.execute("ROLLBACK")
+ return False
--- /dev/null
+#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
+
+"""
+Singleton pattern code
+
+Inspiration for this very simple ABC was taken from various documents /
+tutorials / mailing lists. This may not be thread safe but given that
+(as I write) large chunks of dak aren't even type-safe, I'll live with
+it for now
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2008 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+################################################################################
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# < sgran> NCommander: in SQL, it's better to join than to repeat information
+# < tomv_w> that makes SQL the opposite to Debian mailing lists!
+
+################################################################################
+
+"""
+This class set implements objects that may need to be instantiated multiple
+times, but we don't want the overhead of actually creating and init'ing
+them more than once. It also saves us using globals all over the place
+"""
+
+class Singleton(object):
+ """This is the ABC for other dak Singleton classes"""
+ __single = None
+ def __new__(cls, *args, **kwargs):
+ # Check to see if a __single exists already for this class
+ # Compare class types instead of just looking for None so
+ # that subclasses will create their own __single objects
+ if cls != type(cls.__single):
+ cls.__single = object.__new__(cls, *args, **kwargs)
+ cls.__single._startup(*args, **kwargs)
+ return cls.__single
+
+ def __init__(self, *args, **kwargs):
+ if type(self) == "Singleton":
+ raise NotImplementedError("Singleton is an ABC")
+
+ def _startup(self):
+ """
+ _startup is a private method used instead of __init__ due to the way
+ we instantiate this object
+ """
+ raise NotImplementedError("Singleton is an ABC")
+
file_handle = open_file(f)
except CantOpenError:
rejmsg.append("Could not open file %s for checksumming" % (f))
+ continue
files[f][hash_key(hashname)] = hashfunc(file_handle)
################################################################################
+def temp_dirname(parent=None, prefix="dak", suffix=""):
+ """
+ Return a secure and unique directory by pre-creating it.
+ If 'parent' is non-null, it will be the directory the directory is pre-created in.
+ If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
+ If 'suffix' is non-null, the filename will end with it.
+
+ Returns a pathname to the new directory
+ """
+
+ return tempfile.mkdtemp(suffix, prefix, parent)
+
+################################################################################
+
def is_email_alias(email):
""" checks if the user part of the email is listed in the alias file """
global alias_cache
if which_conf_file() != default_config:
apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
-################################################################################
+###############################################################################
<helix> elmo: I can't believe people pay you to fix computers
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+<mhy> oh no, Ganneff has just corrected my english
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+<mhy> I often wonder if we should use NSA bot or something instead and get dinstall to send emails telling us about its progress :-)
+<mhy> dinstall: I'm processing openoffice
+<mhy> dinstall: I'm choking, please help me
+<Ganneff> yeah. get floods in here, for 600 accepted packages.
+<mhy> hehe
+<Ganneff> im not sure the other opers will like it if i oper up the bot, just so it can flood faster
+<mhy> flood all debian related channels
+<mhy> just to be safe
+<Ganneff> /msg #debian-* dinstall: starting
+<Ganneff> more interesting would be the first message in #debian, the next in #d-devel, then #d-qa
+<Ganneff> and expect people to monitor all.
+<Ganneff> i bet we have enough debian channels to at least put the timestamps in seperate channels each
+<Ganneff> and if not - we can make it go multi-network
+<Ganneff> first oftc, then opn, then ircnet, then - we will find some. quakenet anyone?
+<mhy> I should know better than to give you ideas
TODO
====
-[NB: I use this as a thought record/scribble, not everything on here
- makes sense and/or is actually ever going to get done, so IIWY I
- wouldn't use it as gospel for the future of dak or as a TODO
- list for random hacking.]
+Various
+-------
+* Implement autosigning, see ftpmaster_autosigning on ftp-master host in text/.
-================================================================================
+* Check TODO.old and move still-valid/useful entries over here.
-Others
-------
+* need a testsuite _badly_
- o 'dak check-overrides' should remove the src-only override when a
- binary+source override exists
+* database table "binaries" contains a column 'type TEXT NOT
+ NULL'. This should be made a FK on override_type, as it only contains
+ deb/udeb strings.
- o reject on > or < in a version constraint
+ - sql query to do the db work for it:
+ ALTER TABLE binaries ADD COLUMN new_type INT4 REFERENCES override_type(id);
+ UPDATE BINARIES SET new_type = 7 WHERE type = 'deb';
+ UPDATE BINARIES SET new_type = 8 WHERE type = 'udeb';
+ ALTER TABLE binaries DROP COLUMN type;
+ ALTER TABLE binaries RENAME COLUMN new_type TO type;
- o 'dak reject-proposed-updates' should only start an editor once to
- capture a message; it will usually be the same message for all
- files on the same command line.
+ - needs updateX.py written and then the rest of the code changed to deal
+ with it.
-23:07 < aba> elmo: and, how about enhancing 'dak cruft-report' to spot half-dropped
- binaries on one arch (i.e. package used to build A and B, but B is
- no longer built on some archs)?
+* Checkout SQL Alchemy and probably use that for our database layer.
- o tabnanny the source
+* reject on > or < in a version constraint
- o drop map-unreleased
+* use pythonX.Y-tarfile to check orig.tar.gz timestamps too.
- o check email only portions of addresses match too, iff the names
- don't, helps with the "James Troup <james@nocrew.org>"
- vs. "<james@nocrew.org>" case.
+* the .dak stuff is fundamentally braindamaged for various reasons, it
+ should DIE. If we want to cache information - use a ("temporary")
+ database table and let p-a clean it up, e.g. like contents does.
- o ensure .dsc section/prio match .changes section/prio
+* security global mail overrides should special case buildd stuff so
+ that buildds get ACCEPTED mails. Or maybe send them at
+ new-security-install time. That way upload-security doesn't grow
+ boundlessly.
- o 'dak clean-suites' performance is kind of crap when asked to
- remove a lot of files (e.g. 2k or so).
+* debianqueued sucks. Reimplement in a sane way.
- o we don't handle the case where an identical orig.tar.gz is
- mentioned in the .changes, but not in unchecked; but should we
- care?
-
- o 'dak ls' could do better sanity checking for -g/-G (e.g. not more
- than one suite, etc.)
-
- o use python2.2-tarfile (once it's in stable?) to check orig.tar.gz
- timestamps too.
-
- o need to decide on whether we're tying for most errors at once.. if
- so (probably) then make sure code doesn't assume variables exist and
- either way do something about checking error code of check_dsc and
- later functions so we skip later checks if they're bailing.
-
- o the .dak stuff is fundamentally braindamaged, it's not versioned
- so there's no way to change the format, yay me. need to fix.
- probably by putting a version var as the first thing and checking
- that.. auto-upgrade at least from original format would be good.
- might also be a good idea to put everything in one big dict after
- that?
-
- o [?, wishlist, distant future] RFC2047-ing should be extended to
- all headers of mails sent out.
-
- o reject sparc64 binaries in a non '*64*' package.
-
- o queue.py(source_exists): a) we take arguments as parameters that
- we could figure out for ourselves (we're part of the Upload class
- after all), b) we have this 3rd argument which defaults to "any"
- but could in fact be dropped since no one uses it like that.
-
- o 'dak process-unchecked': doesn't handle bin-only NMUs of stuff
- still in NEW, BYHAND or ACCEPTED (but not the pool) - not a big
- deal, upload can be retried once the source is in the archive, but
- still.
-
- o security global mail overrides should special case buildd stuff so
- that buildds get ACCEPTED mails (or maybe 'dak security-install' (?)), that way
- upload-security doesn't grow boundlessly.
-
- o 'dak security-install' should upload sourceful packages first,
- otherwise with big packages (e.g. X) and esp. when source is !i386,
- half the arches can be uploaded without source, get copied into
- queue/unaccepted and promptly rejected.
-
- o 'dak cruft-report's NVIU check doesn't catch cases where source
- package changed name, should check binaries
- too. [debian-devel@l.d.o, 2004-02-03]
-
- o cnf[Rm::logfile] is misnamed...
-
-<aj> i'd be kinda inclined to go with insisting the .changes file take
- the form ---- BEGIN PGP MESSAGE --- <non -- BEGIN/END lines> --
- BEGIN PGP SIG -- END PGP MESSAGE -- with no lines before or after,
- and rejecting .changes that didn't match that
-
- o 'dak cruft-report' should check for source packages not building any binaries
-
- o 'dak control-suite' should have a diff mode that accepts diff output!
-
- o 'dak clean-proposed-updates' doesn't deal with 'dak rm'-d
- packages, partial replacements etc. and more.
-
- o 'dak reject-proposed-updates' blindly deletes with no check that
- the delete failed which it might well given we only look for
- package/version, not package/version _in p-u_. duh.
-
- o 'dak rm' should remove obsolete changes when removing from p-u, or
- at least warn. or 'dak reject-proposed-updates' should handle it.
-
- o need a testsuite _badly_
-
- o 'dak process-unchecked' crashes if run as a user in -n mode when
- orig.tar.gz is in queue/new...
-
-<elmo_home> [<random>maybe I should reject debian packages with a non-Debian origin or bugs field</>]
-<Kamion> [<random>agreed; dunno what origin does but non-Debian bugs fields would be bad]
-
- o 'dak clean-suites' should make use of select..except select, temporary tables
- etc. rather than looping and calling SQL every time so we can do
- suite removal sanely (see potato-removal document)
-
- o 'dak rm' will happily include packages in the Cc list that aren't
- being removed...
-
- o 'dak rm' doesn't remove udebs when removing the source they build from
-
- o check_dsc_against_db's "delete an entry from files while you're
- not looking" habit is Evil and Bad.
-
- o 'dak process-new' allows you to edit the section and change the
- component, but really shouldn't.
-
- o 'dak rm' needs to, when not sending bug close mails, promote Cc: to
- To: and send the mail anyways.
-
- o the lockfile (Archive_Maintenance_In_Progress) should probably be in a conf file
-
- o 'dak ls' should cross-check the b.source field and if it's not
- null and s.name linked from it != the source given in
- -S/--source-and-binary ignore.
-
- o 'dak reject-proposed-updates' sucks; it should a) only spam d-i
- for sourceful rejections, b) sort stuff so it rejects sourceful
- stuff first. the non-sourceful should probably get a form mail, c)
- automate the non-sourceful stuff (see b).
-
- o 'dak process-unchecked' should do q-d stuff for faster AA [ryan]
-
- o split the morgue into source and binary so binaries can be purged first!
-
- o per-architecture priorities for things like different arch'es
- gcc's, silly BSD libftw, palo, etc.
-
- o use postgres 7.2's built-in stat features to figure out how indices are used etc.
-
- o 'dak init-archive' shouldn't be using location, it should run down suites instead
-
- o 'dak clean-proposed-updates' needs to know about udebs
-
- o by default hamstring dak's mail sending so that it won't send
- anything until someone edits a script; it's been used far too
- much to send spam atm :(
-
- o $ftpdir/indices isn't created by 'dak init-dir' because it's not in dak.conf
-
- o sanity check depends/recommends/suggests too? in fact for any
- empty field?
-
-[minor] 'dak process-accepted's copychanges, copydotdak handling
- sucks, the per-suite thing is static for all packages, so work out
- in advance dummy.
-
-[dak ls] # filenames ?
-[dak ls] # maintainer, component, install date (source only?), fingerprint?
-
- o UrgencyLog stuff should minimize it's bombing out(?)
- o Log stuff should open the log file
-
- o 'dak queue-report' should footnote the actual notes, and also *
- the versions with notes so we can see new versions since being
- noted...
-
- o 'dak queue-report' should have alternative sorting options, including reverse
- and without or without differentiaion.
-
- o 'dak import-users-from-passwd' should sync debadmin and ftpmaster (?)
-
- o <drow> Can't read file.:
- /org/security.debian.org/queue/accepted/accepted/apache-perl_1.3.9-14.1-1.21.20000309-1_sparc.dak.
- You assume that the filenames are relative to accepted/, might want
- to doc or fix that.
-
- o <neuro> the orig was in NEW, the changes that caused it to be NEW
- were pulled out in -2, and we end up with no orig in the archive
- :(
-
- o SecurityQueueBuild doesn't handle the case of foo_3.3woody1 with a
- new .orig.tar.gz followed by a foo_3.3potato1 with the same
- .orig.tar.gz; 'dak process-unchecked' sees it and copes, but the AA
- code doesn't and can't really easily know so the potato AA dir is
- left with no .orig.tar.gz copy. doh.
-
- o orig.tar.gz in accepted not handled properly (?)
-
- o 'dak security-install' doesn't include .orig.tar.gz but it should
-
- o permissions (paranoia, group write, etc.) configurability and overhaul
-
- o remember duplicate copyrights in 'dak process-new' and skip them, per package
-
- o <M>ove option for 'dak process-new' byhand proecessing
-
- o 'dak cruft-report' could do with overrides
-
- o database.get_location_id should handle the lack of archive_id properly
-
- o the whole versioncmp thing should be documented
-
- o 'dak process-new' doesn't do the right thing with -2 and -1 uploads, as you can
- end up with the .orig.tar.gz not in the pool
-
- o 'dak process-new' exits if you check twice (aj)
-
- o 'dak process-new' doesn't trap signals from 'dak examine-package' properly
-
- o queued and/or perl on sparc stable sucks - reimplement it.
-
- o aj's bin nmu changes
-
- o 'dak process-new':
- * priority >> optional
- * arch != {any,all}
- * build-depends wrong (via 'dak compare-suites')
- * suid
- * conflicts
- * notification/stats to admin daily
- o trap 'dak examine-package' exiting
- o distinguish binary only versus others (neuro)
-
- o cache changes parsed from ordering (careful tho: would be caching
- .changes from world writable incoming, not holding)
-
- o dak doesn't recognise binonlyNMUs correctly in terms of telling
- who their source is; source-must-exist does, but the info is not
- propogated down.
-
- o Fix BTS vs. dak sync issues by queueing(via BSMTP) BTS mail so
- that it can be released on deman (e.g. ETRN to exim).
-
- o maintainers file needs overrides
-
- [ change override.maintainer to override.maintainer-from +
- override.maintainer-to and have them reference the maintainers
- table. Then fix 'dak make-maintainers' to use them and write some scripting
- to handle the Santiago situation. ]
-
- o Validate Depends (et al.) [it should match \(\s*(<<|<|<=|=|>=|>|>>)\s*<VERSIONREGEXP>\)]
-
- o Clean up DONE; archive to tar file every 2 weeks, update tar tvzf INDEX file.
-
- o testing-updates suite: if binary-only and version << version in
- unstable and source-ver ~= source-ver in testing; then map
- unstable -> testing-updates ?
-
- o hooks or configurability for debian specific checks (e.g. check_urgency, auto-building support)
-
- o morgue needs auto-cleaning (?)
-
- o dak stats: two modes, all included, seperate
- o dak stats: add non-US
- o dak stats: add ability to control components, architectures, archives, suites
- o dak stats: add key to expand header
-
-================================================================================
-
-queue/approved
+NEW processing
--------------
+* 'dak process-new' allows you to edit the section and change the
+ component, but really shouldn't allow the component change.
- o What to do with multi-suite uploads? Presumably hold in unapproved
- and warn? Or what? Can't accept just for unstable or reject just
- from stable.
-
- o Whenever we check for anything in accepted we also need to check in
- unapproved.
-
- o non-sourceful uploads should go straight through if they have
- source in accepted or the archive.
-
- o security uploads on auric should be pre-approved.
-
-================================================================================
-
-Less Urgent
------------
-
- o change utils.copy to try rename() first
-
- o [hard, long term] unchecked -> accepted should go into the db, not
- a suite, but similar. this would allow dak to get even faster,
- make 'dak ls' more useful, decomplexify specialacceptedautobuild
- and generally be more sane. may even be helpful to have e.g. new
- in the DB, so that we avoid corner cases like the .orig.tar.gz
- disappearing 'cos the package has been entirely removed but was
- still on stayofexecution when it entered new.
-
- o Logging [mostly done] (todo: 'dak clean-suites' (hard), .. ?)
-
- o 'dak process-unchecked': the tar extractor class doesn't need to be redone for each package
-
- o reverse of source-must-exist; i.e. binary-for-source-must-not-exist
- o REJECT reminders in 'dak clean-queues'.
- o 'dak examine-package' should check for conflicts and warn about them visavis priority [rmurray]
- o store a list of removed/files versions; also compare against them.
- [but be careful about scalability]
-
- o dak examine-package: print_copyright should be a lot more intelligent
- @ handle copyright.gz
- @ handle copyright.ja and copyright
- @ handle (detect at least) symlinks to another package's doc directory
- @ handle and/or fall back on source files (?)
-
- o To incorporate from utils:
- @ unreject
-
- o auto-purge out-of-date stuff from non-free/contrib so that testing and stuff works
- o doogie's binary -> source index
- o jt's web stuff, matt's changelog stuff (overlap)
-
- o [Hard] Need to merge non-non-US and non-US DBs.
-
- o experimental needs to auto clean (relative to unstable) [partial:
- 'dak cruft-report' warns about this]
-
- o Do a checkpc(1)-a-like which sanitizes a config files.
- o fix parse_changes()/build_file_list() to sanity check filenames
- o saftey check and/or rename debs so they match what they should be
-
- o Improve 'dak compare-suites'.
- o Need to optimize all the queries by using EXAMINE and building some INDEXs.
- [postgresql 7.2 will help here]
- o Need to enclose all the setting SQL stuff in transactions (mostly done).
- o Need to finish 'dak init-db' (a way to sync dak.conf and the DB)
- o Need the ability to rebuild all other tables from dists _or_ pools (in the event of disaster) (?)
- o Make the --help and --version options do stuff for all scripts
-
- o 'dak make-maintainers' can't handle whitespace-only lines (for the moment, this is feature)
-
- o generic way of saying isabinary and isadsc. (?)
-
- o s/distribution/suite/g
-
- o cron.weekly:
- @ weekly postins to d-c (?)
- @ backup of report (?)
- @ backup of changes.tgz (?)
-
- o --help doesn't work without /etc/dak/dak.conf (or similar) at
- least existing.
-
- o rename 'dak compare-suites' (clashes with existing 'dak compare-suites')...
-
- * Harder:
-
- o interrupting of stracing 'dak process-unchecked' causes exceptions errors from apt_inst calls
- o dependency checking (esp. stable) (partially done)
- o override checks sucks; it needs to track changes made by the
- maintainer and pass them onto ftpmaster instead of warning the
- maintainer.
- o need to do proper rfc822 escaping of from lines (as opposed to s/\.//g)
- o Revisit linking of binary->source in install() in dak.
- o Fix component handling in overrides (aj)
- o Fix lack of entires in source overrides (aj)
- o direport misreports things as section 'devel' (? we don't use direport)
- o vrfy check of every Maintainer+Changed-By address; valid for 3 months.
- o binary-all should be done on a per-source, per-architecture package
- basis to avoid, e.g. the perl-modules problem.
- o a source-missing-diff check: if the version has a - in it, and it
- is sourceful, it needs orig and diff, e.g. if someone uploads
- esound_0.2.22-6, and it is sourceful, and there is no diff ->
- REJECT (version has a dash, therefore not debian native.)
- o check linking of .tar.gz's to .dsc's.. see proftpd 1.2.1 as an example
- o archive needs md5sum'ed regularly, but takes too long to do all
- in one go; make progressive or weekly.
- o something needs to clear out .changes files from p-u when
- removing stuff superseded by newer versions. [but for now we have
- 'dak clean-proposed-updates']
- o test sig checking stuff in test/ (stupid thing is not modularized due to global abuse)
- o when encountering suspicous things (e.g. file tainting) do something more drastic
-
- * Easy:
-
- o suite mapping and component mapping are parsed per changes file,
- they should probably be stored in a dictionary created at startup.
- o don't stat/md5sum files you have entries for in the DB, moron
- boy (Dak.check_source_blah_blah)
- o promote changes["changes"] to mandatory in dak.py(dump_vars)
- after a month or so (or all .dak files contain in the queue
- contain it).
- o 'dak rm' should behave better with -a and without -b; see
- gcc-defaults removal for an example.
- o Reject on misconfigured kernel-package uploads
- o utils.extract_component_from_section: main/utils -> main/utils, main rather than utils, main
- o Fix 'dak process-unchecked' to warn if run when not in incoming or p-u
- o dak should validate multi-suite uploads; only possible valid one
- is "stable unstable"
- o cron.daily* should change umask (aj sucks)
- o 'dak cruft-report' doesn't look at debian-installer but should.
- o 'dak cruft-report' needs to check for binary-less source packages.
- o 'dak cruft-report' could accept a suite argument (?)
- o byhand stuff should send notification
- o 'dak poolize' should udpate db; move files, not the other way around [neuro]
- o 'dak rm' should update the stable changelog [joey]
- o update tagdb.dia
+* 'dak process-new' doesn't do the right thing with -2 and -1 uploads,
+ as you can end up with the .orig.tar.gz not in the pool or belonging
+ to a wrong suite.
- * Bizzare/uncertain:
+* 'dak process-new' doesn't trap signals from 'dak examine-package' properly
- o drop rather dubious currval stuff (?)
- o rationalize os.path.join() usage
- o 'dak cruft-report' also doesn't seem to warn about missing binary packages (??)
- o logging: hostname + pid ?
- o ANAIS should be done in dak (?)
- o Add an 'add' ability to 'dak rm' (? separate prog maybe)
- o Replicate old dinstall report stuff (? needed ?)
- o Handle the case of 1:1.1 which would overwrite 1.1 (?)
- o maybe drop -r/--regex in 'dak ls', make it the default and
- implement -e/--exact (a la joey's "elmo")
- o dsc files are not checked for existence/perms (only an issue if
- they're in the .dsc, but not the .changes.. possible?)
+* 'dak queue-report' should footnote the actual notes, and also * the
+ versions with notes so we can see new versions since being noted...
- * Cleanups & misc:
+* <neuro> the orig was in NEW, the changes that caused it to be NEW
+ were pulled out in -2, and we end up with no orig in the archive :(
- o db_access' get_files needs to use exceptions not this None, > 0, < 0 return val BS (?)
- o The untouchable flag doesn't stop new packages being added to ``untouchable'' suites
-================================================================================
-Packaging
----------
+Override handling
+-----------------
+* 'dak check-overrides' should remove the src-only override when a
+ binary+source override exists
- o Fix stuff to look in sensible places for libs and config file in debian package (?)
+* override checks sucks; it needs to track changes made by the
+ maintainer and pass them onto ftpmaster instead of warning the maintainer.
-================================================================================
+* Fix component handling in overrides
- --help manpage
------------------------------------------------------------------------------
-check-archive X
-check-overrides X X
-check-proposed-updates X
-clean-proposed-updates X
-clean-queues X
-clean-suites X X
-compare-suites X
-control-overrides X X
-control-suite X X
-cruft-report X
-decode-dot-dak X
-examine-package X
-generate-releases X
-import-archive X
-import-users-from-passwd X X
-init-db X
-init-dirs X
-ls X X
-make-maintainers X X
-make-overrides X
-make-suite-file-list X
-poolize X X
-process-accepted X X
-process-new X X
-process-unchecked X
-queue-report X
-rm X X
-security-install X
-stats X
-symlink-dists X
+Cruft
+-----
+* 'dak cruft-report' could do with overrides
-================================================================================
+* cruft-report could spot "half-dropped" binaries. Like if a package
+ used to build A and B, but B is no longer built for half the
+ architectures.
-Random useful-at-some-point SQL
--------------------------------
+* cruft-report's NVIU check doesn't catch cases where source package
+ changed name, should check binaries too. [debian-devel@l.d.o,
+ 2004-02-03]
-UPDATE files SET last_used = '1980-01-01'
- FROM binaries WHERE binaries.architecture = <x>
- AND binaries.file = files.id;
+* 'dak cruft-report' doesn't look at debian-installer but should.
-DELETE FROM bin_associations
- WHERE EXISTS (SELECT id FROM binaries
- WHERE architecture = <x>
- AND id = bin_associations.bin);
-================================================================================
--- /dev/null
+ TODO
+ ====
+
+[NB: I use this as a thought record/scribble, not everything on here
+ makes sense and/or is actually ever going to get done, so IIWY I
+ wouldn't use it as gospel for the future of dak or as a TODO
+ list for random hacking.]
+
+================================================================================
+
+Others
+------
+
+ o 'dak reject-proposed-updates' should only start an editor once to
+ capture a message; it will usually be the same message for all
+ files on the same command line.
+
+ o drop map-unreleased
+
+ o check email only portions of addresses match too, iff the names
+ don't, helps with the "James Troup <james@nocrew.org>"
+ vs. "<james@nocrew.org>" case.
+
+ o ensure .dsc section/prio match .changes section/prio
+
+ o 'dak clean-suites' performance is kind of crap when asked to
+ remove a lot of files (e.g. 2k or so).
+
+ o we don't handle the case where an identical orig.tar.gz is
+ mentioned in the .changes, but not in unchecked; but should we
+ care?
+
+ o 'dak ls' could do better sanity checking for -g/-G (e.g. not more
+ than one suite, etc.)
+
+ o need to decide on whether we're tying for most errors at once.. if
+ so (probably) then make sure code doesn't assume variables exist and
+ either way do something about checking error code of check_dsc and
+ later functions so we skip later checks if they're bailing.
+
+ o [?, wishlist, distant future] RFC2047-ing should be extended to
+ all headers of mails sent out.
+
+ o reject sparc64 binaries in a non '*64*' package.
+
+ o queue.py(source_exists): a) we take arguments as parameters that
+ we could figure out for ourselves (we're part of the Upload class
+ after all), b) we have this 3rd argument which defaults to "any"
+ but could in fact be dropped since no one uses it like that.
+
+ o 'dak process-unchecked': doesn't handle bin-only NMUs of stuff
+ still in NEW, BYHAND or ACCEPTED (but not the pool) - not a big
+ deal, upload can be retried once the source is in the archive, but
+ still.
+
+ o 'dak security-install' should upload sourceful packages first,
+ otherwise with big packages (e.g. X) and esp. when source is !i386,
+ half the arches can be uploaded without source, get copied into
+ queue/unaccepted and promptly rejected.
+
+ o cnf[Rm::logfile] is misnamed...
+
+<aj> i'd be kinda inclined to go with insisting the .changes file take
+ the form ---- BEGIN PGP MESSAGE --- <non -- BEGIN/END lines> --
+ BEGIN PGP SIG -- END PGP MESSAGE -- with no lines before or after,
+ and rejecting .changes that didn't match that
+
+ o 'dak control-suite' should have a diff mode that accepts diff output!
+
+ o 'dak clean-proposed-updates' doesn't deal with 'dak rm'-d
+ packages, partial replacements etc. and more.
+
+ o 'dak reject-proposed-updates' blindly deletes with no check that
+ the delete failed which it might well given we only look for
+ package/version, not package/version _in p-u_. duh.
+
+ o 'dak rm' should remove obsolete changes when removing from p-u, or
+ at least warn. or 'dak reject-proposed-updates' should handle it.
+
+ o 'dak process-unchecked' crashes if run as a user in -n mode when
+ orig.tar.gz is in queue/new...
+
+<elmo_home> [<random>maybe I should reject debian packages with a non-Debian origin or bugs field</>]
+<Kamion> [<random>agreed; dunno what origin does but non-Debian bugs fields would be bad]
+
+ o 'dak clean-suites' should make use of select..except select, temporary tables
+ etc. rather than looping and calling SQL every time so we can do
+ suite removal sanely (see potato-removal document)
+
+ o 'dak rm' will happily include packages in the Cc list that aren't
+ being removed...
+
+ o 'dak rm' doesn't remove udebs when removing the source they build from
+
+ o check_dsc_against_db's "delete an entry from files while you're
+ not looking" habit is Evil and Bad.
+
+ o 'dak rm' needs to, when not sending bug close mails, promote Cc: to
+ To: and send the mail anyways.
+
+ o the lockfile (Archive_Maintenance_In_Progress) should probably be in a conf file
+
+ o 'dak ls' should cross-check the b.source field and if it's not
+ null and s.name linked from it != the source given in
+ -S/--source-and-binary ignore.
+
+ o 'dak reject-proposed-updates' sucks; it should a) only spam d-i
+ for sourceful rejections, b) sort stuff so it rejects sourceful
+ stuff first. the non-sourceful should probably get a form mail, c)
+ automate the non-sourceful stuff (see b).
+
+ o 'dak process-unchecked' should do q-d stuff for faster AA [ryan]
+
+ o split the morgue into source and binary so binaries can be purged first!
+
+ o per-architecture priorities for things like different arch'es
+ gcc's, silly BSD libftw, palo, etc.
+
+ o use postgres 7.2's built-in stat features to figure out how indices are used etc.
+
+ o 'dak init-archive' shouldn't be using location, it should run down suites instead
+
+ o 'dak clean-proposed-updates' needs to know about udebs
+
+ o by default hamstring dak's mail sending so that it won't send
+ anything until someone edits a script; it's been used far too
+ much to send spam atm :(
+
+ o $ftpdir/indices isn't created by 'dak init-dir' because it's not in dak.conf
+
+ o sanity check depends/recommends/suggests too? in fact for any
+ empty field?
+
+[minor] 'dak process-accepted's copychanges, copydotdak handling
+ sucks, the per-suite thing is static for all packages, so work out
+ in advance dummy.
+
+[dak ls] # filenames ?
+[dak ls] # maintainer, component, install date (source only?), fingerprint?
+
+ o UrgencyLog stuff should minimize it's bombing out(?)
+ o Log stuff should open the log file
+
+ o 'dak import-users-from-passwd' should sync debadmin and ftpmaster (?)
+
+ o <drow> Can't read file.:
+ /org/security.debian.org/queue/accepted/accepted/apache-perl_1.3.9-14.1-1.21.20000309-1_sparc.dak.
+ You assume that the filenames are relative to accepted/, might want
+ to doc or fix that.
+
+ o SecurityQueueBuild doesn't handle the case of foo_3.3woody1 with a
+ new .orig.tar.gz followed by a foo_3.3potato1 with the same
+ .orig.tar.gz; 'dak process-unchecked' sees it and copes, but the AA
+ code doesn't and can't really easily know so the potato AA dir is
+ left with no .orig.tar.gz copy. doh.
+
+ o orig.tar.gz in accepted not handled properly (?)
+
+ o 'dak security-install' doesn't include .orig.tar.gz but it should
+
+ o permissions (paranoia, group write, etc.) configurability and overhaul
+
+ o database.get_location_id should handle the lack of archive_id properly
+
+ o the whole versioncmp thing should be documented
+
+ o aj's bin nmu changes
+
+ o 'dak process-new':
+ * priority >> optional
+ * arch != {any,all}
+ * build-depends wrong (via 'dak compare-suites')
+ * suid
+ * conflicts
+ * notification/stats to admin daily
+ o trap 'dak examine-package' exiting
+ o distinguish binary only versus others (neuro)
+
+ o cache changes parsed from ordering (careful tho: would be caching
+ .changes from world writable incoming, not holding)
+
+ o dak doesn't recognise binonlyNMUs correctly in terms of telling
+ who their source is; source-must-exist does, but the info is not
+ propogated down.
+
+ o maintainers file needs overrides
+
+ [ change override.maintainer to override.maintainer-from +
+ override.maintainer-to and have them reference the maintainers
+ table. Then fix 'dak make-maintainers' to use them and write some scripting
+ to handle the Santiago situation. ]
+
+ o Validate Depends (et al.) [it should match \(\s*(<<|<|<=|=|>=|>|>>)\s*<VERSIONREGEXP>\)]
+
+ o Clean up DONE; archive to tar file every 2 weeks, update tar tvzf INDEX file.
+
+ o testing-updates suite: if binary-only and version << version in
+ unstable and source-ver ~= source-ver in testing; then map
+ unstable -> testing-updates ?
+
+ o hooks or configurability for debian specific checks (e.g. check_urgency, auto-building support)
+
+ o morgue needs auto-cleaning (?)
+
+ o dak stats: two modes, all included, seperate
+ o dak stats: add non-US
+ o dak stats: add ability to control components, architectures, archives, suites
+ o dak stats: add key to expand header
+
+================================================================================
+
+queue/approved
+--------------
+
+ o What to do with multi-suite uploads? Presumably hold in unapproved
+ and warn? Or what? Can't accept just for unstable or reject just
+ from stable.
+
+ o Whenever we check for anything in accepted we also need to check in
+ unapproved.
+
+ o non-sourceful uploads should go straight through if they have
+ source in accepted or the archive.
+
+ o security uploads on auric should be pre-approved.
+
+================================================================================
+
+Less Urgent
+-----------
+
+ o change utils.copy to try rename() first
+
+ o [hard, long term] unchecked -> accepted should go into the db, not
+ a suite, but similar. this would allow dak to get even faster,
+ make 'dak ls' more useful, decomplexify specialacceptedautobuild
+ and generally be more sane. may even be helpful to have e.g. new
+ in the DB, so that we avoid corner cases like the .orig.tar.gz
+ disappearing 'cos the package has been entirely removed but was
+ still on stayofexecution when it entered new.
+
+ o Logging [mostly done] (todo: 'dak clean-suites' (hard), .. ?)
+
+ o 'dak process-unchecked': the tar extractor class doesn't need to be redone for each package
+
+ o reverse of source-must-exist; i.e. binary-for-source-must-not-exist
+ o REJECT reminders in 'dak clean-queues'.
+ o 'dak examine-package' should check for conflicts and warn about them visavis priority [rmurray]
+ o store a list of removed/files versions; also compare against them.
+ [but be careful about scalability]
+
+ o dak examine-package: print_copyright should be a lot more intelligent
+ @ handle copyright.gz
+ @ handle copyright.ja and copyright
+ @ handle (detect at least) symlinks to another package's doc directory
+ @ handle and/or fall back on source files (?)
+
+ o To incorporate from utils:
+ @ unreject
+
+ o auto-purge out-of-date stuff from non-free/contrib so that testing and stuff works
+ o doogie's binary -> source index
+ o jt's web stuff, matt's changelog stuff (overlap)
+
+ o [Hard] Need to merge non-non-US and non-US DBs.
+
+ o Do a checkpc(1)-a-like which sanitizes a config files.
+ o fix parse_changes()/build_file_list() to sanity check filenames
+ o saftey check and/or rename debs so they match what they should be
+
+ o Improve 'dak compare-suites'.
+ o Need to optimize all the queries by using EXAMINE and building some INDEXs.
+ [postgresql 7.2 will help here]
+ o Need to enclose all the setting SQL stuff in transactions (mostly done).
+ o Need to finish 'dak init-db' (a way to sync dak.conf and the DB)
+ o Need the ability to rebuild all other tables from dists _or_ pools (in the event of disaster) (?)
+ o Make the --help and --version options do stuff for all scripts
+
+ o 'dak make-maintainers' can't handle whitespace-only lines (for the moment, this is feature)
+
+ o generic way of saying isabinary and isadsc. (?)
+
+ o s/distribution/suite/g
+
+ o --help doesn't work without /etc/dak/dak.conf (or similar) at
+ least existing.
+
+ o rename 'dak compare-suites' (clashes with existing 'dak compare-suites')...
+
+ * Harder:
+
+ o interrupting of stracing 'dak process-unchecked' causes exceptions errors from apt_inst calls
+ o dependency checking (esp. stable) (partially done)
+ o need to do proper rfc822 escaping of from lines (as opposed to s/\.//g)
+ o Revisit linking of binary->source in install() in dak.
+ o binary-all should be done on a per-source, per-architecture package
+ basis to avoid, e.g. the perl-modules problem.
+ o a source-missing-diff check: if the version has a - in it, and it
+ is sourceful, it needs orig and diff, e.g. if someone uploads
+ esound_0.2.22-6, and it is sourceful, and there is no diff ->
+ REJECT (version has a dash, therefore not debian native.)
+ o check linking of .tar.gz's to .dsc's.. see proftpd 1.2.1 as an example
+ o archive needs md5sum'ed regularly, but takes too long to do all
+ in one go; make progressive or weekly.
+ o something needs to clear out .changes files from p-u when
+ removing stuff superseded by newer versions. [but for now we have
+ 'dak clean-proposed-updates']
+ o test sig checking stuff in test/ (stupid thing is not modularized due to global abuse)
+ o when encountering suspicous things (e.g. file tainting) do something more drastic
+
+ * Easy:
+
+ o suite mapping and component mapping are parsed per changes file,
+ they should probably be stored in a dictionary created at startup.
+ o don't stat/md5sum files you have entries for in the DB, moron
+ boy (Dak.check_source_blah_blah)
+ o promote changes["changes"] to mandatory in dak.py(dump_vars)
+ after a month or so (or all .dak files contain in the queue
+ contain it).
+ o 'dak rm' should behave better with -a and without -b; see
+ gcc-defaults removal for an example.
+ o Reject on misconfigured kernel-package uploads
+ o utils.extract_component_from_section: main/utils -> main/utils, main rather than utils, main
+ o Fix 'dak process-unchecked' to warn if run when not in incoming or p-u
+ o dak should validate multi-suite uploads; only possible valid one
+ is "stable unstable"
+ o cron.daily* should change umask (aj sucks)
+ o byhand stuff should send notification
+ o 'dak rm' should update the stable changelog [joey]
+ o update tagdb.dia
+
+ * Bizzare/uncertain:
+
+ o drop rather dubious currval stuff (?)
+ o rationalize os.path.join() usage
+ o logging: hostname + pid ?
+ o ANAIS should be done in dak (?)
+ o Add an 'add' ability to 'dak rm' (? separate prog maybe)
+ o Handle the case of 1:1.1 which would overwrite 1.1 (?)
+ o maybe drop -r/--regex in 'dak ls', make it the default and
+ implement -e/--exact (a la joey's "elmo")
+ o dsc files are not checked for existence/perms (only an issue if
+ they're in the .dsc, but not the .changes.. possible?)
+
+ * Cleanups & misc:
+
+ o db_access' get_files needs to use exceptions not this None, > 0, < 0 return val BS (?)
+ o The untouchable flag doesn't stop new packages being added to ``untouchable'' suites
+
+================================================================================
+
+Packaging
+---------
+
+ o Fix stuff to look in sensible places for libs and config file in debian package (?)
+
+================================================================================
+
+ --help manpage
+-----------------------------------------------------------------------------
+check-archive X
+check-overrides X X
+check-proposed-updates X
+clean-proposed-updates X
+clean-queues X
+clean-suites X X
+compare-suites X
+control-overrides X X
+control-suite X X
+cruft-report X
+decode-dot-dak X
+examine-package X
+generate-releases X
+import-archive X
+import-users-from-passwd X X
+init-db X
+init-dirs X
+ls X X
+make-maintainers X X
+make-overrides X
+make-suite-file-list X
+poolize X X
+process-accepted X X
+process-new X X
+process-unchecked X
+queue-report X
+rm X X
+security-install X
+stats X
+symlink-dists X
+
+
+================================================================================
+
+Random useful-at-some-point SQL
+-------------------------------
+
+UPDATE files SET last_used = '1980-01-01'
+ FROM binaries WHERE binaries.architecture = <x>
+ AND binaries.file = files.id;
+
+DELETE FROM bin_associations
+ WHERE EXISTS (SELECT id FROM binaries
+ WHERE architecture = <x>
+ AND id = bin_associations.bin);
+
+================================================================================
--- /dev/null
+ftpmaster meta package for DSA
+
+
+This is a dummy package that makes Debian's package management
+system believe that certain packages needed for ftpmaster have
+to be installed. The intention is that DSA can easily see what
+needs to be there.
+
+If you, for whatever reason, need a package added to this meta-
+packages dependency list, contact ftpmaster@debian.org, NOT
+the Debian admins.
--- /dev/null
+ftpmaster-lenny (1.0) unstable; urgency=low
+
+ * New "package", to help DSA
+
+ -- Joerg Jaspert <joerg@debian.org> Mon, 09 Mar 2009 14:09:09 +0100
+
--- /dev/null
+This package was put together by:
+
+ Joerg Jaspert <joerg@debian.org> on Mon, 09 Mar 2009 14:07:44 +0100
+
+Copyright:
+
+ <Copyright (C) 2009 Joerg Jaspert>
+
+License:
+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 2 as
+ published by the Free Software Foundation.
+
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this package; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+On Debian systems, the complete text of the GNU General
+Public License can be found in `/usr/share/common-licenses/GPL'.
--- /dev/null
+Section: devel
+Priority: optional
+Standards-Version: 3.8.1
+
+Package: ftpmaster-lenny
+Version: 1.0
+Maintainer: Debian FTP Master <ftpmaster@debian.org>
+Depends: apt-utils, bicyclerepair, binutils-multiarch, build-essential, bzip2, cron, curl, cvs, debian-el, debian-bug, dpkg-dev-el, easypg, devscripts, emacs-goodies-el, emacs22-nox, gnupg, gpgv, graphviz, ikiwiki, irb, libapt-pkg-dev, libdbd-pg-ruby, lintian, mc, mutt, postgresql-plperl-8.3, pychecker, pylint, pymacs, python, python-apt, python-btsutils, python-debian, python-epydoc, python-ldap, python-mode, python-numpy, python-psycopg2, python-pygresql, python-pyrss2gen, python-soappy, python-yaml, r-base, rsync, ruby, ruby-elisp, subversion, git-core, symlinks
+Architecture: all
+Copyright: copyright
+Changelog: changelog
+Readme: README.Debian
+Description: Meta package for DSA listing ftpmaster needs
+ This is a small meta package for the Debian System Administrators
+ so DSA easily knows (and can keep installed) all packages
+ Ftpmaster needs.
+ .
+ If, for whatever reason, you need a package added to this ones
+ Dependencies, ask ftpmaster, not the Debian admins.
+ .
+ This is not only "What DAK needs", but a general "FTPMaster needs this
+ to do the work"
#!/bin/bash
#
-# $Id: ddtp_i18n_check.sh 1186 2008-08-12 18:31:25Z faw $
+# $Id: ddtp_i18n_check.sh 1670 2009-03-31 20:57:49Z nekral-guest $
#
# Copyright (C) 2008, Felipe Augusto van de Wiel <faw@funlabs.org>
-# Copyright (C) 2008, Nicolas François <nicolas.francois@centraliens.net>
+# Copyright (C) 2008, 2009 Nicolas François <nicolas.francois@centraliens.net>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# Otherwise, list all the errors.
DEBUG=0
+# When DRY_RUN=0, generate the compressed version of the Translation-*
+# files.
+DRY_RUN=0
+
+dists_parent_dir=""
+# If no argument indicates the PACKAGES_LISTS_DIR then use '.'
+PACKAGES_LISTS_DIR=""
+
+usage () {
+ echo "Usage: $0 [options] <dists_parent_dir> [<packages_lists_directory>]" >&2
+ echo "" >&2
+ echo " --debug Debug mode: do not stop after the first error" >&2
+ echo " --dry-run Do not generate the compressed version of the " >&2
+ echo " Translation files">&2
+ exit 1
+}
+
+# Parse options
+for opt; do
+ case "$opt" in
+ "--debug")
+ DEBUG=1
+ ;;
+ "--dry-run")
+ DRY_RUN=1
+ ;;
+ "-*")
+ usage
+ ;;
+ "")
+ echo "Empty parameter" >&2
+ echo "" >&2
+ usage
+ ;;
+ *)
+ if [ -z "$dists_parent_dir" ]; then
+ # Removing trailing /
+ dists_parent_dir=${opt%/}
+ elif [ -z "$PACKAGES_LISTS_DIR" ]; then
+ PACKAGES_LISTS_DIR=$opt
+ else
+ echo "$0: Invalid option: $opt" >&2
+ usage
+ fi
+ ;;
+ esac
+done
+PACKAGES_LISTS_DIR=${opt:-.}
+
+if [ ! -d "$dists_parent_dir" ]; then
+ echo "missing dists_parent_dir, or not a directory" >&2
+ echo "" >&2
+ usage
+elif [ ! -d "$PACKAGES_LISTS_DIR" ]; then
+ echo "missing packages_lists_directory, or not a directory" >&2
+ echo "" >&2
+ usage
+fi
+
#STABLE="lenny"
TESTING="squeeze"
UNSTABLE="sid"
# These special files must exist on the top of dists_parent_dir
SPECIAL_FILES="$SHA256SUMS $TIMESTAMP $TIMESTAMP.gpg"
-usage () {
- echo "Usage: $0 <dists_parent_dir> [<packages_lists_directory>]" >&2
- exit 1
-}
-
-if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -d $1 ]
-then
- usage
-fi
-
# Temporary working directory. We need a full path to reduce the
# complexity of checking SHA256SUMS and cleaning/removing TMPDIR
TEMP_WORK_DIR=$(mktemp -d -t ddtp_dinstall_tmpdir.XXXXXX)
cd "$OLDPWD"
unset TEMP_WORK_DIR
-# If it's traped, something bad happened.
+# If it's trapped, something bad happened.
trap_exit () {
rm -rf "$TMP_WORK_DIR"
- rm -f "$dists_parent_dir"/dists/*/main/i18n/Translation-*.{bz2,gz}
+ rm -f "$dists_parent_dir"/dists/*/main/i18n/Translation-*.bz2
exit 1
}
trap trap_exit EXIT HUP INT QUIT TERM
-# If no argument indicates the PACKAGES_LISTS_DIR then use '.'
-PACKAGES_LISTS_DIR=${2:-.}
-
-if [ ! -d "$PACKAGES_LISTS_DIR" ]
-then
- usage
-fi
-
-# Removing trailing /
-dists_parent_dir=${1%/}
-
is_filename_okay () {
ifo_file="$1"
# We do not check if the md5 in Translation-$lang are
# correct.
- # Now generate files
- # Compress the file
- bzip2 -c "$f" > "$f.bz2"
- gzip -c "$f" > "$f.gz"
+ if [ "$DRY_RUN" = "0" ]; then
+ # Now generate the compressed files
+ bzip2 "$f"
+ fi
else
echo "Neither a file or directory: $f" >&2
exit 1
# keep 14 days, all each day
# keep 31 days, 1 each 7th day
# keep 365 days, 1 each 31th day
+ # keep 3650 days, 1 each 365th day
]
TODAY = datetime.today()
--- /dev/null
+This file maps each file available in the Debian GNU/Linux system to
+the package from which it originates. It includes packages from the
+DIST distribution for the ARCH architecture.
+
+You can use this list to determine which package contains a specific
+file, or whether or not a specific file is available. The list is
+updated weekly, each architecture on a different day.
+
+When a file is contained in more than one package, all packages are
+listed. When a directory is contained in more than one package, only
+the first is listed.
+
+As all Contents files are shipped compressed, the best way to search quickly
+for a file is with the Unix `zgrep' utility, as in:
+ `zgrep <regular expression> CONTENTS.gz':
+
+ $ zgrep nose Contents.gz
+ etc/nosendfile net/sendfile
+ usr/X11R6/bin/noseguy x11/xscreensaver
+ usr/X11R6/man/man1/noseguy.1x.gz x11/xscreensaver
+ usr/doc/examples/ucbmpeg/mpeg_encode/nosearch.param graphics/ucbmpeg
+ usr/lib/cfengine/bin/noseyparker admin/cfengine
+
+This list contains files in all packages, even though not all of the
+packages are installed on an actual system at once. If you want to
+find out which packages on an installed Debian system provide a
+particular file, you can use `dpkg --search <filename>':
+
+ $ dpkg --search /usr/bin/dselect
+ dpkg: /usr/bin/dselect
+
+
+FILE LOCATION
--- /dev/null
+From: __DAK_ADDRESS__
+To: __TO_ADDRESS__
+X-Debian: DAK
+X-Debian-Package: __PACKAGE__
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Missing contents for __PACKAGE__ in accepted queue
+
+While processing the accepted queue, I didn't have contents in the
+database for __PACKAGE__ version __VERSION__ for arch __ARCH__. These
+contents should have been put into the database by process-unchecked
+when the package first arrived.
+
+This is probably stew's fault.
override file for the following file(s):
__SUMMARY__
+
+Please note that a list of new sections were recently added to the
+archive: cli-mono, database, debug, fonts, gnu-r, gnustep, haskell,
+httpd, java, kernel, lisp, localization, ocaml, php, ruby, vcs, video,
+xfce, zope. At this time a script was used to reclassify packages into
+these sections. If this is the case, please only reply to this email if
+the new section is inappropriate, otherwise please update your package
+at the next upload.
+
Either the package or the override file is incorrect. If you think
the override is correct and the package wrong please fix the package
so that this disparity is fixed in the next upload. If you feel the
===
-If you don't understand why your files were rejected, or if the
-override file requires editing, reply to this email.
+Please feel free to respond to this email if you don't understand why
+your files were rejected, or if you upload new files which address our
+concerns.
+
[single issue]
-For the stable distribution (etch), this problem has been fixed in version XXX
+For the old stable distribution (etch), this problem has been fixed in version XXX
+__PACKAGE__
+
+For the stable distribution (lenny), this problem has been fixed in version XXX
__PACKAGE__
For the unstable distribution (sid), this problem has been fixed in
version XXX
[multiple issues]
-For the stable distribution (etch), these problems have been fixed in version
+For the old stable distribution (etch), these problems have been fixed in version
+__PACKAGE__
+
+For the stable distribution (lenny), these problems have been fixed in version
__PACKAGE__
For the unstable distribution (sid), these problems have been fixed in
Debian GNU/Linux 4.0 alias etch
-------------------------------
+Debian GNU/Linux 5.0 alias lenny
+--------------------------------
+
__ADVISORY_TEXT__
# Author: Filippo Giunchedi <filippo@debian.org>
# Version: 0.4
+import cgi
import os
import os.path
import cPickle
return False
description = "<pre>Description: %s\nChanges: %s\n</pre>" % \
- (utf2ascii(msg['Description']), utf2ascii(msg['Changes']))
+ (utf2ascii(cgi.escape(msg['Description'])), utf2ascii(cgi.escape(msg['Changes'])))
+
+ link = "http://ftp-master.debian.org/new/%s_%s.html" % \
+ (msg['Source'], msg['Version'])
feed.items.insert(0,
PyRSS2Gen.RSSItem(
title,
pubDate = pubdate,
description = description,
- author = utf2ascii(msg['Maintainer']),
- link = "http://ftp-master.debian.org/new/%s_%s.html" % \
- (msg['Source'], msg['Version'])
+ author = utf2ascii(cgi.escape(msg['Maintainer'])),
+ link = link,
+ guid = link
)
)
$rss->add_item(title => "$reason",
link => "http://ftp-master.debian.org/removals.txt?" . $link,
- description => qq[<pre>$body</pre>],
+ description => qq[<pre>$body</pre>],
dc => {
creator => "$ftpmaster",
}
-
-<h1>Really crappy page documenting archive criteria</h1>
-
-<h2>Architectures</h2>
-
-<p><b>Release candidates:</b> alpha, amd64, hppa, i386, ia64, mips, mipsel, powerpc
-<br><b>Release hopefuls:</b> arm, s390, sparc
-
-<p><b>Requalification expected:</b> m68k
-<br><b>Future linux ports:</b> armeb
-<br><b>New OS hopefuls:</b> <a href="http://wiki.debian.org/ArchiveQualification/kfreebsd-i386">kfreebsd-i386</a>, win32-i386
-
-<h2>Requirements for architectures</h2>
-
-<p>Examples: amd64, arm, armeb, m68k, s390, sparc
-
-<ul>
-<li>Is port cursed?
-<li>Are machines available to general public?
-<li>Is full source available?
-<li>Is this architecture related to other architectures already in the archive,
-or that also should be considered, either now or in the future? Can the related
-architectures be supported in a single architecture (eg, with a biarch arrangement)?
-<li>Are there 3 or more developers (or n-ms) actively maintaining the port? Who are they?
-<li>What sort of architecture is this? Desktop/workstation? Mainframe/supercomputer? Embedded? Something else?
-<li>Does it have any users? If a desktop system, are there Debian admins who
-run Debian systems on the arch? If an embedded system are there real systems
-shipping that a Debian port will be useful for? If a mainframe system are there
-real systems with many users that a Debian port will be useful for? Who are they?
-<li>Is there kernel and toolchain support? At what level? Are the latest versions supported, or are
-legacy releases required for compatability with some hardware?
-<li>Has the ABI stabalised, or are there major ABI changes coming
-up? Is the ABI stable enough to ensure users will be able just "apt-get
-dist-upgrade" from one version to the next?
-<li>How do you install a system? (URL to a HOWTO)
-<li>Has a buildd been setup? How much of the archive has been built (count by
-source package, builds of old versions are fine for this case)?
-<li>What hardware is potentially available as a fast buildd?
-<li>Is there any corporate support of this arch, and the Debian port in particular?
-<li>Is there an example box developers can login to to see if it works?
-</ul>
-
-<p>It's also worth considering whether the port has any special
-requirements. If the port is mainly for embedded systems, it may be
-appropriate to have different installation or release arrangements
-compared to normal desktop/workstation architectures.
-
-<h2>Further requirements for OSes</h2>
-
-<p>Examples: hurd, opensolaris, kfreebsd
-
-<ul>
-<li>Are there existing comprehensive free distributions of this OS? If
-so, why is a Debian distribution useful?
-<li>What demonstrable benefits does this OS have over existing Debian OSes?
-<li>Does this system have a standard Unix API?
-<li>Does the OS support modern glibc and gcc?
-<li>What is the license on the kernel and libraries? Is it free? Is it GPL
-compatible? (Note that if it's not free, building software for it violates the
-Social Contract; and if it's not GPL compatible, GPL software such as dpkg can't be
-linked to it)
-<li>Does the OS build largely without source changes? If so, what proportion of
-the archive has built?
-</ul>
-
-<p>It's worth thinking about whether it makes sense to integrate the
-port with Debian's Linux-based distribution -- having separate sources
-may not only reduce the impact on the release architectures, but also
-make it easier to do development on the new OS as well.
-
-<p>Note that if significant changes are needed to more than just a small
-number of packages, your porting team will not only need to provide
-patches for most of those changes and make sure they work, but also
-ensure they don't cause problems for existing ports.
-
-
-
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
+ <head>
+ <meta http-equiv="content-type" content="text/xhtml+xml; charset=utf-8" />
+ <link type="text/css" rel="stylesheet" href="style.css" />
+ <link rel="shortcut icon" href="http://www.debian.org/favicon.ico" />
+ <title>
+ Debian Archive criteria
+ </title>
+ </head>
+ <body id="ARCHIVE">
+
+ <div id="logo">
+ <a href="http://www.debian.org/">
+ <img src="http://www.debian.org/logos/openlogo-nd-50.png"
+ alt="debian logo" /></a>
+ <a href="http://www.debian.org/">
+ <img src="http://www.debian.org/Pics/debian.png"
+ alt="Debian Project" /></a>
+ </div>
+ <div id="titleblock">
+
+ <img src="http://www.debian.org/Pics/red-upperleft.png"
+ id="red-upperleft" alt="corner image"/>
+
+ <img src="http://www.debian.org/Pics/red-lowerleft.png"
+ id="red-lowerleft" alt="corner image"/>
+ <img src="http://www.debian.org/Pics/red-upperright.png"
+ id="red-upperright" alt="corner image"/>
+ <img src="http://www.debian.org/Pics/red-lowerright.png"
+ id="red-lowerright" alt="corner image"/>
+ <span class="title">
+ Debian Archive criteria
+ </span>
+ </div>
+
+ <h2>Definitions</h2>
+ <table class="DEFINITION">
+ <thead>
+ <tr>
+ <th> </th>
+ <th>Example</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr class="odd">
+ <td>Architecture</td>
+ <td>amd64, armel, alpha, m68k. Basically everything that uses
+ the Linux kernel.</td>
+ </tr>
+ <tr class="even">
+ <td>OS</td>
+ <td>hurd, opensolaris, kfreebsd. Ports that do not use the
+ Linux kernel, but their own.</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <p>
+ A new architecture has to follow the <em>Rules for new architectures</em>,
+ and answer all <em>Questions for new architectures</em>.
+ </p>
+ <p>
+ A new OS has to follow the <em>Rules for new architectures</em> and
+ answer all <em>Questions for new architectures</em> as well as all
+ <em>Further questions for OSes</em>.
+ </p>
+
+ <p>To have the answers all at one location, please create a page below
+ <a href="http://wiki.debian.org/ArchiveQualification/">wiki.debian.org/ArchiveQualification/</a>.
+ </p>
+
+ <h1>Rules for existing architectures</h1>
+
+ <ul>
+ <li>If an architecture fails to be included in 2 successive
+ official releases, it is moved out of the official archive (and
+ away from the ftp-master.debian.org host).</li>
+
+ <li>If a removed architecture later can prove it will be able to
+ make the next official release, it can be re-included into the
+ official archive. This step additionally needs the acceptance of
+ the Security, the Release and the Debian Admin Team. (It needs
+ security autobuilders, porter machines, etc.)</li>
+ </ul>
+
+ <h1>Rules for new architectures</h1>
+ <ul>
+ <li>A newly included architecture has to be completely built using
+ packages available in plain Debian sources. External patches cannot
+ be used.</li>
+
+ <li>At the time of inclusion a minimal set of binary packages will be
+ imported into the archive, just enough to get build-essential ready to
+ go and an official buildd setup and running. Everything else will be
+ rebuilt from scratch. As soon as enough is rebuilt to get the initial
+ toolchain built using "native" Debian, this will be rebuilt too.</li>
+
+ <li>The packages imported from external source and used for the initial
+ build run must be signed by one of the lead porters, who must be a DD.</li>
+
+ <li>There must be at least two machines ready to be maintained
+ by the Debian System Administrators, so at the start of its
+ lifetime there will be at least one buildd and one porter machine.<br />
+
+ The inclusion into the archive will almost certainly happen before
+ the machines are handed over to DSA, but this should happen as
+ soon as feasible afterwards.
+
+ (Note that this is the minimum to get into the archive. The release team
+ may have additional requirements to allow the architecture to release, so
+ there would normally need to be more machines, especially more
+ buildds.)<br />
+
+ <b>Note:</b> The machines, their setup and hosting etc should be
+ coordinated with DSA and needs to be acceptable to DSA. Please
+ <a href="mailto:debian-admin@lists.debian.org">coordinate with
+ them</a>, they might be able to help you in more ways
+ you can imagine, but at least they can help to avoid useless work
+ if a hosting wouldnt be acceptable. :)
+ </li>
+ </ul>
+
+ <h1>Questions for new architectures</h1>
+ <ul>
+ <!-- <li>Is port cursed?</li> -->
+ <li>Are machines available to buy for the general public?</li>
+ <li>Is full source available?</li>
+ <li>Is this architecture related to other architectures already in
+ the archive, or that also should be considered, either now or in
+ the future? Can the related architectures be supported in a single
+ architecture (eg, with a biarch arrangement)?</li>
+ <li>Are there 3 or more developers (or NMs) actively maintaining
+ the port? Who are they?</li>
+ <li>What sort of architecture is this? Desktop/workstation?
+ Mainframe/supercomputer? Embedded? Something else?</li>
+ <li>Does it have any users? If a desktop system, are there Debian
+ admins who run Debian systems on the arch? If an embedded system
+ are there real systems shipping that a Debian port will be useful
+ for? If a mainframe system are there real systems with many users
+ that a Debian port will be useful for? Who are they?</li>
+ <li>Is there kernel and toolchain support? At what level? Are the
+ latest versions supported, or are legacy releases required for
+ compatability with some hardware?</li>
+ <li>Has the ABI stabalised, or are there major ABI changes coming
+ up? Is the ABI stable enough to ensure users will be able just
+ "apt-get dist-upgrade" from one version to the next?</li>
+ <li>How do you install a system? (URL to a HOWTO)</li>
+ <li>Has a buildd been setup? How much of the archive has been
+ built (count by source package, builds of old versions are fine
+ for this case)?</li>
+ <li>What hardware is potentially available as a fast buildd?</li>
+ <li>Is there an example box developers can login to to see if it
+ works?</li>
+ </ul>
+
+ <p>It's also worth considering whether the port has any special
+ requirements. If the port is mainly for embedded systems, it may be
+ appropriate to have different installation or release arrangements
+ compared to normal desktop/workstation architectures.</p>
+
+ <h1>Further questions for OSes</h1>
+
+ <ul>
+ <li>Are there existing comprehensive free distributions of this OS? If
+ so, why is a Debian distribution useful?</li>
+ <li>What demonstrable benefits does this OS have over existing
+ Debian OSes?</li>
+ <li>Does this system have a standard Unix API?</li>
+ <li>Does the OS support modern glibc and gcc?</li>
+ <li>What is the license on the kernel and core libraries? Is the license
+ free? Is the license GPL compatible? (Note that if it's not free, distributing
+ the software violates the Social Contract; and if it's not GPL compatible,
+ GPL software such as dpkg can't be linked to it)</li>
+ <li>Does the OS build largely without source changes? If so, what proportion of
+ the archive has built?</li>
+ </ul>
+
+ <p>It's worth thinking about whether it makes sense to integrate the
+ port with Debian's Linux-based distribution -- having separate sources
+ may not only reduce the impact on the release architectures, but also
+ make it easier to do development on the new OS as well.</p>
+
+ <p>Note that if significant changes are needed to more than just a small
+ number of packages, your porting team will not only need to provide
+ patches for most of those changes and make sure they work, but also
+ ensure they don't cause problems for existing ports.</p>
+
+
+ <div class="footer">
+ <p>
+ <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-xhtml10"
+ alt="Valid XHTML 1.0 Strict" height="31" width="88" /></a>
+ <a href="http://jigsaw.w3.org/css-validator/">
+ <img style="border:0;width:88px;height:31px" src="http://jigsaw.w3.org/css-validator/images/vcss"
+ alt="Valid CSS!" />
+ </a>
+ </p>
+ </div> </body> </html>
<p>The members of ftpmaster currently are divided into two groups, FTP Master and FTP Assistants.</p>
<p>Members of FTP Master are:</p>
<ul>
- <li>Ryan Murray</li>
<li>Joerg Jaspert</li>
+ <li>Mark Hymers</li>
</ul>
<p>The FTP Assistants are:</p>
<ul>
- <li>Mark Hymers</li>
<li>Kalle Kivimaa</li>
<li>Frank Lichtenheld</li>
<li>Mike O'Connor</li>
projects / dak.git / commitdiff