From: Joerg Jaspert Date: Sat, 25 Apr 2009 22:39:22 +0000 (+0200) Subject: Merge commit 'buxy/bugfixes' into merge X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=35cd0972b5e14dc8727403e13fccd30776f3ae02;hp=425e44739cd77ffa01294f23e94ae7eabd5f5ec8;p=dak.git Merge commit 'buxy/bugfixes' into merge * commit 'buxy/bugfixes': dak update-db: fixes to make it work on a fresh install dak init-db: properly initialize suite_architectures Signed-off-by: Joerg Jaspert --- diff --git a/config/debian/apt.conf b/config/debian/apt.conf index 0e5d337b..ea370008 100644 --- a/config/debian/apt.conf +++ b/config/debian/apt.conf @@ -200,39 +200,3 @@ tree "dists/experimental" BinOverride "override.sid.$(SECTION)"; SrcOverride "override.sid.$(SECTION).src"; }; - -tree "dists/etch-m68k" -{ - FakeDI "dists/unstable"; - FileList "/srv/ftp.debian.org/database/dists/etch-m68k_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/ftp.debian.org/database/dists/etch-m68k_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "m68k source"; - BinOverride "override.etch.$(SECTION)"; - ExtraOverride "override.etch.extra.$(SECTION)"; - SrcOverride "override.etch.$(SECTION).src"; -}; - -tree "dists/etch-m68k/main" -{ - FileList "/srv/ftp.debian.org/database/dists/etch-m68k_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "m68k"; - BinOverride "override.etch.main.$(SECTION)"; - SrcOverride "override.etch.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb"; -}; - -tree "dists/etch-m68k/non-free" -{ - FileList "/srv/ftp.debian.org/database/dists/etch-m68k_non-free_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "m68k"; - BinOverride "override.etch.main.$(SECTION)"; - SrcOverride "override.etch.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - Contents "$(DIST)/../Contents-udeb-nf"; -}; diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index d5776af1..332aa036 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -238,6 +238,7 @@ function buildd() { function buildd_dir() { # Rebuilt the buildd dir to avoid long times of 403 log "Regenerating the buildd incoming dir" + STAMP=$(date "+%Y%m%d%H%M") make_buildd_dir } @@ -268,6 +269,12 @@ function expire() { $scriptsdir/expire_dumps -d . -p -f "dump_*" } +function transitionsclean() { + log "Removing out of date transitions..." + cd $base + dak transitions -c -a +} + function reports() { # Send a report on NEW/BYHAND packages log "Nagging ftpteam about NEW/BYHAND packages" @@ -291,8 +298,10 @@ function bts() { function merkel2() { # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached - log "Trigger merkels projectb sync" + log "Trigger merkel/flotows projectb sync" ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1 + # Also trigger flotow, the ftpmaster test box + ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1 } function merkel3() { @@ -330,6 +339,8 @@ function stats() { cd $configdir $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data R --slave --vanilla < $base/misc/ftpstats.R + dak stats arch-space > $webdir/arch-space + dak stats pkg-nums > $webdir/pkg-nums } function aptftpcleanup() { @@ -726,6 +737,14 @@ GO=( ) stage $GO +GO=( + FUNC="transitionsclean" + TIME="transitionsclean" + ARGS="" + ERR="" +) +stage $GO + GO=( FUNC="reports" TIME="reports" @@ -804,7 +823,7 @@ GO=( FUNC="aptftpcleanup" TIME="apt-ftparchive cleanup" ARGS="" - ERR="" + ERR="false" ) stage $GO diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 0fb81470..756a37c3 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -47,6 +47,7 @@ Dinstall Transitions { + Notifications "team@release.debian.org"; TempPath "/srv/ftp.debian.org/tmp/"; }; @@ -173,6 +174,7 @@ Clean-Suites Process-New { AcceptedLockFile "/srv/ftp.debian.org/lock/unchecked.lock"; + LockDir "/srv/ftp.debian.org/lock/new/"; }; Check-Overrides @@ -503,6 +505,7 @@ SuiteMappings "map testing-security testing-proposed-updates"; "map-unreleased testing unstable"; "map-unreleased testing-proposed-updates unstable"; + "reject etch-m68k"; }; AutomaticByHandPackages { diff --git a/config/debian/dak.conf-etc b/config/debian/dak.conf-etc new file mode 100644 index 00000000..351f9d2c --- /dev/null +++ b/config/debian/dak.conf-etc @@ -0,0 +1,11 @@ +Config +{ + ries.debian.org + { + AllowLocalConfig "false"; + DatabaseHostname "ftp-master"; + DakConfig "/srv/ftp.debian.org/dak/config/debian/dak.conf"; + AptConfig "/srv/ftp.debian.org/dak/config/debian/apt.conf"; + } +} + diff --git a/dak/clean_suites.py b/dak/clean_suites.py index b5904836..d6a966b5 100755 --- a/dak/clean_suites.py +++ b/dak/clean_suites.py @@ -201,7 +201,6 @@ def clean(): before = time.time() sys.stdout.write("[Deleting from source table... ") projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date)) - projectB.query("DELETE FROM src_uploaders WHERE EXISTS (SELECT 1 FROM source s, files f WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = src_uploaders.source)" % (delete_date)) projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date)) sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))) diff --git a/dak/contents.py b/dak/contents.py index ebeb2bc7..b3929c1a 100755 --- a/dak/contents.py +++ b/dak/contents.py @@ -39,6 +39,8 @@ import os import logging import math import gzip +import threading +import Queue import apt_pkg from daklib import utils from daklib.binary import Binary @@ -71,9 +73,6 @@ OPTIONS -s, --suite={stable,testing,unstable,...} only operate on a single suite - - -a, --arch={i386,amd64} - only operate on a single architecture """ sys.exit(exit_code) @@ -111,42 +110,47 @@ olddeb_q = """PREPARE olddeb_q(int) as LIMIT 1""" # find me all of the contents for a given .deb -contents_q = """PREPARE contents_q(int,int,int,int) as +contents_q = """PREPARE contents_q(int,int) as + SELECT (p.path||'/'||n.file) AS fn, + s.section, + b.package, + b.architecture + FROM content_associations c join content_file_paths p ON (c.filepath=p.id) + JOIN content_file_names n ON (c.filename=n.id) + JOIN binaries b ON (b.id=c.binary_pkg) + JOIN override o ON (o.package=b.package) + JOIN section s ON (s.id=o.section) + WHERE o.suite = $1 AND o.type = $2 + AND b.type='deb' + ORDER BY fn""" + +# find me all of the contents for a given .udeb +udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as SELECT (p.path||'/'||n.file) AS fn, - comma_separated_list(s.section||'/'||b.package) - FROM content_associations c - JOIN content_file_paths p ON (c.filepath=p.id) - JOIN content_file_names n ON (c.filename=n.id) - JOIN binaries b ON (b.id=c.binary_pkg) - JOIN bin_associations ba ON (b.id=ba.bin) - JOIN override o ON (o.package=b.package) - JOIN section s ON (s.id=o.section) - WHERE (b.architecture = $1 OR b.architecture = $2) - AND ba.suite = $3 - AND o.suite = $3 - AND b.type = 'deb' - AND o.type = $4 - GROUP BY fn - ORDER BY fn""" - -udeb_contents_q = """PREPARE udeb_contents_q(int,int,int,int,int) as - SELECT (p.path||'/'||n.file) as fn, - comma_separated_list(s.section||'/'||b.package) - FROM content_associations c - JOIN content_file_paths p ON (c.filepath=p.id) - JOIN content_file_names n ON (c.filename=n.id) - JOIN binaries b ON (b.id=c.binary_pkg) - JOIN bin_associations ba ON (b.id=ba.bin) - JOIN override o ON (o.package=b.package) - JOIN section s ON (s.id=o.section) - WHERE (b.architecture = $1 OR b.architecture = $2) - AND s.id = $3 - AND ba.suite = $4 - AND o.suite = $4 - AND b.type = 'udeb' - AND o.type = $5 - GROUP BY fn - ORDER BY fn""" + s.section, + b.package, + b.architecture + FROM content_associations c join content_file_paths p ON (c.filepath=p.id) + JOIN content_file_names n ON (c.filename=n.id) + JOIN binaries b ON (b.id=c.binary_pkg) + JOIN override o ON (o.package=b.package) + JOIN section s ON (s.id=o.section) + WHERE o.suite = $1 AND o.type = $2 + AND s.id = $3 + AND b.type='udeb' + ORDER BY fn""" + +# FROM content_file_paths p join content_associations c ON (c.filepath=p.id) +# JOIN content_file_names n ON (c.filename=n.id) +# JOIN binaries b ON (b.id=c.binary_pkg) +# JOIN override o ON (o.package=b.package) +# JOIN section s ON (s.id=o.section) +# WHERE o.suite = $1 AND o.type = $2 +# AND s.id = $3 +# AND b.id in (SELECT ba.bin from bin_associations ba join binaries b on b.id=ba.bin where (b.architecture=$3 or b.architecture=$4)and ba.suite=$1 and b.type='udeb') +# GROUP BY fn +# ORDER BY fn;""" + # clear out all of the temporarily stored content associations @@ -168,19 +172,85 @@ remove_filepath_cruft_q = """DELETE FROM content_file_paths LEFT JOIN content_associations ca ON ca.filepath=cfn.id WHERE ca.id IS NULL)""" -class Contents(object): + +class EndOfContents(object): """ - Class capable of generating Contents-$arch.gz files + A sentry object for the end of the filename stream + """ + pass - Usage GenerateContents().generateContents( ["main","contrib","non-free"] ) +class GzippedContentWriter(object): + """ + An object which will write contents out to a Contents-$arch.gz + file on a separate thread """ - def __init__(self): - self.header = None + header = None # a class object holding the header section of contents file - def reject(self, message): - log.error("E: %s" % message) + def __init__(self, filename): + """ + @ptype filename: string + @param filename: the name of the file to write to + """ + self.queue = Queue.Queue() + self.current_file = None + self.first_package = True + self.output = self.open_file(filename) + self.thread = threading.Thread(target=self.write_thread, + name='Contents writer') + self.thread.start() + + def open_file(self, filename): + """ + opens a gzip stream to the contents file + """ + filepath = Config()["Contents::Root"] + filename + filedir = os.path.dirname(filepath) + if not os.path.isdir(filedir): + os.makedirs(filedir) + return gzip.open(filepath, "w") + def write(self, filename, section, package): + """ + enqueue content to be written to the file on a separate thread + """ + self.queue.put((filename,section,package)) + + def write_thread(self): + """ + the target of a Thread which will do the actual writing + """ + while True: + next = self.queue.get() + if isinstance(next, EndOfContents): + self.output.write('\n') + self.output.close() + break + + (filename,section,package)=next + if next != self.current_file: + # this is the first file, so write the header first + if not self.current_file: + self.output.write(self._getHeader()) + + self.output.write('\n%s\t' % filename) + self.first_package = True + + self.current_file=filename + + if not self.first_package: + self.output.write(',') + else: + self.first_package=False + self.output.write('%s/%s' % (section,package)) + + def finish(self): + """ + enqueue the sentry object so that writers will know to terminate + """ + self.queue.put(EndOfContents()) + + @classmethod def _getHeader(self): """ Internal method to return the header for Contents.gz files @@ -188,52 +258,38 @@ class Contents(object): This is boilerplate which explains the contents of the file and how it can be used. """ - if self.header == None: + if not GzippedContentWriter.header: if Config().has_key("Contents::Header"): try: h = open(os.path.join( Config()["Dir::Templates"], Config()["Contents::Header"] ), "r") - self.header = h.read() + GzippedContentWriter.header = h.read() h.close() except: log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"], traceback.format_exc() )) - self.header = False + GzippedContentWriter.header = None else: - self.header = False + GzippedContentWriter.header = None - return self.header + return GzippedContentWriter.header - # goal column for section column - _goal_column = 54 - def _write_content_file(self, cursor, filename): - """ - Internal method for writing all the results to a given file. - The cursor should have a result set generated from a query already. - """ - filepath = Config()["Contents::Root"] + filename - filedir = os.path.dirname(filepath) - if not os.path.isdir(filedir): - os.makedirs(filedir) - f = gzip.open(filepath, "w") - try: - header = self._getHeader() +class Contents(object): + """ + Class capable of generating Contents-$arch.gz files - if header: - f.write(header) + Usage GenerateContents().generateContents( ["main","contrib","non-free"] ) + """ - while True: - contents = cursor.fetchone() - if not contents: - return + def __init__(self): + self.header = None - num_tabs = max(1, - int(math.ceil((self._goal_column - len(contents[0])-1) / 8))) - f.write(contents[0] + ( '\t' * num_tabs ) + contents[-1] + "\n") + def reject(self, message): + log.error("E: %s" % message) - finally: - f.close() + # goal column for section column + _goal_column = 54 def cruft(self): """ @@ -291,39 +347,99 @@ class Contents(object): """ Generate Contents-$arch.gz files for every available arch in each given suite. """ - cursor = DBConn().cursor(); + cursor = DBConn().cursor() - DBConn().prepare( "arches_q", arches_q ) - DBConn().prepare( "contents_q", contents_q ) - DBConn().prepare( "udeb_contents_q", udeb_contents_q ) + DBConn().prepare("arches_q", arches_q) + DBConn().prepare("contents_q", contents_q) + DBConn().prepare("udeb_contents_q", udeb_contents_q) debtype_id=DBConn().get_override_type_id("deb") udebtype_id=DBConn().get_override_type_id("udeb") + arch_all_id = DBConn().get_architecture_id("all") suites = self._suites() + # Get our suites, and the architectures for suite in [i.lower() for i in suites]: suite_id = DBConn().get_suite_id(suite) arch_list = self._arches(cursor, suite_id) - arch_all_id = DBConn().get_architecture_id("all") + file_writers = {} + + try: + for arch_id in arch_list: + file_writers[arch_id[0]] = GzippedContentWriter("dists/%s/Contents-%s.gz" % (suite, arch_id[1])) + + cursor.execute("EXECUTE contents_q(%d,%d);" % (suite_id, debtype_id)) + + while True: + r = cursor.fetchone() + if not r: + break + + filename, section, package, arch = r + + if not file_writers.has_key( arch ): + continue + + if arch == arch_all_id: + ## its arch all, so all contents files get it + for writer in file_writers.values(): + writer.write(filename, section, package) + + else: + file_writers[arch].write(filename, section, package) + + finally: + # close all the files + for writer in file_writers.values(): + writer.finish() - for arch_id in arch_list: - cursor.execute("EXECUTE contents_q(%d,%d,%d,%d)" % (arch_id[0], arch_all_id, suite_id, debtype_id)) - self._write_content_file(cursor, "dists/%s/Contents-%s.gz" % (suite, arch_id[1])) # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free) # This is HORRIBLY debian specific :-/ - for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"), - ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]: + for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"), + ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]: - for arch_id in arch_list: - section_id = DBConn().get_section_id(section) # all udebs should be here) - if section_id != -1: - cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d,%d,%d)" % (arch_id[0], arch_all_id, section_id, suite_id, udebtype_id)) + section_id = DBConn().get_section_id(section) # all udebs should be here) + if section_id != -1: + + # Get our suites, and the architectures + for suite in [i.lower() for i in suites]: + suite_id = DBConn().get_suite_id(suite) + arch_list = self._arches(cursor, suite_id) + + file_writers = {} + + try: + for arch_id in arch_list: + file_writers[arch_id[0]] = GzippedContentWriter(fn_pattern % (suite, arch_id[1])) + + cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id)) + + while True: + r = cursor.fetchone() + if not r: + break + + filename, section, package, arch = r + + if not file_writers.has_key( arch ): + continue + + if arch == arch_all_id: + ## its arch all, so all contents files get it + for writer in file_writers.values(): + writer.write(filename, section, package) + + else: + file_writers[arch].write(filename, section, package) + finally: + # close all the files + for writer in file_writers.values(): + writer.finish() - self._write_content_file(cursor, fn_pattern % (suite, arch_id[1])) ################################################################################ @@ -343,25 +459,21 @@ class Contents(object): """ return a list of archs to operate on """ - arch_list = [ ] - if Config().has_key( "%s::%s" %(options_prefix,"Arch")): - archs = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Arch")]) - for arch_name in archs: - arch_list.append((DBConn().get_architecture_id(arch_name), arch_name)) - else: - cursor.execute("EXECUTE arches_q(%d)" % (suite)) - while True: - r = cursor.fetchone() - if not r: - break + arch_list = [] + cursor.execute("EXECUTE arches_q(%d)" % (suite)) + while True: + r = cursor.fetchone() + if not r: + break - if r[1] != "source" and r[1] != "all": - arch_list.append((r[0], r[1])) + if r[1] != "source" and r[1] != "all": + arch_list.append((r[0], r[1])) return arch_list ################################################################################ + def main(): cnf = Config() @@ -369,7 +481,6 @@ def main(): ('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"), ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")), ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")), - ('a',"arch", "%s::%s" % (options_prefix,"Arch"),"HasArg"), ] commands = {'generate' : Contents.generate, diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 3c3d73cb..30650d41 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -168,7 +168,7 @@ def parse_nfu(architecture): f.close() else: - utils.warn("No wanna-build dump file for architecture %s", architecture) + utils.warn("No wanna-build dump file for architecture %s" % architecture) return ret ################################################################################ @@ -206,7 +206,7 @@ def do_nbs(real_nbs): output = "Not Built from Source\n" output += "---------------------\n\n" - nbs_to_remove = [] + cmd_output = "" nbs_keys = real_nbs.keys() nbs_keys.sort() for source in nbs_keys: @@ -216,21 +216,22 @@ def do_nbs(real_nbs): output += " but no longer builds:\n" versions = real_nbs[source].keys() versions.sort(apt_pkg.VersionCompare) + all_packages = [] for version in versions: packages = real_nbs[source][version].keys() packages.sort() - for pkg in packages: - nbs_to_remove.append(pkg) + all_packages.extend(packages) output += " o %s: %s\n" % (version, ", ".join(packages)) + if all_packages: + all_packages.sort() + cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite, " ".join(all_packages)) output += "\n" - if nbs_to_remove: + if len(cmd_output): print output - - print "Suggested command:" - print " dak rm -m \"[auto-cruft] NBS\" -s %s -b %s" % (suite, " ".join(nbs_to_remove)) - print + print "Suggested commands:\n" + print cmd_output ################################################################################ @@ -428,6 +429,8 @@ def main (): for component in check_components: architectures = filter(utils.real_arch, database.get_suite_architectures(suite)) for architecture in architectures: + if component == 'main/debian-installer' and re.match("kfreebsd", architecture): + continue filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture) # apt_pkg.ParseTagFile needs a real file handle (fd, temp_filename) = utils.temp_filename() diff --git a/dak/dakdb/__init__.py b/dak/dakdb/__init__.py index e69de29b..a35616dd 100644 --- a/dak/dakdb/__init__.py +++ b/dak/dakdb/__init__.py @@ -0,0 +1,33 @@ +""" +Database update scripts for usage with B{dak update-db} + +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@license: GNU General Public License version 2 or later + +Update scripts have to C{import psycopg2} and +C{from daklib.dak_exceptions import DBUpdateError}. + +There has to be B{at least} the function C{do_update(self)} to be +defined. It should take all neccessary steps to update the +database. If the update fails the changes have to be rolled back and the +C{DBUpdateError} exception raised to properly halt the execution of any +other update. + +Example:: + def do_update(self): + print "Doing something" + + try: + c = self.db.cursor() + c.execute("SOME SQL STATEMENT") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to do whatever, rollback issued. Error message : %s" % (str(msg)) + +This function can do whatever it wants and use everything from dak and +daklib. + +""" diff --git a/dak/dakdb/update1.py b/dak/dakdb/update1.py index 92fc4498..0c833773 100755 --- a/dak/dakdb/update1.py +++ b/dak/dakdb/update1.py @@ -1,7 +1,12 @@ #!/usr/bin/env python -""" Database Update Script - Saner DM db schema """ -# Copyright (C) 2008 Michael Casadevall +""" +Saner DM db schema + +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/dak/dakdb/update10.py b/dak/dakdb/update10.py new file mode 100755 index 00000000..5cd4f30d --- /dev/null +++ b/dak/dakdb/update10.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add constraints to src_uploaders + +@contact: Debian FTP Master +@copyright: 2009 Mark Hymers +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# oh no, Ganneff has just corrected my english + +################################################################################ + +import psycopg2 +import time +from daklib.dak_exceptions import DBUpdateError +from daklib.utils import get_conf + +################################################################################ + +def do_update(self): + print "Add constraints to src_uploaders" + Cnf = get_conf() + + try: + c = self.db.cursor() + # Deal with out-of-date src_uploaders entries + c.execute("DELETE FROM src_uploaders WHERE source NOT IN (SELECT id FROM source)") + c.execute("DELETE FROM src_uploaders WHERE maintainer NOT IN (SELECT id FROM maintainer)") + # Add constraints + c.execute("ALTER TABLE src_uploaders ADD CONSTRAINT src_uploaders_maintainer FOREIGN KEY (maintainer) REFERENCES maintainer(id) ON DELETE CASCADE") + c.execute("ALTER TABLE src_uploaders ADD CONSTRAINT src_uploaders_source FOREIGN KEY (source) REFERENCES source(id) ON DELETE CASCADE") + c.execute("UPDATE config SET value = '10' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)) diff --git a/dak/dakdb/update11.py b/dak/dakdb/update11.py new file mode 100755 index 00000000..53f1572a --- /dev/null +++ b/dak/dakdb/update11.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Adding process-new comments to the DB + +@contact: Debian FTP Master +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + + +################################################################################ + +import psycopg2 +import time + +################################################################################ + +def do_update(self): + print "Adding process-new comments to the DB" + + try: + c = self.db.cursor() + c.execute("""CREATE TABLE new_comments ( + id SERIAL PRIMARY KEY NOT NULL, + package TEXT NOT NULL, + version TEXT NOT NULL, + comment TEXT NOT NULL, + author TEXT NOT NULL + )""") + + c.execute("GRANT SELECT ON new_comments TO ftptrainee;") + c.execute("GRANT INSERT ON new_comments TO ftptrainee;") + c.execute("GRANT UPDATE ON new_comments TO ftptrainee;") + c.execute("GRANT SELECT ON new_comments TO ftpteam;") + c.execute("GRANT INSERT ON new_comments TO ftpteam;") + c.execute("GRANT UPDATE ON new_comments TO ftpteam;") + c.execute("GRANT ALL ON new_comments TO ftpmaster;") + + c.execute("UPDATE config SET value = '11' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply process-new comments update, rollback issued. Error message : %s" % (str(msg)) diff --git a/dak/dakdb/update12.py b/dak/dakdb/update12.py new file mode 100755 index 00000000..70a9e18e --- /dev/null +++ b/dak/dakdb/update12.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Adding a date field to the process-new notes + +@contact: Debian FTP Master +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + + +################################################################################ + +import psycopg2 +import time + +################################################################################ + +def do_update(self): + print "Adding a date field to the process-new notes" + + try: + c = self.db.cursor() + c.execute("ALTER TABLE new_comments ADD COLUMN notedate TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()") + + c.execute("UPDATE config SET value = '12' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply process-new update 12, rollback issued. Error message : %s" % (str(msg)) diff --git a/dak/dakdb/update2.py b/dak/dakdb/update2.py index 850e3ab5..2e3cb446 100755 --- a/dak/dakdb/update2.py +++ b/dak/dakdb/update2.py @@ -1,9 +1,14 @@ #!/usr/bin/env python # coding=utf8 -""" Database Update Script - debversion """ -# Copyright © 2008 Michael Casadevall -# Copyright © 2008 Roger Leigh +""" +debversion + +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@copyright: 2008 Roger Leigh +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/dak/dakdb/update3.py b/dak/dakdb/update3.py index ccd463c4..406cc89b 100755 --- a/dak/dakdb/update3.py +++ b/dak/dakdb/update3.py @@ -1,8 +1,13 @@ #!/usr/bin/env python -""" Database Update Script - Remove unused versioncmp """ -# Copyright (C) 2008 Michael Casadevall -# Copyright (C) 2009 Joerg Jaspert +""" +Remove unused versioncmp + +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/dak/dakdb/update4.py b/dak/dakdb/update4.py old mode 100644 new mode 100755 index 1a9d9c3a..477944c8 --- a/dak/dakdb/update4.py +++ b/dak/dakdb/update4.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ -Database Update Script - Get suite_architectures table use sane values +Get suite_architectures table use sane values @contact: Debian FTP Master @copyright: 2009 Joerg Jaspert diff --git a/dak/dakdb/update5.py b/dak/dakdb/update5.py index c89813e6..49e33891 100755 --- a/dak/dakdb/update5.py +++ b/dak/dakdb/update5.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -Database Update Script - Fix bin_assoc_by_arch view +Fix bin_assoc_by_arch view @contact: Debian FTP Master @copyright: 2009 Joerg Jaspert diff --git a/dak/dakdb/update6.py b/dak/dakdb/update6.py old mode 100644 new mode 100755 index 4537579a..c7b0b17f --- a/dak/dakdb/update6.py +++ b/dak/dakdb/update6.py @@ -2,11 +2,12 @@ # coding=utf8 """ -Debian Archive Kit Database Update Script -Copyright © 2008 Michael Casadevall -Copyright © 2008 Roger Leigh +Adding content fields -Debian Archive Kit Database Update Script 2 +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@copyright: 2008 Roger Leigh +@license: GNU General Public License version 2 or later """ # This program is free software; you can redistribute it and/or modify diff --git a/dak/dakdb/update7.py b/dak/dakdb/update7.py index c8828535..6f91eb37 100755 --- a/dak/dakdb/update7.py +++ b/dak/dakdb/update7.py @@ -2,11 +2,13 @@ # coding=utf8 """ -Debian Archive Kit Database Update Script -Copyright © 2008 Michael Casadevall -Copyright © 2009 Joerg Jaspert +Moving suite config into DB + +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later -Debian Archive Kit Database Update Script 7 """ # This program is free software; you can redistribute it and/or modify diff --git a/dak/dakdb/update8.py b/dak/dakdb/update8.py index fc505f7a..2f92c4d8 100755 --- a/dak/dakdb/update8.py +++ b/dak/dakdb/update8.py @@ -2,11 +2,12 @@ # coding=utf8 """ -Debian Archive Kit Database Update Script -Copyright © 2008 Michael Casadevall -Copyright © 2009 Joerg Jaspert +More suite config into the DB -Debian Archive Kit Database Update Script 8 +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later """ # This program is free software; you can redistribute it and/or modify diff --git a/dak/dakdb/update9.py b/dak/dakdb/update9.py old mode 100644 new mode 100755 index 09785773..2ca3c51e --- a/dak/dakdb/update9.py +++ b/dak/dakdb/update9.py @@ -2,11 +2,12 @@ # coding=utf8 """ -Debian Archive Kit Database Update Script -Copyright © 2008 Michael Casadevall -Copyright © 2009 Mike O'Connor +Pending contents disinguished by arch -Debian Archive Kit Database Update Script 8 +@contact: Debian FTP Master +@copyright: 2008 Michael Casadevall +@copyright: 2009 Mike O'Connor +@license: GNU General Public License version 2 or later """ # This program is free software; you can redistribute it and/or modify diff --git a/dak/process_accepted.py b/dak/process_accepted.py index e55ac54d..9883bb20 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -1,8 +1,14 @@ #!/usr/bin/env python -""" Installs Debian packages from queue/accepted into the pool """ -# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup +""" +Installs Debian packages from queue/accepted into the pool +@contact: Debian FTP Master +@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later + +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -218,7 +224,7 @@ def usage (exit_code=0): ############################################################################### -def action (): +def action (queue=""): (summary, short_summary) = Upload.build_summaries() (prompt, answer) = ("", "XXX") @@ -250,7 +256,7 @@ def action (): if not installing_to_stable: install() else: - stable_install(summary, short_summary) + stable_install(summary, short_summary, queue) elif answer == 'Q': sys.exit(0) @@ -488,10 +494,15 @@ def install (): ################################################################################ -def stable_install (summary, short_summary): +def stable_install (summary, short_summary, fromsuite="proposed-updates"): global install_count - print "Installing to stable." + fromsuite = fromsuite.lower() + tosuite = "Stable" + if fromsuite == "oldstable-proposed-updates": + tosuite = "OldStable" + + print "Installing from %s to %s." % (fromsuite, tosuite) # Begin a transaction; if we bomb out anywhere between here and # the COMMIT WORK below, the DB won't be changed. @@ -507,9 +518,9 @@ def stable_install (summary, short_summary): if not ql: utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version)) source_id = ql[0][0] - suite_id = database.get_suite_id('proposed-updates') + suite_id = database.get_suite_id(fromsuite) projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)) - suite_id = database.get_suite_id('stable') + suite_id = database.get_suite_id(tosuite.lower()) projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id)) # Add the binaries to stable (and remove it/them from proposed-updates) @@ -524,9 +535,9 @@ def stable_install (summary, short_summary): utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture)) binary_id = ql[0][0] - suite_id = database.get_suite_id('proposed-updates') + suite_id = database.get_suite_id(fromsuite) projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)) - suite_id = database.get_suite_id('stable') + suite_id = database.get_suite_id(tosuite.lower()) projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id)) projectB.query("COMMIT WORK") @@ -534,17 +545,17 @@ def stable_install (summary, short_summary): utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/process-accepted/' + os.path.basename(pkg.changes_file)) ## Update the Stable ChangeLog file - new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog" - changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog" + new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::%s::ChangeLogBase" % (tosuite)] + ".ChangeLog" + changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::%s::ChangeLogBase" % (tosuite)] + "ChangeLog" if os.path.exists(new_changelog_filename): os.unlink (new_changelog_filename) new_changelog = utils.open_file(new_changelog_filename, 'w') for newfile in files.keys(): if files[newfile]["type"] == "deb": - new_changelog.write("stable/%s/binary-%s/%s\n" % (files[newfile]["component"], files[newfile]["architecture"], newfile)) + new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.lower(), files[newfile]["component"], files[newfile]["architecture"], newfile)) elif re_issource.match(newfile): - new_changelog.write("stable/%s/source/%s\n" % (files[newfile]["component"], newfile)) + new_changelog.write("%s/%s/source/%s\n" % (tosuite.lower(), files[newfile]["component"], newfile)) else: new_changelog.write("%s\n" % (newfile)) chop_changes = re_fdnic.sub("\n", changes["changes"]) @@ -560,19 +571,19 @@ def stable_install (summary, short_summary): install_count += 1 if not Options["No-Mail"] and changes["architecture"].has_key("source"): - Subst["__SUITE__"] = " into stable" + Subst["__SUITE__"] = " into %s" % (tosuite) Subst["__SUMMARY__"] = summary mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-accepted.install") utils.send_mail(mail_message) Upload.announce(short_summary, 1) # Finally remove the .dak file - dot_dak_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyDotDak"], os.path.basename(Upload.pkg.changes_file[:-8]+".dak")) + dot_dak_file = os.path.join(Cnf["Suite::%s::CopyDotDak" % (fromsuite)], os.path.basename(Upload.pkg.changes_file[:-8]+".dak")) os.unlink(dot_dak_file) ################################################################################ -def process_it (changes_file): +def process_it (changes_file, queue=""): global reject_message reject_message = "" @@ -588,7 +599,7 @@ def process_it (changes_file): if installing_to_stable: old = Upload.pkg.changes_file Upload.pkg.changes_file = os.path.basename(old) - os.chdir(Cnf["Suite::Proposed-Updates::CopyDotDak"]) + os.chdir(Cnf["Suite::%s::CopyDotDak" % (queue)]) Upload.init_vars() Upload.update_vars() @@ -598,7 +609,7 @@ def process_it (changes_file): Upload.pkg.changes_file = old check() - action() + action(queue) # Restore CWD os.chdir(pkg.directory) @@ -620,7 +631,12 @@ def main(): utils.fubar("Archive maintenance in progress. Try again later.") # If running from within proposed-updates; assume an install to stable - if os.getcwd().find('proposed-updates') != -1: + queue = "" + if os.getenv('PWD').find('oldstable-proposed-updates') != -1: + queue = "Oldstable-Proposed-Updates" + installing_to_stable = 1 + elif os.getenv('PWD').find('proposed-updates') != -1: + queue = "Proposed-Updates" installing_to_stable = 1 # Obtain lock if not in no-action mode and initialize the log @@ -650,7 +666,7 @@ def main(): # Process the changes files for changes_file in changes_files: print "\n" + changes_file - process_it (changes_file) + process_it (changes_file, queue) if install_count: sets = "set" diff --git a/dak/process_new.py b/dak/process_new.py index 9ecfcdc6..52e1f4e9 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -5,6 +5,7 @@ @contact: Debian FTP Master @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup +@copyright: 2009 Joerg Jaspert @license: GNU General Public License version 2 or later """ # This program is free software; you can redistribute it and/or modify @@ -40,6 +41,8 @@ ################################################################################ +from __future__ import with_statement + import copy import errno import os @@ -47,6 +50,8 @@ import readline import stat import sys import time +import contextlib +import pwd import apt_pkg, apt_inst import examine_package from daklib import database @@ -54,6 +59,7 @@ from daklib import logging from daklib import queue from daklib import utils from daklib.regexes import re_no_epoch, re_default_answer, re_isanum +from daklib.dak_exceptions import CantOpenError, AlreadyLockedError # Globals Cnf = None #: Configuration, apt_pkg.Configuration @@ -112,7 +118,7 @@ def recheck(): if reject_message.find("Rejected") != -1: answer = "XXX" - if Options["No-Action"] or Options["Automatic"]: + if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: answer = 'S' print "REJECT\n" + reject_message, @@ -219,7 +225,7 @@ def sort_changes(changes_files): mtime = os.stat(d["filename"])[stat.ST_MTIME] if mtime < oldest: oldest = mtime - have_note += (d.has_key("process-new note")) + have_note += (database.has_new_comment(d["source"], d["version"])) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none @@ -301,11 +307,10 @@ def print_new (new, indexed, file=sys.stdout): line = "%-20s %-20s %-20s" % (pkg, priority, section) line = line.strip()+'\n' file.write(line) - note = Upload.pkg.changes.get("process-new note") - if note: - print "*"*75 - print note - print "*"*75 + note = database.get_new_comments(Upload.pkg.changes.get("source")) + if len(note) > 0: + for line in note: + print line return broken, note ################################################################################ @@ -468,18 +473,15 @@ def edit_overrides (new): def edit_note(note): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() - temp_file = os.fdopen(fd, 'w') - temp_file.write(note) - temp_file.close() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) temp_file = utils.open_file(temp_filename) - note = temp_file.read().rstrip() + newnote = temp_file.read().rstrip() temp_file.close() - print "Note:" - print utils.prefix_multi_line_string(note," ") + print "New Note:" + print utils.prefix_multi_line_string(newnote," ") prompt = "[D]one, Edit, Abandon, Quit ?" answer = "XXX" while prompt.find(answer) == -1: @@ -494,8 +496,7 @@ def edit_note(note): elif answer == 'Q': end() sys.exit(0) - Upload.pkg.changes["process-new note"] = note - Upload.dump_vars(Cnf["Dir::Queue::New"]) + database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami()) ################################################################################ @@ -572,16 +573,21 @@ def add_overrides (new): ################################################################################ -def prod_maintainer (): +def prod_maintainer (note): # Here we prepare an editor and get them ready to prod... (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + if len(note) > 0: + for line in note: + temp_file.write(line) + temp_file.close() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - f = os.fdopen(fd) - prod_message = "".join(f.readlines()) - f.close() + temp_fh = utils.open_file(temp_filename) + prod_message = "".join(temp_fh.readlines()) + temp_fh.close() print "Prod message:" print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) prompt = "[P]rod, Edit, Abandon, Quit ?" @@ -592,12 +598,12 @@ def prod_maintainer (): if answer == "": answer = m.group(1) answer = answer[:1].upper() - os.unlink(temp_filename) - if answer == 'A': - return - elif answer == 'Q': - end() - sys.exit(0) + os.unlink(temp_filename) + if answer == 'A': + return + elif answer == 'Q': + end() + sys.exit(0) # Otherwise, do the proding... user_email_address = utils.whoami() + " <%s>" % ( Cnf["Dinstall::MyAdminAddress"]) @@ -680,25 +686,27 @@ def do_new(): answer = m.group(1) answer = answer[:1].upper() - if answer == 'A': + if answer == 'A' and not Options["Trainee"]: done = add_overrides (new) elif answer == 'C': check_pkg() - elif answer == 'E': + elif answer == 'E' and not Options["Trainee"]: new = edit_overrides (new) - elif answer == 'M': - aborted = Upload.do_reject(1, Options["Manual-Reject"]) + elif answer == 'M' and not Options["Trainee"]: + aborted = Upload.do_reject(manual=1, + reject_message=Options["Manual-Reject"], + note=database.get_new_comments(changes.get("source", ""))) if not aborted: os.unlink(Upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'N': - edit_note(changes.get("process-new note", "")) - elif answer == 'P': - prod_maintainer() - elif answer == 'R': + edit_note(database.get_new_comments(changes.get("source", ""))) + elif answer == 'P' and not Options["Trainee"]: + prod_maintainer(database.get_new_comments(changes.get("source", ""))) + elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - del changes["process-new note"] + database.delete_new_comments(changes.get("source"), changes.get("version")) elif answer == 'S': done = 1 elif answer == 'Q': @@ -716,6 +724,7 @@ def usage (exit_code=0): -C, --comments-dir=DIR use DIR as comments-dir, for [o-]p-u-new -m, --manual-reject=MSG manual reject with `msg' -n, --no-action don't do anything + -t, --trainee FTP Trainee mode -V, --version display the version number and exit""" sys.exit(exit_code) @@ -730,9 +739,10 @@ def init(): ('h',"help","Process-New::Options::Help"), ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), + ('t',"trainee","Process-New::Options::Trainee"), ('n',"no-action","Process-New::Options::No-Action")] - for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]: + for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: if not Cnf.has_key("Process-New::Options::%s" % (i)): Cnf["Process-New::Options::%s" % (i)] = "" @@ -748,7 +758,10 @@ def init(): Upload = queue.Upload(Cnf) if not Options["No-Action"]: - Logger = Upload.Logger = logging.Logger(Cnf, "process-new") + try: + Logger = Upload.Logger = logging.Logger(Cnf, "process-new") + except CantOpenError, e: + Options["Trainee"] = "Oh yes" projectB = Upload.projectB @@ -825,6 +838,29 @@ def get_accept_lock(): else: raise + +@contextlib.contextmanager +def lock_package(package): + """ + Lock C{package} so that noone else jumps in processing it. + + @type package: string + @param package: source package name to lock + """ + + path = os.path.join(Cnf["Process-New::LockDir"], package) + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) + except OSError, e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') + raise AlreadyLockedError, user + + try: + yield fd + finally: + os.unlink(path) + def move_to_dir (dest, perms=0660, changesperms=0664): utils.move (Upload.pkg.changes_file, dest, perms=changesperms) file_keys = Upload.pkg.files.keys() @@ -949,19 +985,23 @@ def do_pkg(changes_file): Upload.update_subst() files = Upload.pkg.files - if not recheck(): - return - - (new, byhand) = check_status(files) - if new or byhand: - if new: - do_new() - if byhand: - do_byhand() - (new, byhand) = check_status(files) - - if not new and not byhand: - do_accept() + try: + with lock_package(Upload.pkg.changes["source"]): + if not recheck(): + return + + (new, byhand) = check_status(files) + if new or byhand: + if new: + do_new() + if byhand: + do_byhand() + (new, byhand) = check_status(files) + + if not new and not byhand: + do_accept() + except AlreadyLockedError, e: + print "Seems to be locked by %s already, skipping..." % (e) ################################################################################ @@ -976,7 +1016,7 @@ def end(): sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) Logger.log(["total",accept_count,accept_bytes]) - if not Options["No-Action"]: + if not Options["No-Action"] and not Options["Trainee"]: Logger.close() ################################################################################ diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index 8f9857f4..5fce9fa9 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -437,6 +437,15 @@ def check_files(): deb_file.close() # Can't continue, none of the checks on control would work. continue + + # Check for mandantory "Description:" + deb_file.seek ( 0 ) + try: + apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n' + except: + reject("%s: Missing Description in binary package" % (f)) + continue + deb_file.close() # Check for mandatory fields @@ -1003,10 +1012,10 @@ def lookup_uid_from_fingerprint(fpr): """ Return the uid,name,isdm for a given gpg fingerprint - @ptype fpr: string + @type fpr: string @param fpr: a 40 byte GPG fingerprint - @return (uid, name, isdm) + @return: (uid, name, isdm) """ cursor = DBConn().cursor() cursor.execute( "SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr)) @@ -1014,7 +1023,7 @@ def lookup_uid_from_fingerprint(fpr): if qs: return qs else: - return (None, None, None) + return (None, None, False) def check_signed_by_key(): """Ensure the .changes is signed by an authorized uploader.""" @@ -1024,17 +1033,22 @@ def check_signed_by_key(): uid_name = "" # match claimed name with actual name: - if uid == None: + if uid is None: + # This is fundamentally broken but need us to refactor how we get + # the UIDs/Fingerprints in order for us to fix it properly uid, uid_email = changes["fingerprint"], uid may_nmu, may_sponsor = 1, 1 # XXX by default new dds don't have a fingerprint/uid in the db atm, # and can't get one in there if we don't allow nmu/sponsorship - elif is_dm is "t": - uid_email = uid - may_nmu, may_sponsor = 0, 0 - else: + elif is_dm is False: + # If is_dm is False, we allow full upload rights uid_email = "%s@debian.org" % (uid) may_nmu, may_sponsor = 1, 1 + else: + # Assume limited upload rights unless we've discovered otherwise + uid_email = uid + may_nmu, may_sponsor = 0, 0 + if uid_email in [changes["maintaineremail"], changes["changedbyemail"]]: sponsored = 0 @@ -1053,6 +1067,7 @@ def check_signed_by_key(): if sponsored and not may_sponsor: reject("%s is not authorised to sponsor uploads" % (uid)) + cursor = DBConn().cursor() if not sponsored and not may_nmu: source_ids = [] cursor.execute( "SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = %(source)s AND s.dm_upload_allowed = 'yes'", changes ) @@ -1477,7 +1492,7 @@ def acknowledge_new (summary, short_summary): Logger.log(["Moving to new", pkg.changes_file]) Upload.dump_vars(Cnf["Dir::Queue::New"]) - move_to_dir(Cnf["Dir::Queue::New"]) + move_to_dir(Cnf["Dir::Queue::New"], perms=0640, changesperms=0644) if not Options["No-Mail"]: print "Sending new ack." diff --git a/dak/queue_report.py b/dak/queue_report.py index cc59a9f8..a4bcea0f 100755 --- a/dak/queue_report.py +++ b/dak/queue_report.py @@ -38,6 +38,7 @@ import copy, glob, os, stat, sys, time import apt_pkg import cgi from daklib import queue +from daklib import database from daklib import utils from daklib.dak_exceptions import * @@ -45,6 +46,7 @@ Cnf = None Upload = None direction = [] row_number = 0 +projectB = None ################################################################################ @@ -327,7 +329,7 @@ def process_changes_files(changes_files, type, log): else: if mtime < oldest: oldest = mtime - have_note += (d.has_key("process-new note")) + have_note += (database.has_new_comment(d["source"], d["version"])) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none @@ -531,6 +533,7 @@ def main(): usage() Upload = queue.Upload(Cnf) + projectB = Upload.projectB if Cnf.has_key("Queue-Report::Options::New"): header() diff --git a/dak/transitions.py b/dak/transitions.py index 90d2f620..00348f2d 100755 --- a/dak/transitions.py +++ b/dak/transitions.py @@ -66,14 +66,15 @@ def init(): Cnf = utils.get_conf() - Arguments = [('h',"help","Edit-Transitions::Options::Help"), + Arguments = [('a',"automatic","Edit-Transitions::Options::Automatic"), + ('h',"help","Edit-Transitions::Options::Help"), ('e',"edit","Edit-Transitions::Options::Edit"), ('i',"import","Edit-Transitions::Options::Import", "HasArg"), ('c',"check","Edit-Transitions::Options::Check"), ('s',"sudo","Edit-Transitions::Options::Sudo"), ('n',"no-action","Edit-Transitions::Options::No-Action")] - for i in ["help", "no-action", "edit", "import", "check", "sudo"]: + for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]: if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)): Cnf["Edit-Transitions::Options::%s" % (i)] = "" @@ -107,6 +108,7 @@ Options: -i, --import check and import transitions from file -c, --check check the transitions file, remove outdated entries -S, --sudo use sudo to update transitions file + -a, --automatic don't prompt (only affects check). -n, --no-action don't do anything (only affects check)""" sys.exit(exit_code) @@ -389,11 +391,14 @@ def edit_transitions(): def check_transitions(transitions): """ Check if the defined transitions still apply and remove those that no longer do. - @note: Asks the user for confirmation first. + @note: Asks the user for confirmation first unless -a has been set. """ + global Cnf + to_dump = 0 to_remove = [] + info = {} # Now look through all defined transitions for trans in transitions: t = transitions[trans] @@ -403,7 +408,8 @@ def check_transitions(transitions): # Will be None if nothing is in testing. current = database.get_suite_version(source, "testing") - print_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + info[trans] = get_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + print info[trans] if current == None: # No package in testing @@ -432,6 +438,8 @@ def check_transitions(transitions): if Options["no-action"]: answer="n" + elif Options["automatic"]: + answer="y" else: answer = utils.our_raw_input(prompt).lower() @@ -443,9 +451,25 @@ def check_transitions(transitions): sys.exit(0) elif answer == 'y': print "Committing" + subst = {} + subst['__TRANSITION_MESSAGE__'] = "The following transitions were removed:\n" for remove in to_remove: + subst['__TRANSITION_MESSAGE__'] += info[remove] + '\n' del transitions[remove] + # If we have a mail address configured for transitions, + # send a notification + subst['__TRANSITION_EMAIL__'] = Cnf.get("Transitions::Notifications", "") + if subst['__TRANSITION_EMAIL__'] != "": + print "Sending notification to %s" % subst['__TRANSITION_EMAIL__'] + subst['__DAK_ADDRESS__'] = Cnf["Dinstall::MyEmailAddress"] + subst['__BCC__'] = 'X-DAK: dak transitions' + if Cnf.has_key("Dinstall::Bcc"): + subst["__BCC__"] += '\nBcc: %s' % Cnf["Dinstall::Bcc"] + message = utils.TemplateSubst(subst, + os.path.join(Cnf["Dir::Templates"], 'transition.removed')) + utils.send_mail(message) + edit_file = temp_transitions_file(transitions) write_transitions_from_file(edit_file) @@ -456,7 +480,7 @@ def check_transitions(transitions): ################################################################################ -def print_info(trans, source, expected, rm, reason, packages): +def get_info(trans, source, expected, rm, reason, packages): """ Print information about a single transition. @@ -479,21 +503,20 @@ def print_info(trans, source, expected, rm, reason, packages): @param packages: list of blocked packages """ - print """Looking at transition: %s + return """Looking at transition: %s Source: %s New Version: %s Responsible: %s Description: %s Blocked Packages (total: %d): %s """ % (trans, source, expected, rm, reason, len(packages), ", ".join(packages)) - return ################################################################################ def transition_info(transitions): """ Print information about all defined transitions. - Calls L{print_info} for every transition and then tells user if the transition is + Calls L{get_info} for every transition and then tells user if the transition is still ongoing or if the expected version already hit testing. @type transitions: dict @@ -507,7 +530,7 @@ def transition_info(transitions): # Will be None if nothing is in testing. current = database.get_suite_version(source, "testing") - print_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + print get_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) if current == None: # No package in testing diff --git a/dak/update_db.py b/dak/update_db.py index 55dda5da..b559e544 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError Cnf = None projectB = None -required_database_schema = 9 +required_database_schema = 12 ################################################################################ diff --git a/daklib/binary.py b/daklib/binary.py index 88d78761..bd7f1cc1 100755 --- a/daklib/binary.py +++ b/daklib/binary.py @@ -56,10 +56,10 @@ import utils class Binary(object): def __init__(self, filename, reject=None): """ - @ptype filename: string + @type filename: string @param filename: path of a .deb - @ptype reject: function + @type reject: function @param reject: a function to log reject messages to """ self.filename = filename @@ -165,12 +165,12 @@ class Binary(object): the hopefully near future, it should also include gathering info from the control file. - @ptype bootstrap_id: int + @type bootstrap_id: int @param bootstrap_id: the id of the binary these packages should be associated or zero meaning we are not bootstrapping so insert into a temporary table - @return True if the deb is valid and contents were imported + @return: True if the deb is valid and contents were imported """ result = False rejected = not self.valid_deb(relaxed) @@ -212,12 +212,11 @@ class Binary(object): the hopefully near future, it should also include gathering info from the control file. - @ptype bootstrap_id: int - @param bootstrap_id: the id of the binary these packages - should be associated or zero meaning we are not bootstrapping - so insert into a temporary table + @type package: string + @param package: the name of the package to be checked - @return True if the deb is valid and contents were imported + @rtype: boolean + @return: True if the deb is valid and contents were imported """ rejected = not self.valid_deb(True) self.__unpack() diff --git a/daklib/dak_exceptions.py b/daklib/dak_exceptions.py index ccd63e50..21fce9be 100755 --- a/daklib/dak_exceptions.py +++ b/daklib/dak_exceptions.py @@ -60,7 +60,8 @@ dakerrors = { "NoSourceFieldError": """Exception raised - we cant find the source - wtf?""", "MissingContents": """Exception raised - we could not determine contents for this deb""", "DBUpdateError": """Exception raised - could not update the database""", - "ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded""" + "ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded""", + "AlreadyLockedError": """Exception raised - package already locked by someone else""" } #: All dak exceptions def construct_dak_exception(name, description): diff --git a/daklib/database.py b/daklib/database.py index 93d9ad53..a5255568 100755 --- a/daklib/database.py +++ b/daklib/database.py @@ -4,8 +4,9 @@ @group readonly: get_suite_id, get_section_id, get_priority_id, get_override_type_id, get_architecture_id, get_archive_id, get_component_id, get_location_id, get_source_id, get_suite_version, get_files_id, get_maintainer, get_suites, - get_suite_architectures + get_suite_architectures, get_new_comments, has_new_comment @group read/write: get_or_set*, set_files_id +@group writeonly: add_new_comment, delete_new_comments @contact: Debian FTP Master @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup @@ -838,6 +839,89 @@ def get_suites(pkgname, src=False): ################################################################################ +def get_new_comments(package): + """ + Returns all the possible comments attached to C{package} in NEW. All versions. + + @type package: string + @param package: name of the package + + @rtype: list + @return: list of strings containing comments for all versions from all authors for package + """ + + comments = [] + query = projectB.query(""" SELECT version, comment, author, notedate + FROM new_comments + WHERE package = '%s' + ORDER BY notedate + """ % (package)) + + for row in query.getresult(): + comments.append("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s\n" % (row[2], row[0], row[3], row[1])) + comments.append("-"*72) + + return comments + +def has_new_comment(package, version): + """ + Returns true if the given combination of C{package}, C{version} has a comment. + + @type package: string + @param package: name of the package + + @type version: string + @param version: package version + + @rtype: boolean + @return: true/false + """ + + exists = projectB.query("""SELECT 1 FROM new_comments + WHERE package='%s' + AND version='%s' + LIMIT 1""" + % (package, version) ).getresult() + + if not exists: + return False + else: + return True + +def add_new_comment(package, version, comment, author): + """ + Add a new comment for C{package}, C{version} written by C{author} + + @type package: string + @param package: name of the package + + @type version: string + @param version: package version + + @type comment: string + @param comment: the comment + + @type author: string + @param author: the authorname + """ + + projectB.query(""" INSERT INTO new_comments (package, version, comment, author) + VALUES ('%s', '%s', '%s', '%s') + """ % (package, version, comment, author) ) + + return + +def delete_new_comments(package, version): + """ + Delete a comment for C{package}, C{version}, if one exists + """ + + projectB.query(""" DELETE FROM new_comments + WHERE package = '%s' AND version = '%s' + """ % (package, version)) + return + +################################################################################ def copy_temporary_contents(package, version, arch, deb, reject): """ copy the previously stored contents from the temp table to the permanant one diff --git a/daklib/dbconn.py b/daklib/dbconn.py index b532c351..b2913722 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -203,8 +203,8 @@ class DBConn(Singleton): Returns database id for given override C{type}. Results are kept in a cache during runtime to minimize database queries. - @type type: string - @param type: The name of the override type + @type override_type: string + @param override_type: The name of the override type @rtype: int @return: the database id for the given override type @@ -500,10 +500,10 @@ class DBConn(Singleton): @type bin_id: int @param bin_id: the id of the binary - @type fullpath: string - @param fullpath: the path of the file being associated with the binary + @type fullpaths: list + @param fullpaths: the list of paths of the file being associated with the binary - @return True upon success + @return: True upon success """ c = self.db_con.cursor() @@ -514,6 +514,8 @@ class DBConn(Singleton): for fullpath in fullpaths: (path, file) = os.path.split(fullpath) + if path.startswith( "./" ): + path = path[2:] # Get the necessary IDs ... file_id = self.get_or_set_contents_file_id(file) path_id = self.get_or_set_contents_path_id(path) @@ -539,7 +541,7 @@ class DBConn(Singleton): @type fullpaths: list @param fullpaths: the list of paths of the file being associated with the binary - @return True upon success + @return: True upon success """ c = self.db_con.cursor() diff --git a/daklib/queue.py b/daklib/queue.py index 599b077f..35754c8d 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -803,7 +803,7 @@ distribution.""" ########################################################################### - def do_reject (self, manual = 0, reject_message = ""): + def do_reject (self, manual = 0, reject_message = "", note = ""): """ Reject an upload. If called without a reject message or C{manual} is true, spawn an editor so the user can write one. @@ -821,6 +821,11 @@ distribution.""" # editor so the user can add one in... if manual and not reject_message: (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + if len(note) > 0: + for line in note: + temp_file.write(line) + temp_file.close() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': diff --git a/daklib/utils.py b/daklib/utils.py index 651e13ae..c40fe8e0 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -745,6 +745,15 @@ def where_am_i (): def which_conf_file (): res = socket.gethostbyaddr(socket.gethostname()) + # In case we allow local config files per user, try if one exists + if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): + homedir = os.getenv("HOME") + confpath = os.path.join(homedir, "/etc/dak.conf") + if os.path.exists(confpath): + apt_pkg.ReadConfigFileISC(Cnf,default_config) + + # We are still in here, so there is no local config file or we do + # not allow local files. Do the normal stuff. if Cnf.get("Config::" + res[0] + "::DakConfig"): return Cnf["Config::" + res[0] + "::DakConfig"] else: @@ -752,6 +761,13 @@ def which_conf_file (): def which_apt_conf_file (): res = socket.gethostbyaddr(socket.gethostname()) + # In case we allow local config files per user, try if one exists + if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): + homedir = os.getenv("HOME") + confpath = os.path.join(homedir, "/etc/dak.conf") + if os.path.exists(confpath): + apt_pkg.ReadConfigFileISC(Cnf,default_config) + if Cnf.get("Config::" + res[0] + "::AptConfig"): return Cnf["Config::" + res[0] + "::AptConfig"] else: diff --git a/docs/README.quotes b/docs/README.quotes index 55c42be0..e531a241 100644 --- a/docs/README.quotes +++ b/docs/README.quotes @@ -345,10 +345,6 @@ Canadians: This is a lighthouse. Your call. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - oh no, Ganneff has just corrected my english - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - I often wonder if we should use NSA bot or something instead and get dinstall to send emails telling us about its progress :-) dinstall: I'm processing openoffice dinstall: I'm choking, please help me @@ -364,3 +360,10 @@ Canadians: This is a lighthouse. Your call. and if not - we can make it go multi-network first oftc, then opn, then ircnet, then - we will find some. quakenet anyone? I should know better than to give you ideas + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + !!!!11111iiiiiioneoneoneone + mhy: Error: "!!!11111iiiiiioneoneoneone" is not a valid command. + dak: oh shut up + mhy: Error: "oh" is not a valid command. diff --git a/docs/TODO b/docs/TODO index f142c66b..3a99ad94 100644 --- a/docs/TODO +++ b/docs/TODO @@ -3,6 +3,7 @@ Various ------- + * Implement autosigning, see ftpmaster_autosigning on ftp-master host in text/. * Check TODO.old and move still-valid/useful entries over here. diff --git a/docs/meta/lenny/ftpmaster-lenny b/docs/meta/lenny/ftpmaster-lenny index 14488625..4de221f5 100644 --- a/docs/meta/lenny/ftpmaster-lenny +++ b/docs/meta/lenny/ftpmaster-lenny @@ -5,7 +5,7 @@ Standards-Version: 3.8.1 Package: ftpmaster-lenny Version: 1.0 Maintainer: Debian FTP Master -Depends: apt-utils, bicyclerepair, binutils-multiarch, build-essential, bzip2, cron, curl, cvs, debian-el, debian-bug, dpkg-dev-el, easypg, devscripts, emacs-goodies-el, emacs22-nox, gnupg, gpgv, graphviz, ikiwiki, irb, libapt-pkg-dev, libdbd-pg-ruby, lintian, mc, mutt, postgresql-plperl-8.3, pychecker, pylint, pymacs, python, python-apt, python-btsutils, python-debian, python-epydoc, python-ldap, python-mode, python-numpy, python-psycopg2, python-pygresql, python-pyrss2gen, python-soappy, python-yaml, r-base, rsync, ruby, ruby-elisp, subversion, git-core, symlinks +Depends: apt-utils, bicyclerepair, binutils-multiarch, build-essential, bzip2, cron, curl, cvs, debian-el, dpkg-dev-el, easypg, devscripts, emacs-goodies-el, emacs22-nox, gnupg, gpgv, graphviz, ikiwiki, irb, libapt-pkg-dev, libdbd-pg-ruby, lintian, mc, mutt, postgresql-plperl-8.3, pychecker, pylint, pymacs, python, python-apt, python-btsutils, python-debian, python-epydoc, python-ldap, python-mode, python-numpy, python-psycopg2, python-pygresql, python-pyrss2gen, python-soappy, python-yaml, r-base, rsync, ruby, ruby-elisp, subversion, git-core, symlinks Architecture: all Copyright: copyright Changelog: changelog diff --git a/templates/contents b/templates/contents index 16b624f0..48950314 100644 --- a/templates/contents +++ b/templates/contents @@ -30,4 +30,4 @@ particular file, you can use `dpkg --search ': dpkg: /usr/bin/dselect -FILE LOCATION +FILE LOCATION \ No newline at end of file diff --git a/templates/transition.removed b/templates/transition.removed new file mode 100644 index 00000000..32b0aca1 --- /dev/null +++ b/templates/transition.removed @@ -0,0 +1,15 @@ +From: __DAK_ADDRESS__ +To: __TRANSITION_EMAIL__ +__BCC__ +X-Debian: DAK +Precedence: bulk +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 8bit +Subject: Transitions Completed + +The following transitions are complete and have been removed +from the transitions list: + +__TRANSITION_MESSAGE__ +