From: Joerg Jaspert Date: Thu, 1 May 2008 21:48:02 +0000 (+0200) Subject: Merge from security X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=212ecef347f2ec35e3f0261ed8885569f22235b7;hp=7de170d4c4e553e59d15fa8444fa025df3bef2c6;p=dak.git Merge from security --- diff --git a/ChangeLog b/ChangeLog index 55beb2a7..397994fd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -214,6 +214,17 @@ * dak/examine_package.py (check_deb): Remove linda call. It provides no added benefit to lintian anymore. +2008-01-07 Joerg Jaspert + + * dak/examine_package.py (check_deb): Remove linda call. It + provides no added benefit to lintian anymore. + +2008-01-06 Joerg Jaspert + + * dak/examine_package.py (do_lintian): lintian now supports html + coloring, so use it. + (do_command): Dont escape html chars if param escaped = 1 + 2008-01-06 Joerg Jaspert * dak/examine_package.py (do_lintian): lintian now supports html @@ -226,6 +237,84 @@ when processing result of check_dsc_against_db so we don't promote warnings to rejections. +2007-12-31 Anthony Towns + + * dak/process_new.py (recheck): pass "" for prefix_str to reject() + when processing result of check_dsc_against_db so we don't promote + warnings to rejections. + +2007-12-30 Joerg Jaspert + + * dak/dak.py (init): add show-new. This is based on a patch + submitted by Thomas Viehmann in Bug #408318, but large parts of + handling it are rewritten and show-new is done by me. + + * dak/queue_report.py (table_row): Add link to generated html page + for NEW package. + + * dak/show_new.py: new file, generates html overview for NEW + packages, similar to what we see with examine-package. + + * config/debian/cron.hourly: Add show-new call + + * config/debian/dak.conf: Add HTMLPath for Show-New + + * dak/examine_package.py (print_copyright): ignore stderr when + finding copyright file. + (main): add html option + (html_escape): new function + (escape_if_needed): ditto + (headline): ditto + (colour_output): ditto + (print_escaped_text): ditto + (print_formatted_text): ditto + - use those functions everywhere where we generate output, as they + easily know if we want html or not and just DTRT + (do_lintian): new function + (check_deb): use it + (output_deb_info): Use print_escaped_text, not print_formatted_text. + Also import daklib.queue, determine_new now lives there + + Also add a variable to see if we want html output. Default is + disabled, show_new enables it for its use. + Most of html, besides header/footer are in examine_package instead + of show_new, as it makes it a whole lot easier to deal with it at + the point the info is generated. + + + * dak/process_new.py (determine_new): Moved out of here. + (check_valid): Moved out of here. + (get_type): Moved out of here. + + * daklib/queue.py (determine_new): Moved here. + (check_valid): Moved here. + (get_type): Moved here. + + * dak/init_db.py (do_section): Remove non-US code + + * dak/make_overrides.py (main): ditto + + * dak/process_new.py (determine_new): ditto + + * daklib/queue.py (Upload.in_override_p), + (Upload.check_override): ditto + + * daklib/utils.py (extract_component_from_section):, + (poolify): ditto + + * dak/import_archive.py (update_section): ditto + + * dak/symlink_dists.py (fix_component_section): ditto + + * scripts/debian/mkmaintainers: ditto + + * scripts/debian/update-mirrorlists (masterlist): ditto + + * config/debian-non-US/*: Remove subdir + + * scripts/debian/update-readmenonus: Removed. + + 2007-12-30 Joerg Jaspert * dak/dak.py (init): add show-new. This is based on a patch @@ -298,6 +387,31 @@ * scripts/debian/update-readmenonus: Removed. +2007-12-28 Anthony Towns + + * daklib/utils.py (check_signature): add NOTATION_DATA and + NOTATION_NAME to known keywords. + + * daklib/queue.py (Upload.check_source_against_db): + + * dak/make_suite_file_list.py: add -f/--force option. + + * dak/generate_releases.py: add -a/--apt-conf=FILE and + -f/--force-touch options. Pull version info from the database. + Make suite description optional. + + * config/debian/dak.conf: update + Reject-Proposed-Updates::MoreInfoURL. Comment out + Suite::Stable::Version and ::Description. + + * config/debian/apt.conf: Add hurd-i386 to unstable + debian-installer stanza. + +2007-12-28 Joerg Jaspert + + * KEYEXPIRED is actually a known keyword. We do check it earlier + on and reject in case the sig is bad (or unknown) + 2007-12-28 Anthony Towns * daklib/utils.py (check_signature): add NOTATION_DATA and @@ -332,6 +446,20 @@ process data.tar.bz2 (or whatever format it will be in the future). +2007-12-24 Joerg Jaspert + + * Also run lintian on the .dsc file to check the source itself. + + * Fix the direct usage of ar | tar etc to get the copyright file + and use dpkg-deb, which is made for this and makes us able to + process data.tar.bz2 (or whatever format it will be in the + future). + +2007-12-21 Joerg Jaspert + + * Remove the (now useless) check for a pre-depends on dpkg for + binaries that contain bzip2 compressed data tarballs. + 2007-12-21 Joerg Jaspert * Remove the (now useless) check for a pre-depends on dpkg for @@ -350,10 +478,77 @@ holding queues, don't worry if dak has its own reasons for rejecting the package as well as the SRMs. +2007-08-28 Anthony Towns + + * process_unchecked.py: Add support for automatic BYHAND + processing. + * config/debian/dak.conf, scripts/debian/byhand-tag: Automatic + processing of tag-overrides. + * examine_package.py: Summarise duplicate copyright file entries + (same md5sum) with a reference to the previous instance, rather + than repeating them. + * process_new.py: When rejecting from the p-u-new or o-p-u-new + holding queues, don't worry if dak has its own reasons for + rejecting the package as well as the SRMs. + 2007-06-19 Anthony Towns * Add nm.debian.org pseudopackage +2007-06-19 Anthony Towns + + * Add nm.debian.org pseudopackage + +2007-06-18 Anthony Towns + + * daklib/logging.py: Set umask to not exclude group-writability + so we don't get reminded at the start of each month. Thanks to + Random J. + * dak/override.py: More changes from Herr von Wifflepuck: warn + if section of source is different to binary section; restore + functionality on source-only overrides; croak if trying to set + priority of a source override; never set priority of source + overrides; correct typo in logging (s/priority/section/ at + one place) + + * config/debian/apt.conf.oldstable: Added for oldstable point releases. + * config/debian/cron.daily: auotmatically accept/reject + oldstable-proposed-updates based on COMMENTS directory + +2007-06-18 Anthony Towns + + * config/debian/apt.conf, config/debian/apt.conf.stable, + config/debian/dak.conf: update for 4.0r0 (etch), and 3.1r6 + (sarge), support for oldstable-proposed-updates, dropping m68k + from etch, creating etch-m68k suite, creating lenny. + + * config/debian/vars: update for lenny + + * config/debian/dak.conf: typo fix for Dinstall::GPGKeyring, + drop upload limitations, add release postgres user + + * dak/process_new.py: support for automatically accepting and rejecting + packages from proposed-updates holding queues via COMMENTS directory + * cron.daily: automatically process COMMENTS-based approvals + and rejections for proposed-updates holding queues + + * dak/process_unchecked.py: add support for oldproposedupdates + holding queue + + * dak/control_suite.py: allow control-suite to work with etch-m68k + + * dak/generate_releases.py: unlink old Release files before updating + them if nlinks > 1 (ie, if two files used to be the same, maybe they + shouldn't be when generate-releases is run) + + * dak/generate_releases.py: add a couple of commented lines to make + it easier to deal with point releases + + * dak/make_overrides.py: generate overrides for !contrib udebs + + * docs/README.stable-point-release: update docs for doing a + point release + 2007-06-18 Anthony Towns * daklib/logging.py: Set umask to not exclude group-writability @@ -411,6 +606,13 @@ * config/debian/cron.unchecked: push version info to debbugs using ssh-move. +2007-03-05 Anthony Towns + + * config/debian/dak.conf: update for 3.1r5. + * scripts/debian/ssh-move: add ssh-move script from debbugs + * config/debian/cron.unchecked: push version info to debbugs using + ssh-move. + 2007-02-14 James Troup * docs/README.config: remove Dinstall::GroupOverrideFilename. @@ -424,6 +626,109 @@ (nmu_p): remove entire class - now unused. (Upload.__init__): don't use nmu_p. +2007-02-14 James Troup + + * docs/README.config: remove Dinstall::GroupOverrideFilename. + * config/debian/dak.conf: likewise. + * config/debian-non-US/dak.conf: likewise. + * config/debian-security/dak.conf: likewise. + + * daklib/queue.py (Upload.close_bugs): no longer handle NMUs or + experimental differently, just close the bugs and let version + tracking sort it out. + (nmu_p): remove entire class - now unused. + (Upload.__init__): don't use nmu_p. + +2007-02-08 Anthony Towns + + * config/debian/dak.conf: update for 3.1r4. Use new 'etch' + signing key. Drop maximum index diffs down to 14. + + * config/debian/apt.conf: add udeb support for non-free (testing, + unstable) and experimental. + * config/debian/dak.conf: likewise. + + * dak/generate_releases.py (main): handle udebs in any component. + + * daklib/queue.py (Upload.build_summaries): handle files without a + 'type' gracefully. + + * dak/generate_releases.py (print_sha256_files): new function. + (main): use it. + + * dak/process_accepted.py (stable_install): fix name of template + mail. + + * dak/process_unchecked.py (is_stableupdate): fix invocation of + database.get_suite_id(). + + * templates/process-new.bxa_notification: Update on request + of/after discussion with BIS staff. + + * scripts/debian/mkfilesindices: also handle proposed-updates. + +2007-02-08 Ryan Murray + + * config/debian/cron.monthly: use $ftpgroup instead of hardcoding + group name for chgrp of mail archives. + + * daklib/queue.py (Upload.check_dsc_against_db): handle multiple + orig.tar.gz's by picking the first one by file id. + + * dak/override.py (main): limit to binary overrides only for now. + (usage): update to match. + + * config/debian/cron.daily: track when we have the accepted lock + and clean it up on exit if we have it. Take/check the + cron.unchecked lock just before traping to cleanup on exit. + Remove potato override handling. Remove any dangling symlinks in + /srv/incoming.d.o/buildd. Clean up apt-ftparchive's databases. + + * config/debian/apt.conf: change default compression scheme for + both Sources and Packages to gzip and bzip2 rather than + uncompressed and gzip (Packages) and gzip (Sources). Use old + defaults for proposed-updates. + + * dak/control_overrides.py (main): refuse to operate on + untouchable suites. + + * config/debian/pseudo-packages.maintainers: drop install, + installation, boot-floppy, slink-cd, potato-cd and + nonus.debian.org. Update base. + * config/debian/pseudo-packages.description: likewise. + + * daklib/utils.py (re_srchasver): new regex. + (parse_changes): use regex to split 'Source (Version)' style + Source fields into 'source' and 'source-version'. + + * config/debian/cron.daily: use $base instead of hardcoding path + name. + + * scripts/debian/mkfilesindices: source 'vars' file and use it's + variables instead of hardcoding path names. + + * config/debian/apt.conf: switch from /org to /srv. + * config/debian/apt.conf.buildd: likewise. + * config/debian/apt.conf.stable: likewise. + * config/debian/cron.daily: likewise. + * config/debian/cron.hourly: likewise. + * config/debian/cron.monthly: likewise. + * config/debian/cron.unchecked: likewise. + * config/debian/cron.weekly: likewise. + * config/debian/dak.conf: likewise. + * config/debian/vars: likewise. + * scripts/debian/mkfilesindices: likewise. + +2007-02-08 James Troup + + * dak/process_unchecked.py (check_signed_by_key): new function to + ensure .changes files are signed by an authorized uploader. + (process_it): use it. + + * config/debian/dak.conf (Binary-Upload-Restrictions): new stanza + to configure per suite/component/architecture binary upload + restrictions. + 2007-02-08 Anthony Towns * config/debian/dak.conf: update for 3.1r4. Use new 'etch' @@ -519,12 +824,28 @@ * dak/process_unchecked.py (check_timestamps): change match to search as recent versions of python-apt prefix the string with 'E: '. +2006-10-09 James Troup + + * dak/process_unchecked.py (check_timestamps): change match to + search as recent versions of python-apt prefix the string with 'E: '. + 2006-06-26 Ryan Murray * dak/process_unchecked.py (check_files): strip optional source version from Source: field in changes file, and ensure what is left is a valid package name. +2006-06-26 Ryan Murray + + * dak/process_unchecked.py (check_files): strip optional source version + from Source: field in changes file, and ensure what is left is a valid + package name. + +2006-06-23 Ryan Murray + + * dak/process_unchecked.py (check_files): also check ProposedUpdates + queue for source. + 2006-06-23 Ryan Murray * dak/process_unchecked.py (check_files): also check ProposedUpdates @@ -541,6 +862,17 @@ * dak/config/debian-security/apt.conf: set Packages::Compress to gzip and bzip2 for etch. +2006-06-18 Ryan Murray + + * dak/scripts/debian/update-ftpstats: look for dak named processes in + the log, too. + + * dak/process_unchecked.py (check_files): only check embargoed and + unembargoed queues if the keys are set. + + * dak/config/debian-security/apt.conf: set Packages::Compress to gzip + and bzip2 for etch. + 2006-06-16 James Troup * dak/dak.py (init): add new-security-install. diff --git a/config/debian-security/apt.conf b/config/debian-security/apt.conf index 41b10ef1..7d34e11d 100644 --- a/config/debian-security/apt.conf +++ b/config/debian-security/apt.conf @@ -46,7 +46,7 @@ tree "dists/testing/updates" FileList "/org/security.debian.org/dak-database/dists/testing_updates/$(SECTION)_binary-$(ARCH).list"; SourceFileList "/org/security.debian.org/dak-database/dists/testing_updates/$(SECTION)_source.list"; Sections "main contrib non-free"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source"; + Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source"; BinOverride "override.lenny.$(SECTION)"; ExtraOverride "override.lenny.extra.$(SECTION)"; SrcOverride "override.lenny.$(SECTION).src"; diff --git a/config/debian-security/cron.buildd b/config/debian-security/cron.buildd index 96607e48..7e75bcbb 100755 --- a/config/debian-security/cron.buildd +++ b/config/debian-security/cron.buildd @@ -4,7 +4,7 @@ ARCHS_oldstable="alpha arm hppa i386 ia64 m68k mips mipsel powerpc sparc s390 amd64" ARCHS_stable="alpha amd64 arm hppa i386 ia64 mips mipsel powerpc sparc s390" -ARCHS_testing="$ARCHS_stable" +ARCHS_testing="alpha amd64 armel hppa i386 ia64 mips mipsel powerpc sparc s390" DISTS="oldstable stable testing" SSH_SOCKET=~/.ssh/buildd.debian.org.socket diff --git a/config/debian-security/cron.daily b/config/debian-security/cron.daily old mode 100644 new mode 100755 index dbc34b6a..d8d2bd1b --- a/config/debian-security/cron.daily +++ b/config/debian-security/cron.daily @@ -11,7 +11,7 @@ export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars # Fix overrides # disabled by ajt 2008-01-01: requires auth -#rsync -ql ftp-master::indices/override\* $overridedir +rsync --password-file /srv/non-us.debian.org/s3kr1t/rsync-password -ql security-master@ftp-master::indices/override\* $overridedir cd $overridedir find . -name override\*.gz -type f -maxdepth 1 -mindepth 1 | xargs gunzip -f diff --git a/config/debian-security/dak.conf b/config/debian-security/dak.conf index fb219e5e..bc978ee2 100644 --- a/config/debian-security/dak.conf +++ b/config/debian-security/dak.conf @@ -196,6 +196,7 @@ Suite amd64; alpha; arm; + armel; hppa; i386; ia64; @@ -251,6 +252,7 @@ Dir Reject "/org/security.debian.org/queue/reject/"; Unchecked "/org/security.debian.org/queue/unchecked/"; ProposedUpdates "/does/not/exist/"; // XXX fixme + OldProposedUpdates "/does/not/exist/"; // XXX fixme Embargoed "/org/security.debian.org/queue/embargoed/"; Unembargoed "/org/security.debian.org/queue/unembargoed/"; @@ -274,6 +276,7 @@ Architectures alpha "DEC Alpha"; hppa "HP PA RISC"; arm "ARM"; + armel "ARM EABI"; i386 "Intel ia32"; ia64 "Intel ia64"; m68k "Motorola Mc680x0"; diff --git a/config/debian/cron.daily b/config/debian/cron.daily index 8d9ecd5a..b3d43502 100755 --- a/config/debian/cron.daily +++ b/config/debian/cron.daily @@ -180,6 +180,8 @@ cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $scriptsdir/dm-monitor >$webdir/dm-uploaders.html +$scriptsdir/dm-monitor >$webdir/dm-uploaders.html + ################################################################################ # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 97fb6210..4181fb01 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -68,6 +68,31 @@ Binary-Upload-Restrictions }; }; +Binary-Upload-Restrictions +{ + Components + { + //main; + //contrib; + // Yay for consensus through GRs voted on by people not actually involved in the affected architectures + none; + }; + unstable + { + arm + { + 9BF093BC475BABF8B6AEA5F6D7C3F131AB2A91F5; + 70BC7F9D8C60D2265B7076A23760DBCFFD6645AB; + F849E2025D1C194DE62BC6C829BE5D2268FD549F; + }; + alpha + { + 9BF093BC475BABF8B6AEA5F6D7C3F131AB2A91F5; + 70BC7F9D8C60D2265B7076A23760DBCFFD6645AB; + }; + }; +}; + Generate-Index-Diffs { Options diff --git a/dak/dak.py b/dak/dak.py index 0eeb9d7b..0f228010 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -53,6 +53,26 @@ class UserExtension: ################################################################################ +class UserExtension: + def __init__(self, user_extension = None): + if user_extension: + m = imp.load_source("dak_userext", user_extension) + d = m.__dict__ + else: + m, d = None, {} + self.__dict__["_module"] = m + self.__dict__["_d"] = d + + def __getattr__(self, a): + if a in self.__dict__: return self.__dict__[a] + if a[0] == "_": raise AttributeError, a + return self._d.get(a, None) + + def __setattr__(self, a, v): + self._d[a] = v + +################################################################################ + def init(): """Setup the list of modules and brief explanation of what they do.""" diff --git a/dak/process_new.py b/dak/process_new.py index 54c2b487..085e0913 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -83,7 +83,11 @@ def recheck(): and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()): source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) - if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): + found = 0 + for q in ["Accepted", "Embargoed", "Unembargoed"]: + if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): + found = 1 + if not found: reject("no source found for %s %s (%s)." % (source_package, source_version, file)) # Version and file overwrite checks @@ -777,26 +781,42 @@ def do_byhand(): ################################################################################ +def get_accept_lock(): + retry = 0 + while retry < 10: + try: + lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) + retry = 10 + except OSError, e: + if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': + retry += 1 + if (retry >= 10): + daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.") + else: + print("Unable to get accepted lock (try %d of 10)" % retry) + time.sleep(60) + else: + raise + +def move_to_dir (dest, perms=0660, changesperms=0664): + daklib.utils.move (Upload.pkg.changes_file, dest, perms=changesperms) + file_keys = Upload.pkg.files.keys() + for file in file_keys: + daklib.utils.move (file, dest, perms=perms) + def do_accept(): print "ACCEPT" if not Options["No-Action"]: - retry = 0 - while retry < 10: - try: - lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) - retry = 10 - except OSError, e: - if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': - retry += 1 - if (retry >= 10): - daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.") - else: - print("Unable to get accepted lock (try %d of 10)" % retry) - time.sleep(60) - else: - raise + get_accept_lock() (summary, short_summary) = Upload.build_summaries() - Upload.accept(summary, short_summary) + if Cnf.FindB("Dinstall::SecurityQueueHandling"): + Upload.dump_vars(Cnf["Dir::Queue::Embargoed"]) + move_to_dir(Cnf["Dir::Queue::Embargoed"]) + Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) + # Check for override disparities + Upload.Subst["__SUMMARY__"] = summary + else: + Upload.accept(summary, short_summary) os.unlink(Upload.pkg.changes_file[:-8]+".dak") os.unlink(Cnf["Process-New::AcceptedLockFile"]) diff --git a/daklib/utils.py b/daklib/utils.py index 0d22bd1d..94b7cc53 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -46,6 +46,8 @@ re_gpg_uid = re.compile('^uid.*<([^>]*)>') re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$") re_verwithext = re.compile(r"^(\d+)(?:\.(\d+))(?:\s+\((\S+)\))?$") +re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$") + changes_parse_error_exc = "Can't parse line in .changes file" invalid_dsc_format_exc = "Invalid .dsc file" nk_format_exc = "Unknown Format: in .changes file" diff --git a/setup/init_pool.sql b/setup/init_pool.sql index 9925148c..7a6e2a49 100644 --- a/setup/init_pool.sql +++ b/setup/init_pool.sql @@ -28,6 +28,12 @@ CREATE TABLE maintainer ( name TEXT UNIQUE NOT NULL ); +CREATE TABLE src_uploaders ( + id SERIAL PRIMARY KEY, + source INT4 NOT NULL REFERENCES source, + maintainer INT4 NOT NULL REFERENCES maintainer +); + CREATE TABLE uid ( id SERIAL PRIMARY KEY, uid TEXT UNIQUE NOT NULL,