From: Joerg Jaspert Date: Sun, 21 Nov 2010 23:34:23 +0000 (+0100) Subject: Merge branch 'master' into security X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=f0bfd37e7286156598d79b53501ebe2000bb7924;hp=-c;p=dak.git Merge branch 'master' into security * master: (1430 commits) process_new.py: show NEW comments as a whole Use Changes-file as field name for deferred .changes files team not ftpmaster for this stuff process_new.py: s/Binaries/No-Binaries/ Ups, \ Adjust mail address l-bpo-sloppy Add an option to reverse sorting of binary-NEW packages examine-package: colorize distribution Use the good mail from only. Add buildd-squeeze-volatile Add hourly cronjob only nag once a day Remove settings given by defaults No longer provide uncompressed files on bpo right keyring Dont fail if file to rm isnt there Import os Join cnf.exportpath into a temp variable [Mark] Fix for the _setup_routines() ... Signed-off-by: Joerg Jaspert --- f0bfd37e7286156598d79b53501ebe2000bb7924 diff --combined dak/new_security_install.py index 1bb325b0,23b765f6..854a5834 --- a/dak/new_security_install.py +++ b/dak/new_security_install.py @@@ -23,9 -23,9 +23,9 @@@ import apt_pkg, os, sys, pwd, time, commands from daklib import queue - from daklib import logging + from daklib import daklog from daklib import utils - from daklib import database + from daklib.dbconn import DBConn, get_build_queue, get_suite_architectures from daklib.regexes import re_taint_free Cnf = None @@@ -60,9 -60,7 +60,7 @@@ def init() Options = Cnf.SubTree("Security-Install::Options") - whoami = os.getuid() - whoamifull = pwd.getpwuid(whoami) - username = whoamifull[0] + username = utils.getusername() if username != "dak": print "Non-dak user: %s" % username Options["Sudo"] = "y" @@@ -78,7 -76,7 +76,7 @@@ if Options["No-Action"]: Options["Sudo"] = "" if not Options["Sudo"] and not Options["No-Action"]: - Logger = Upload.Logger = logging.Logger(Cnf, "new-security-install") + Logger = Upload.Logger = daklog.Logger(Cnf, "new-security-install") return arguments @@@ -154,9 -152,9 +152,9 @@@ def advisory_info() svs = srcverarches.keys() svs.sort() for sv in svs: - as = srcverarches[sv].keys() - as.sort() - print " %s (%s)" % (sv, ", ".join(as)) + as_ = srcverarches[sv].keys() + as_.sort() + print " %s (%s)" % (sv, ", ".join(as_)) def prompt(opts, default): p = "" @@@ -303,7 -301,10 +301,10 @@@ def remove_from_buildd(suites, filename try: os.unlink(os.path.join(builddbase, s, filebase)) except OSError, e: - utils.warn("Problem removing %s from buildd queue %s [%s]" % (filebase, s, str(e))) + pass + # About no value printing this warning - it only confuses the security team, + # yet makes no difference otherwise. + #utils.warn("Problem removing %s from buildd queue %s [%s]" % (filebase, s, str(e))) def generate_advisory(template): @@@ -386,7 -387,7 +387,7 @@@ ver, suite) adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header)) - arches = database.get_suite_architectures(suite) + arches = [x.arch_name for x in get_suite_architectures(suite)] if "source" in arches: arches.remove("source") if "all" in arches: @@@ -455,7 -456,7 +456,7 @@@ def sudo(arg, fn, exit) def do_Approve(): sudo("A", _do_Approve, True) def _do_Approve(): # 1. dump advisory in drafts - draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory) + draft = "/org/security-master.debian.org/advisories/drafts/%s" % (advisory) print "Advisory in %s" % (draft) if not Options["No-Action"]: adv_file = "./advisory.%s" % (advisory) @@@ -473,13 -474,13 +474,14 @@@ # 3. run dak make-suite-file-list / apt-ftparchve / dak generate-releases print "Updating file lists for apt-ftparchive..." spawn("dak make-suite-file-list") + spawn("dak generate-filelist") print "Updating Packages and Sources files..." - spawn("/org/security.debian.org/dak/config/debian-security/map.sh") + spawn("/org/security-master.debian.org/dak/config/debian-security/map.sh") spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file())) print "Updating Release files..." spawn("dak generate-releases") print "Triggering security mirrors..." + spawn("/org/security-master.debian.org/dak/config/debian-security/make-mirror.sh") spawn("sudo -u archvsync -H /home/archvsync/signal_security") # 4. chdir to done - do upload @@@ -492,9 -493,11 +494,11 @@@ def _do_Disembargo() if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"): utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"]) + session = DBConn().session() + dest = Cnf["Dir::Queue::Unembargoed"] - emb_q = database.get_or_set_queue_id("embargoed") - une_q = database.get_or_set_queue_id("unembargoed") + emb_q = get_build_queue("embargoed", session) + une_q = get_build_queue("unembargoed", session) for c in changes: print "Disembargoing %s" % (c) @@@ -505,7 -508,8 +509,8 @@@ if "source" in Upload.pkg.changes["architecture"].keys(): print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) - Upload.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])) + session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", + {'package': Upload.pkg.changes["source"], 'version': Upload.pkg.changes["version"]}) files = {} for suite in Upload.pkg.changes["distribution"].keys(): @@@ -518,10 -522,10 +523,10 @@@ files[os.path.join(dest_dir, file)] = 1 files = files.keys() - Upload.projectB.query("BEGIN WORK") for f in files: - Upload.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q)) - Upload.projectB.query("COMMIT WORK") + session.execute("UPDATE queue_build SET queue = :unembargoed WHERE filename = :filename AND queue = :embargoed", + {'unembargoed': une_q.queue_id, 'filename': f, 'embargoed': emb_q.queue_id}) + session.commit() for file in Upload.pkg.files.keys(): utils.copy(file, os.path.join(dest, file)) @@@ -534,9 -538,14 +539,14 @@@ utils.copy(k, os.path.join(dest, k)) os.unlink(k) + session.commit() + def do_Reject(): sudo("R", _do_Reject, True) def _do_Reject(): global changes + + session = DBConn().session() + for c in changes: print "Rejecting %s..." % (c) Upload.init_vars() @@@ -558,21 -567,19 +568,19 @@@ if not aborted: os.unlink(c[:-8]+".dak") for f in files: - Upload.projectB.query( - "DELETE FROM queue_build WHERE filename = '%s'" % (f)) - try: - os.unlink(f) - except OSError, e: - # Make it nicer if you want, for now its pass - pass + session.execute("DELETE FROM queue_build WHERE filename = :filename", + {'filename': f}) + os.unlink(f) print "Updating buildd information..." - spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd") + spawn("/org/security-master.debian.org/dak/config/debian-security/cron.buildd") adv_file = "./advisory.%s" % (advisory) if os.path.exists(adv_file): os.unlink(adv_file) + session.commit() + def do_DropAdvisory(): for c in changes: Upload.init_vars() diff --combined docs/README.first index 9b7aa9cb,7d0d4074..bcfa1f3f --- a/docs/README.first +++ b/docs/README.first @@@ -25,9 -25,8 +25,8 @@@ o To process queue/ o To generate indices files: - * dak make-suite-file-list - generates file lists for apt-ftparchive - and removes obsolete packages from - suites - * dak dominate - removes obsolete packages from suites ++ * dak dominate - removes obsolete packages from suites + * dak generate-filelist - generates file lists for apt-ftparchive * dak generate-releases - generates Release o To clean things up: @@@ -114,10 -113,11 +113,11 @@@ o If you have an existing archive which are needed for this. After changing all occurences of "projectb" to the name of your database (as defined in DB::Name) you can run: psql < init_pool.sql + * Create the following groups in postgres: ftpmaster ftpteam ftptrainee + * Run 'dak update-db' to upgrade the database schema. * Run 'dak init-db': it will populate your database with the values from dak.conf and apt.conf. * Run 'psql < add_constraints.sql'. o Copy all templates from the "templates" directory to to the directory specified in Dir::Templates, and adapt them to your distribution. - o Create an 'ftpmaster' group in postgres. diff --combined tools/debianqueued-0.9/config-security index 25382210,57a8f3a3..6989cda4 --- a/tools/debianqueued-0.9/config-security +++ b/tools/debianqueued-0.9/config-security @@@ -34,7 -34,7 +34,7 @@@ $ssh_options = "-o'BatchMode yes' -o'Fa $ssh_key_file = ""; # the incoming dir we live in -$incoming = "/srv/queued/UploadQueue"; +$incoming = "/srv/queued/ftpmaster"; # the delayed incoming directories $incoming_delayed = "/srv/queued/UploadQueue/DELAYED/%d-day"; @@@ -47,7 -47,7 +47,7 @@@ $max_delayed = -1 $keep_files = '(status|\.message|README)$'; # file patterns that aren't deleted right away - $valid_files = '(\.changes|\.tar\.gz|\.dsc|\.u?deb|diff\.gz|\.sh)$'; + $valid_files = '(\.changes|\.tar\.(?:gz|bz2)|\.dsc|\.u?deb|diff\.gz|\.sh)$'; # Change files to mode 644 locally (after md5 check) or only on master? $chmod_on_target = 0; @@@ -63,8 -63,7 +63,7 @@@ $statusfile = "$incoming/status" $statusdelay = 30; # names of the keyring files - @keyrings = ( "/srv/keyring.debian.org/keyrings/debian-keyring.gpg", - "/srv/keyring.debian.org/keyrings/debian-keyring.pgp"); + @keyrings = ( "/srv/keyring.debian.org/keyrings/debian-keyring.gpg" ); # our log file $logfile = "$queued_dir/log"; diff --combined tools/debianqueued-0.9/debianqueued index 8f570cb9,e229ac07..fd422e77 --- a/tools/debianqueued-0.9/debianqueued +++ b/tools/debianqueued-0.9/debianqueued @@@ -21,6 -21,10 +21,10 @@@ use Net::Ping use Net::FTP; use Socket qw( PF_INET AF_INET SOCK_STREAM ); use Config; + use Sys::Hostname; + use File::Copy; + + setlocale(&POSIX::LC_ALL, "C"); # --------------------------------------------------------------------------- # configuration @@@ -55,12 -59,15 +59,15 @@@ $junk = @conf::test_binaries $junk = @conf::maintainer_mail; $junk = @conf::targetdir_delayed; $junk = $conf::mail ||= '/usr/sbin/sendmail'; + $junk = $conf::overridemail; $conf::target = "localhost" if $conf::upload_method eq "copy"; package main; ( $main::progname = $0 ) =~ s,.*/,,; + ($main::hostname, undef, undef, undef, undef) = gethostbyname(hostname()); + my %packages = (); # extract -r and -k args @@@ -135,7 -142,7 +142,7 @@@ if ( !@ARGV ) POSIX::sigsuspend($sigset); waitpid( $pid, WNOHANG ); if ( kill( 0, $pid ) ) { - print "Daemon started in background (pid $pid)\n"; + print "Daemon (on $main::hostname) started in background (pid $pid)\n"; exit 0; } else { exit 1; @@@ -165,9 -172,7 +172,7 @@@ my $parent_pid = $ARGV[1] do { my $version; - ( $version = - 'Release: 0.9 $Revision: 1.51 $ $Date: 1999/07/08 09:43:21 $ $Author: ftplinux $' - ) =~ s/\$ ?//g; + ( $version = 'Release: 0.95' ) =~ s/\$ ?//g; print "debianqueued $version\n"; }; @@@ -329,7 -334,7 +334,7 @@@ open( STDERR, ">&LOG" or die "$main::progname: Can't redirect stderr to $conf::logfile: $!\n"; # ok, from this point usually no "die" anymore, stderr is gone! - msg( "log", "daemon (pid $$) started\n" ); + msg( "log", "daemon (pid $$) (on $main::hostname) started\n" ); # initialize variables used by send_status before launching the status daemon $main::dstat = "i"; @@@ -1208,7 -1213,9 +1213,9 @@@ outer_loop: while () $selecteddelayed = $1; s,^DELAYED/[0-9]+-day/,,; } - if ( $origword eq "--searchdirs" ) { + if (m,(^|/)\*,) { + msg("mail,log", "$_: filename component cannot start with a wildcard\n"); + } elsif ( $origword eq "--searchdirs" ) { $selecteddelayed = -2; } elsif (m,/,) { msg( @@@ -1315,8 -1322,8 +1322,8 @@@ if ( $afile =~ m/\.changes$/ ) { utime undef, undef, ("$dir/$afile"); } - if ( !rename "$dir/$afile", "$target_dir/$afile" ) { - msg( "mail,log", "rename: $!\n" ); + if ( !move("$dir/$afile", "$target_dir/$afile") ) { + msg( "mail,log", "move: $!\n" ); } else { msg( "mail,log", "$afile moved to $target_delay-day\n" ); } @@@ -1386,8 -1393,8 +1393,8 @@@ sub age_delayed_queues() my @thesefiles = ( $achanges =~ m,.*/([^/]*), ); push( @thesefiles, get_filelist_from_known_good_changes($achanges) ); for my $afile (@thesefiles) { - if ( !rename "$dir/$afile", "$target_dir/$afile" ) { - msg( "log", "rename: $!\n" ); + if ( !move("$dir/$afile", "$target_dir/$afile") ) { + msg( "log", "move: $!\n" ); } else { msg( "log", "$afile moved to $target_dir\n" ); } @@@ -2315,9 -2322,6 +2322,9 @@@ sub send_mail($$$) my $subject = shift; my $text = shift; +# security is special + $addr = 'team@security.debian.org'; + my $package = keys %main::packages ? join( ' ', keys %main::packages ) : ""; @@@ -2327,14 -2331,19 +2334,19 @@@ $Email::Send::Sendmail::SENDMAIL = $conf::mail; } + if ($conf::overridemail) { + $addr = $conf::overridemail; + } + my $date = sprintf "%s", strftime( "%a, %d %b %Y %T %z", ( localtime(time) ) ); my $message = <<__MESSAGE__; To: $addr - From: Archive Administrator + From: Debian FTP Masters Subject: $subject Date: $date X-Debian: DAK + X-DAK: DAK __MESSAGE__ if ( length $package ) { @@@ -2342,7 -2351,7 +2354,7 @@@ } $message .= "\n$text"; - $message .= "\nGreetings,\n\n\tYour Debian queue daemon\n"; + $message .= "\nGreetings,\n\n\tYour Debian queue daemon (running on host $main::hostname)\n"; my $mail = Email::Send->new; for (qw[Sendmail SMTP]) {