From: Mike O'Connor Date: Wed, 18 Feb 2009 13:56:09 +0000 (-0500) Subject: merge from master X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=9f371390bb90395b2f2f0b65c91c21047896774a;hp=-c;p=dak.git merge from master Signed-off-by: Mike O'Connor --- 9f371390bb90395b2f2f0b65c91c21047896774a diff --combined config/debian/dak.conf index 3e73e4ba,dae2c9d9..e5a82864 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@@ -1,16 -1,14 +1,16 @@@ Dinstall { GPGKeyring { - "/srv/keyring.debian.org/keyrings/debian-keyring.gpg"; - "/srv/keyring.debian.org/keyrings/debian-keyring.pgp"; - "/srv/ftp.debian.org/keyrings/debian-maintainers.gpg"; + "/usr/share/keyrings/debian-keyring.gpg"; + "/usr/share/keyrings/debian-keyring.pgp"; + "/usr/share/keyrings/debian-keyring.pgp"; +// "/srv/ftp.debian.org/keyrings/debian-maintainers.gpg"; }; SigningKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg"; SigningPubKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; SigningKeyIds "6070D3A1"; - SendmailCommand "/usr/sbin/sendmail -odq -oi -t"; +// SendmailCommand "/usr/sbin/sendmail -odq -oi -t"; + SendmailCommand "/bin/echo -odq -oi -t"; MyEmailAddress "Debian Installer "; MyAdminAddress "ftpmaster@debian.org"; MyHost "debian.org"; // used for generating user@my_host addresses in e.g. manual_reject() @@@ -37,6 -35,14 +37,14 @@@ NoSourceOnly "true"; ReleaseTransitions "/srv/ftp.debian.org/web/transitions.yaml"; }; + // if you setup an own dak repository and want to upload Debian packages you most possibly want + // to set the following option to a real path/filename and then enter those mail addresses that + // you want to be able to receive mails generated by your dak installation. This avoids spamming + // the real maintainers of a package you upload with mail. + // format of entries: one entry per line. Either an email address directly, or a regular expression, + // prefixed by "RE:". Examples: "jane.doe@domain.com" or "RE:jane[^@]@domain.com", where the first will + // only allow to mail jane.doe@domain.com while the second will mail all of jane*@domain.com + // MailWhiteList "/some/path/to/a/file"; }; Transitions @@@ -140,7 -146,7 +148,7 @@@ Reject-Proposed-Update { StableRejector "the Stable Release Team"; StableMail "debian-release@lists.debian.org"; - MoreInfoURL "http://release.debian.org/stable/4.0/4.0r6/"; + MoreInfoURL "http://release.debian.org/stable/4.0/4.0r7/"; }; Import-LDAP-Fingerprints @@@ -193,7 -199,7 +201,7 @@@ Check-Override Suite { - Stable + OldStable { Components { @@@ -225,6 -231,107 +233,107 @@@ OverrideCodeName "etch"; Priority "5"; Untouchable "1"; + ChangeLogBase "dists/oldstable/"; + UdebComponents + { + main; + non-free; + }; + }; + + Oldstable-Proposed-Updates + { + Components + { + main; + contrib; + non-free; + }; + Architectures + { + source; + all; + alpha; + amd64; + arm; + hppa; + i386; + ia64; + mips; + mipsel; + powerpc; + s390; + sparc; + }; + Announce "debian-changes@lists.debian.org"; + CopyChanges "dists/oldstable-proposed-updates/"; + CopyDotDak "/srv/ftp.debian.org/queue/oldstable-proposed-updates/"; + CommentsDir "/srv/ftp.debian.org/queue/o-p-u-new/COMMENTS/"; + Version "4.0-updates"; + Origin "Debian"; + Description "Debian 4.0 Proposed Updates - Not Released"; + CodeName "etch-proposed-updates"; + OverrideCodeName "etch"; + OverrideSuite "oldstable"; + ValidTime 604800; // 7 days + Priority "4"; + VersionChecks + { + MustBeNewerThan + { + OldStable; + }; + MustBeOlderThan + { + Stable; + Testing; + Unstable; + Experimental; + }; + Enhances + { + OldStable; + }; + }; + UdebComponents + { + main; + }; + }; + + Stable + { + Components + { + main; + contrib; + non-free; + }; + Architectures + { + source; + all; + alpha; + amd64; + arm; + armel; + hppa; + i386; + ia64; + mips; + mipsel; + powerpc; + s390; + sparc; + }; + Announce "debian-changes@lists.debian.org"; + // Version "5.0r0"; + Origin "Debian"; + // Description "Debian 5.0r0 Released 14 February 2009"; + CodeName "lenny"; + OverrideCodeName "lenny"; + Priority "5"; + Untouchable "1"; ChangeLogBase "dists/stable/"; UdebComponents { @@@ -248,6 -355,7 +357,7 @@@ alpha; amd64; arm; + armel; hppa; i386; ia64; @@@ -261,11 -369,11 +371,11 @@@ CopyChanges "dists/proposed-updates/"; CopyDotDak "/srv/ftp.debian.org/queue/proposed-updates/"; CommentsDir "/srv/ftp.debian.org/queue/p-u-new/COMMENTS/"; - Version "4.0-updates"; + Version "5.0-updates"; Origin "Debian"; - Description "Debian 4.0 Proposed Updates - Not Released"; - CodeName "etch-proposed-updates"; - OverrideCodeName "etch"; + Description "Debian 5.0 Proposed Updates - Not Released"; + CodeName "lenny-proposed-updates"; + OverrideCodeName "lenny"; OverrideSuite "stable"; ValidTime 604800; // 7 days Priority "4"; @@@ -289,6 -397,7 +399,7 @@@ UdebComponents { main; + non-free; }; }; @@@ -306,7 -415,6 +417,6 @@@ all; alpha; amd64; - arm; armel; hppa; i386; @@@ -320,8 -428,8 +430,8 @@@ Announce "debian-testing-changes@lists.debian.org"; Origin "Debian"; Description "Debian Testing distribution - Not Released"; - CodeName "lenny"; - OverrideCodeName "lenny"; + CodeName "squeeze"; + OverrideCodeName "squeeze"; ValidTime 604800; // 7 days Priority "5"; UdebComponents @@@ -345,7 -453,6 +455,6 @@@ all; alpha; amd64; - arm; armel; hppa; i386; @@@ -359,8 -466,8 +468,8 @@@ Announce "debian-testing-changes@lists.debian.org"; Origin "Debian"; Description "Debian Testing distribution updates - Not Released"; - CodeName "lenny-proposed-updates"; - OverrideCodeName "lenny"; + CodeName "squeeze-proposed-updates"; + OverrideCodeName "squeeze"; OverrideSuite "testing"; ValidTime 604800; // 7 days Priority "6"; @@@ -430,7 -537,6 +539,6 @@@ all; alpha; amd64; - arm; armel; hppa; hurd-i386; @@@ -480,7 -586,6 +588,6 @@@ all; alpha; amd64; - arm; armel; hppa; hurd-i386; @@@ -523,11 -628,11 +630,11 @@@ SuiteMappings { - // "propup-version oldstable-security stable testing testing-proposed-updates unstable"; + "propup-version oldstable-security stable testing testing-proposed-updates unstable"; "propup-version stable-security testing testing-proposed-updates unstable"; "propup-version testing-security unstable"; - // "map oldstable oldstable-proposed-updates"; - // "map oldstable-security oldstable-proposed-updates"; + "map oldstable oldstable-proposed-updates"; + "map oldstable-security oldstable-proposed-updates"; "map stable proposed-updates"; "map stable-security proposed-updates"; "map stable-proposed-updates proposed-updates"; @@@ -750,8 -855,3 +857,8 @@@ Urgenc critical; }; }; + +Content +{ + Header "contents" +} diff --combined dak/dak.py index aa6efdee,981a31a9..15c3e1ad --- a/dak/dak.py +++ b/dak/dak.py @@@ -112,8 -112,6 +112,8 @@@ def init() "Generate package <-> file mapping"), ("generate-releases", "Generate Release files"), + ("contents", + "Generate contest files"), ("generate-index-diffs", "Generate .diff/Index files"), ("clean-suites", @@@ -171,10 -169,10 +171,12 @@@ "Split queue/done into a date-based hierarchy"), ("stats", "Generate statistics"), + ("calculate-shasums", + "Calculate missing sha1sums and sha256sums"), ("bts-categorize", "Categorize uncategorized bugs filed against ftp.debian.org"), + ("add-user", + "Add a user to the archive"), ] return functionality diff --combined dak/dakdb/update2.py index e411662c,850e3ab5..10407a9c --- a/dak/dakdb/update2.py +++ b/dak/dakdb/update2.py @@@ -21,18 -21,18 +21,20 @@@ ################################################################################ - import psycopg2, time + import psycopg2 + import time + from daklib.dak_exceptions import DBUpdateError ################################################################################ def do_update(self): +vvvvvvvvvvvvvvvvvvvv print "Note: to be able to enable the the PL/Perl (plperl) procedural language, we do" print "need postgresql-plperl-$postgres-version installed. Make sure that this is the" print "case before you continue. Interrupt if it isn't, sleeping 5 seconds now." print "(We need to be database superuser for this to work!)" time.sleep (5) +^^^^^^^^^^^^^^^^^^^^ try: c = self.db.cursor() @@@ -393,6 -393,4 +395,4 @@@ $ except psycopg2.ProgrammingError, msg: self.db.rollback() - print "FATAL: Unable to apply debversion table update 2!" - print "Error Message: " + str(msg) - print "Database changes have been rolled back." + raise DBUpdateError, "Unable to appy debversion updates, rollback issued. Error message : %s" % (str(msg)) diff --combined dak/update_db.py index 8bb88f65,5d5fef47..f7d13750 --- a/dak/update_db.py +++ b/dak/update_db.py @@@ -37,7 -37,7 +37,7 @@@ from daklib import util Cnf = None projectB = None -required_database_schema = 3 +required_database_schema = 4 ################################################################################ @@@ -137,9 -137,15 +137,15 @@@ Updates dak's database schema to the la for i in range (database_revision, required_database_schema): print "updating databse schema from " + str(database_revision) + " to " + str(i+1) - dakdb = __import__("dakdb", globals(), locals(), ['update'+str(i+1)]) - update_module = getattr(dakdb, "update"+str(i+1)) - update_module.do_update(self) + try: + dakdb = __import__("dakdb", globals(), locals(), ['update'+str(i+1)]) + update_module = getattr(dakdb, "update"+str(i+1)) + update_module.do_update(self) + except DBUpdateError, e: + # Seems the update did not work. + print "Was unable to update database schema from %s to %s." % (str(database_revision), str(i+1)) + print "The error message received was %s" % (e) + utils.fubar("DB Schema upgrade failed") database_revision += 1 ################################################################################ diff --combined daklib/utils.py index 5cb502d6,cb5df31c..1b35feef --- a/daklib/utils.py +++ b/daklib/utils.py @@@ -37,11 -37,13 +37,14 @@@ import sta import apt_pkg import database import time +import tarfile + import re + import string + import email as modemail from dak_exceptions import * from regexes import re_html_escaping, html_escaping, re_single_line_field, \ re_multi_line_field, re_srchasver, re_verwithext, \ - re_parse_maintainer, re_taint_free, re_gpg_uid + re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark ################################################################################ @@@ -599,6 -601,67 +602,67 @@@ def send_mail (message, filename="") os.write (fd, message) os.close (fd) + if Cnf.has_key("Dinstall::MailWhiteList") and \ + Cnf["Dinstall::MailWhiteList"] != "": + message_in = open_file(filename) + message_raw = modemail.message_from_file(message_in) + message_in.close(); + + whitelist = []; + whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"]) + try: + for line in whitelist_in: + if re_re_mark.match(line): + whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1))) + else: + whitelist.append(re.compile(re.escape(line.strip()))) + finally: + whitelist_in.close() + + # Fields to check. + fields = ["To", "Bcc", "Cc"] + for field in fields: + # Check each field + value = message_raw.get(field, None) + if value != None: + match = []; + for item in value.split(","): + (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip()) + mail_whitelisted = 0 + for wr in whitelist: + if wr.match(email): + mail_whitelisted = 1 + break + if not mail_whitelisted: + print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"]) + continue + match.append(item) + + # Doesn't have any mail in whitelist so remove the header + if len(match) == 0: + del message_raw[field] + else: + message_raw.replace_header(field, string.join(match, ", ")) + + # Change message fields in order if we don't have a To header + if not message_raw.has_key("To"): + fields.reverse() + for field in fields: + if message_raw.has_key(field): + message_raw[fields[-1]] = message_raw[field] + del message_raw[field] + break + else: + # Clean up any temporary files + # and return, as we removed all recipients. + if message: + os.unlink (filename); + return; + + fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0700); + os.write (fd, message_raw.as_string(True)); + os.close (fd); + # Invoke sendmail (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename)) if (result != 0): @@@ -1437,53 -1500,3 +1501,53 @@@ if which_conf_file() != default_config apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) ################################################################################ + +def generate_contents_information(filename): + """ + Generate a list of flies contained in a .deb + + @type filename: string + @param filename: the path to a data.tar.gz or data.tar.bz2 + + @rtype: list + @return: a list of files in the data.tar.* portion of the .deb + """ + cmd = "ar t %s" % (filename) + (result, output) = commands.getstatusoutput(cmd) + if result != 0: + reject("%s: 'ar t' invocation failed." % (filename)) + reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") + + # Ugh ... this is ugly ... Code ripped from process_unchecked.py + chunks = output.split('\n') + + contents = [] + try: + cmd = "ar x %s %s" % (filename, chunks[2]) + (result, output) = commands.getstatusoutput(cmd) + if result != 0: + reject("%s: '%s' invocation failed." % (filename, cmd)) + reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") + + # Got deb tarballs, now lets go through and determine what bits + # and pieces the deb had ... + if chunks[2] == "data.tar.gz": + data = tarfile.open("data.tar.gz", "r:gz") + elif chunks[2] == "data.tar.bz2": + data = tarfile.open("data.tar.bz2", "r:bz2") + else: + os.remove(chunks[2]) + reject("couldn't find data.tar.*") + + for tarinfo in data: + if not tarinfo.isdir(): + contents.append(tarinfo.name[2:]) + + finally: + if os.path.exists( chunks[2] ): + shutil.rmtree( chunks[2] ) + os.remove( chunks[2] ) + + return contents + +###############################################################################