]> git.decadent.org.uk Git - dak.git/commitdiff
merge from master
authorMike O'Connor <stew@vireo.org>
Wed, 18 Feb 2009 13:56:09 +0000 (08:56 -0500)
committerMike O'Connor <stew@vireo.org>
Wed, 18 Feb 2009 13:56:09 +0000 (08:56 -0500)
Signed-off-by: Mike O'Connor <stew@vireo.org>
1  2 
config/debian/dak.conf
dak/dak.py
dak/dakdb/update2.py
dak/update_db.py
daklib/utils.py

diff --combined config/debian/dak.conf
index 3e73e4bac6a0d57848bb4dd908018495b7eec7f2,dae2c9d987bba732c21e265b56d40cac9c89e5a8..e5a82864f49a4fe6203a5e32f83310192638d603
@@@ -1,16 -1,14 +1,16 @@@
  Dinstall
  {
     GPGKeyring {
 -      "/srv/keyring.debian.org/keyrings/debian-keyring.gpg"; 
 -      "/srv/keyring.debian.org/keyrings/debian-keyring.pgp";
 -      "/srv/ftp.debian.org/keyrings/debian-maintainers.gpg";
 +      "/usr/share/keyrings/debian-keyring.gpg";
 +      "/usr/share/keyrings/debian-keyring.pgp";
 +      "/usr/share/keyrings/debian-keyring.pgp";
 +//      "/srv/ftp.debian.org/keyrings/debian-maintainers.gpg";
     };
     SigningKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg";
     SigningPubKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg";
     SigningKeyIds "6070D3A1";
 -   SendmailCommand "/usr/sbin/sendmail -odq -oi -t";
 +//   SendmailCommand "/usr/sbin/sendmail -odq -oi -t";
 +   SendmailCommand "/bin/echo -odq -oi -t";
     MyEmailAddress "Debian Installer <installer@ftp-master.debian.org>";
     MyAdminAddress "ftpmaster@debian.org";
     MyHost "debian.org";  // used for generating user@my_host addresses in e.g. manual_reject()
       NoSourceOnly "true";
       ReleaseTransitions "/srv/ftp.debian.org/web/transitions.yaml";
     };
+    // if you setup an own dak repository and want to upload Debian packages you most possibly want
+    // to set the following option to a real path/filename and then enter those mail addresses that
+    // you want to be able to receive mails generated by your dak installation. This avoids spamming
+    // the real maintainers of a package you upload with mail.
+    // format of entries: one entry per line. Either an email address directly, or a regular expression,
+    // prefixed by "RE:". Examples: "jane.doe@domain.com" or "RE:jane[^@]@domain.com", where the first will
+    // only allow to mail jane.doe@domain.com while the second will mail all of jane*@domain.com
+    //  MailWhiteList "/some/path/to/a/file";
  };
  
  Transitions
@@@ -140,7 -146,7 +148,7 @@@ Reject-Proposed-Update
  {
     StableRejector "the Stable Release Team";
     StableMail "debian-release@lists.debian.org";
-    MoreInfoURL "http://release.debian.org/stable/4.0/4.0r6/";
+    MoreInfoURL "http://release.debian.org/stable/4.0/4.0r7/";
  };
  
  Import-LDAP-Fingerprints
@@@ -193,7 -199,7 +201,7 @@@ Check-Override
  
  Suite
  {
-   Stable
+   OldStable
    {
        Components
        {
        OverrideCodeName "etch";
        Priority "5";
        Untouchable "1";
+       ChangeLogBase "dists/oldstable/";
+       UdebComponents
+       {
+         main;
+         non-free;
+       };
+   };
+   Oldstable-Proposed-Updates
+   {
+       Components
+       {
+         main;
+         contrib;
+         non-free;
+       };
+       Architectures
+       {
+         source;
+         all;
+         alpha;
+         amd64;
+         arm;
+         hppa;
+         i386;
+         ia64;
+         mips;
+         mipsel;
+         powerpc;
+         s390;
+         sparc;
+       };
+       Announce "debian-changes@lists.debian.org";
+       CopyChanges "dists/oldstable-proposed-updates/";
+       CopyDotDak "/srv/ftp.debian.org/queue/oldstable-proposed-updates/";
+       CommentsDir "/srv/ftp.debian.org/queue/o-p-u-new/COMMENTS/";
+       Version "4.0-updates";
+       Origin "Debian";
+       Description "Debian 4.0 Proposed Updates - Not Released";
+       CodeName "etch-proposed-updates";
+       OverrideCodeName "etch";
+       OverrideSuite "oldstable";
+       ValidTime 604800; // 7 days
+       Priority "4";
+       VersionChecks
+       {
+         MustBeNewerThan
+         {
+           OldStable;
+         };
+         MustBeOlderThan
+         {
+           Stable;
+           Testing;
+           Unstable;
+           Experimental;
+         };
+         Enhances
+           {
+           OldStable;
+         };
+       };
+       UdebComponents
+       {
+         main;
+       };
+   };
+   Stable
+   {
+       Components
+       {
+         main;
+         contrib;
+         non-free;
+       };
+       Architectures
+       {
+         source;
+         all;
+         alpha;
+         amd64;
+         arm;
+         armel;
+         hppa;
+         i386;
+         ia64;
+         mips;
+         mipsel;
+         powerpc;
+         s390;
+         sparc;
+       };
+       Announce "debian-changes@lists.debian.org";
+       // Version "5.0r0";
+       Origin "Debian";
+       // Description "Debian 5.0r0 Released 14 February 2009";
+       CodeName "lenny";
+       OverrideCodeName "lenny";
+       Priority "5";
+       Untouchable "1";
        ChangeLogBase "dists/stable/";
        UdebComponents
        {
          alpha;
          amd64;
          arm;
+           armel;
          hppa;
          i386;
          ia64;
        CopyChanges "dists/proposed-updates/";
        CopyDotDak "/srv/ftp.debian.org/queue/proposed-updates/";
        CommentsDir "/srv/ftp.debian.org/queue/p-u-new/COMMENTS/";
-       Version "4.0-updates";
+       Version "5.0-updates";
        Origin "Debian";
-       Description "Debian 4.0 Proposed Updates - Not Released";
-       CodeName "etch-proposed-updates";
-       OverrideCodeName "etch";
+       Description "Debian 5.0 Proposed Updates - Not Released";
+       CodeName "lenny-proposed-updates";
+       OverrideCodeName "lenny";
        OverrideSuite "stable";
        ValidTime 604800; // 7 days
        Priority "4";
        UdebComponents
        {
          main;
+         non-free;
        };
    };
  
          all;
          alpha;
          amd64;
-         arm;
          armel;
          hppa;
          i386;
        Announce "debian-testing-changes@lists.debian.org";
        Origin "Debian";
        Description "Debian Testing distribution - Not Released";
-       CodeName "lenny";
-       OverrideCodeName "lenny";
+       CodeName "squeeze";
+       OverrideCodeName "squeeze";
        ValidTime 604800; // 7 days
        Priority "5";
        UdebComponents
          all;
          alpha;
          amd64;
-         arm;
          armel;
          hppa;
          i386;
        Announce "debian-testing-changes@lists.debian.org";
        Origin "Debian";
        Description "Debian Testing distribution updates - Not Released";
-       CodeName "lenny-proposed-updates";
-       OverrideCodeName "lenny";
+       CodeName "squeeze-proposed-updates";
+       OverrideCodeName "squeeze";
        OverrideSuite "testing";
        ValidTime 604800; // 7 days
        Priority "6";
          all;
          alpha;
          amd64;
-         arm;
          armel;
          hppa;
          hurd-i386;
          all;
          alpha;
          amd64;
-         arm;
          armel;
          hppa;
          hurd-i386;
  
  SuiteMappings
  {
// "propup-version oldstable-security stable testing testing-proposed-updates unstable";
+  "propup-version oldstable-security stable testing testing-proposed-updates unstable";
   "propup-version stable-security testing testing-proposed-updates unstable";
   "propup-version testing-security unstable";
// "map oldstable oldstable-proposed-updates";
// "map oldstable-security oldstable-proposed-updates";
+  "map oldstable oldstable-proposed-updates";
+  "map oldstable-security oldstable-proposed-updates";
   "map stable proposed-updates";
   "map stable-security proposed-updates";
   "map stable-proposed-updates proposed-updates";
@@@ -750,8 -855,3 +857,8 @@@ Urgenc
      critical;
    };
  };
 +
 +Content
 +{
 +  Header "contents"
 +}
diff --combined dak/dak.py
index aa6efdee840555190e25b9590a3700368b487e6d,981a31a95d0d52e6408c458218c012726a8229bd..15c3e1add0efcdbcccdf187ee5d1b84c1608f39a
@@@ -112,8 -112,6 +112,8 @@@ def init()
           "Generate package <-> file mapping"),
          ("generate-releases",
           "Generate Release files"),
 +        ("contents",
 +         "Generate contest files"),
          ("generate-index-diffs",
           "Generate .diff/Index files"),
          ("clean-suites",
           "Split queue/done into a date-based hierarchy"),
          ("stats",
           "Generate statistics"),
 +        ("calculate-shasums",
 +         "Calculate missing sha1sums and sha256sums"),
          ("bts-categorize",
           "Categorize uncategorized bugs filed against ftp.debian.org"),
+         ("add-user",
+          "Add a user to the archive"),
          ]
      return functionality
  
diff --combined dak/dakdb/update2.py
index e411662c69fd50860c135173ceeda6cacb02c2c2,850e3ab550a5b2c993604d96fe862bf9cf1a0dcd..10407a9c510ce7daa38405b1c165c014025b00be
  
  ################################################################################
  
- import psycopg2, time
+ import psycopg2
+ import time
+ from daklib.dak_exceptions import DBUpdateError
  
  ################################################################################
  
  def do_update(self):
 +vvvvvvvvvvvvvvvvvvvv
      print "Note: to be able to enable the the PL/Perl (plperl) procedural language, we do"
      print "need postgresql-plperl-$postgres-version installed. Make sure that this is the"
      print "case before you continue. Interrupt if it isn't, sleeping 5 seconds now."
      print "(We need to be database superuser for this to work!)"
      time.sleep (5)
 +^^^^^^^^^^^^^^^^^^^^
  
      try:
          c = self.db.cursor()
@@@ -393,6 -393,4 +395,4 @@@ $
  
      except psycopg2.ProgrammingError, msg:
          self.db.rollback()
-         print "FATAL: Unable to apply debversion table update 2!"
-         print "Error Message: " + str(msg)
-         print "Database changes have been rolled back."
+         raise DBUpdateError, "Unable to appy debversion updates, rollback issued. Error message : %s" % (str(msg))
diff --combined dak/update_db.py
index 8bb88f65c41a71685a7b2aff385bd82fba374992,5d5fef477b06972e57bc26b9ec937a283b7840b5..f7d13750d9be48738ee8cb7c78d6ac4c77a28b3a
@@@ -37,7 -37,7 +37,7 @@@ from daklib import util
  
  Cnf = None
  projectB = None
 -required_database_schema = 3
 +required_database_schema = 4
  
  ################################################################################
  
@@@ -137,9 -137,15 +137,15 @@@ Updates dak's database schema to the la
  
          for i in range (database_revision, required_database_schema):
              print "updating databse schema from " + str(database_revision) + " to " + str(i+1)
-             dakdb = __import__("dakdb", globals(), locals(), ['update'+str(i+1)])
-             update_module = getattr(dakdb, "update"+str(i+1))
-             update_module.do_update(self)
+             try:
+                 dakdb = __import__("dakdb", globals(), locals(), ['update'+str(i+1)])
+                 update_module = getattr(dakdb, "update"+str(i+1))
+                 update_module.do_update(self)
+             except DBUpdateError, e:
+                 # Seems the update did not work.
+                 print "Was unable to update database schema from %s to %s." % (str(database_revision), str(i+1))
+                 print "The error message received was %s" % (e)
+                 utils.fubar("DB Schema upgrade failed")
              database_revision += 1
  
  ################################################################################
diff --combined daklib/utils.py
index 5cb502d6d3b51a15fc08fba8843ce9530637bda6,cb5df31ce30fd77e2c086c6266209720fbffec88..1b35feef2eb697e930152b85ca612b3f1ea9e29e
@@@ -37,11 -37,13 +37,14 @@@ import sta
  import apt_pkg
  import database
  import time
 +import tarfile
+ import re
+ import string
+ import email as modemail
  from dak_exceptions import *
  from regexes import re_html_escaping, html_escaping, re_single_line_field, \
                      re_multi_line_field, re_srchasver, re_verwithext, \
-                     re_parse_maintainer, re_taint_free, re_gpg_uid
+                     re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark
  
  ################################################################################
  
@@@ -599,6 -601,67 +602,67 @@@ def send_mail (message, filename="")
          os.write (fd, message)
          os.close (fd)
  
+     if Cnf.has_key("Dinstall::MailWhiteList") and \
+            Cnf["Dinstall::MailWhiteList"] != "":
+         message_in = open_file(filename)
+         message_raw = modemail.message_from_file(message_in)
+         message_in.close();
+         whitelist = [];
+         whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
+         try:
+             for line in whitelist_in:
+                 if re_re_mark.match(line):
+                     whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
+                 else:
+                     whitelist.append(re.compile(re.escape(line.strip())))
+         finally:
+             whitelist_in.close()
+         # Fields to check.
+         fields = ["To", "Bcc", "Cc"]
+         for field in fields:
+             # Check each field
+             value = message_raw.get(field, None)
+             if value != None:
+                 match = [];
+                 for item in value.split(","):
+                     (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip())
+                     mail_whitelisted = 0
+                     for wr in whitelist:
+                         if wr.match(email):
+                             mail_whitelisted = 1
+                             break
+                     if not mail_whitelisted:
+                         print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
+                         continue
+                     match.append(item)
+                 # Doesn't have any mail in whitelist so remove the header
+                 if len(match) == 0:
+                     del message_raw[field]
+                 else:
+                     message_raw.replace_header(field, string.join(match, ", "))
+         # Change message fields in order if we don't have a To header
+         if not message_raw.has_key("To"):
+             fields.reverse()
+             for field in fields:
+                 if message_raw.has_key(field):
+                     message_raw[fields[-1]] = message_raw[field]
+                     del message_raw[field]
+                     break
+             else:
+                 # Clean up any temporary files
+                 # and return, as we removed all recipients.
+                 if message:
+                     os.unlink (filename);
+                 return;
+         fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0700);
+         os.write (fd, message_raw.as_string(True));
+         os.close (fd);
      # Invoke sendmail
      (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
      if (result != 0):
@@@ -1437,53 -1500,3 +1501,53 @@@ if which_conf_file() != default_config
      apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
  
  ################################################################################
 +
 +def generate_contents_information(filename):
 +    """
 +    Generate a list of flies contained in a .deb
 +
 +    @type filename: string
 +    @param filename: the path to a data.tar.gz or data.tar.bz2
 +
 +    @rtype: list
 +    @return: a list of files in the data.tar.* portion of the .deb
 +    """
 +    cmd = "ar t %s" % (filename)
 +    (result, output) = commands.getstatusoutput(cmd)
 +    if result != 0:
 +        reject("%s: 'ar t' invocation failed." % (filename))
 +        reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
 +
 +    # Ugh ... this is ugly ... Code ripped from process_unchecked.py
 +    chunks = output.split('\n')
 +
 +    contents = []
 +    try:
 +        cmd = "ar x %s %s" % (filename, chunks[2])
 +        (result, output) = commands.getstatusoutput(cmd)
 +        if result != 0:
 +            reject("%s: '%s' invocation failed." % (filename, cmd))
 +            reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
 +
 +        # Got deb tarballs, now lets go through and determine what bits
 +        # and pieces the deb had ...
 +        if chunks[2] == "data.tar.gz":
 +            data = tarfile.open("data.tar.gz", "r:gz")
 +        elif chunks[2] == "data.tar.bz2":
 +            data = tarfile.open("data.tar.bz2", "r:bz2")
 +        else:
 +            os.remove(chunks[2])
 +            reject("couldn't find data.tar.*")
 +
 +        for tarinfo in data:
 +            if not tarinfo.isdir():
 +                contents.append(tarinfo.name[2:])
 +
 +    finally:
 +        if os.path.exists( chunks[2] ):
 +            shutil.rmtree( chunks[2] )
 +            os.remove( chunks[2] )
 +
 +    return contents
 +
 +###############################################################################