daklib/queue.py

   1 #!/usr/bin/env python
   2
   3 # Queue utility functions for dak
   4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ###############################################################################
  21
  22 import cPickle, errno, os, pg, re, stat, sys, time
  23 import apt_inst, apt_pkg
  24 import utils, database
  25
  26 from types import *
  27
  28 ###############################################################################
  29
  30 re_isanum = re.compile (r"^\d+$")
  31 re_default_answer = re.compile(r"\[(.*)\]")
  32 re_fdnic = re.compile(r"\n\n")
  33 re_bin_only_nmu = re.compile(r"\+b\d+$")
  34
  35 ###############################################################################
  36
  37 # Convenience wrapper to carry around all the package information in
  38
  39 class Pkg:
  40     def __init__(self, **kwds):
  41         self.__dict__.update(kwds)
  42
  43     def update(self, **kwds):
  44         self.__dict__.update(kwds)
  45
  46 ###############################################################################
  47
  48 class nmu_p:
  49     # Read in the group maintainer override file
  50     def __init__ (self, Cnf):
  51         self.group_maint = {}
  52         self.Cnf = Cnf
  53         if Cnf.get("Dinstall::GroupOverrideFilename"):
  54             filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"]
  55             file = utils.open_file(filename)
  56             for line in file.readlines():
  57                 line = utils.re_comments.sub('', line).lower().strip()
  58                 if line != "":
  59                     self.group_maint[line] = 1
  60             file.close()
  61
  62     def is_an_nmu (self, pkg):
  63         Cnf = self.Cnf
  64         changes = pkg.changes
  65         dsc = pkg.dsc
  66
  67         i = utils.fix_maintainer (dsc.get("maintainer",
  68                                           Cnf["Dinstall::MyEmailAddress"]).lower())
  69         (dsc_rfc822, dsc_rfc2047, dsc_name, dsc_email) = i
  70         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  71         if dsc_name == changes["maintainername"].lower() and \
  72            (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name):
  73             return 0
  74
  75         if dsc.has_key("uploaders"):
  76             uploaders = dsc["uploaders"].lower().split(",")
  77             uploadernames = {}
  78             for i in uploaders:
  79                 (rfc822, rfc2047, name, email) = utils.fix_maintainer (i.strip())
  80                 uploadernames[name] = ""
  81             if uploadernames.has_key(changes["changedbyname"].lower()):
  82                 return 0
  83
  84         # Some group maintained packages (e.g. Debian QA) are never NMU's
  85         if self.group_maint.has_key(changes["maintaineremail"].lower()):
  86             return 0
  87
  88         return 1
  89
  90 ###############################################################################
  91
  92 class Upload:
  93
  94     def __init__(self, Cnf):
  95         self.Cnf = Cnf
  96         # Read in the group-maint override file
  97         self.nmu = nmu_p(Cnf)
  98         self.accept_count = 0
  99         self.accept_bytes = 0L
 100         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 101                        legacy_source_untouchable = {})
 102
 103         # Initialize the substitution template mapping global
 104         Subst = self.Subst = {}
 105         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
 106         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
 107         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
 108         Subst["__DAK_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
 109
 110         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
 111         database.init(Cnf, self.projectB)
 112
 113     ###########################################################################
 114
 115     def init_vars (self):
 116         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 117             exec "self.pkg.%s.clear();" % (i)
 118         self.pkg.orig_tar_id = None
 119         self.pkg.orig_tar_location = ""
 120         self.pkg.orig_tar_gz = None
 121
 122     ###########################################################################
 123
 124     def update_vars (self):
 125         dump_filename = self.pkg.changes_file[:-8]+".dak"
 126         dump_file = utils.open_file(dump_filename)
 127         p = cPickle.Unpickler(dump_file)
 128         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 129             exec "self.pkg.%s.update(p.load());" % (i)
 130         for i in [ "orig_tar_id", "orig_tar_location" ]:
 131             exec "self.pkg.%s = p.load();" % (i)
 132         dump_file.close()
 133
 134     ###########################################################################
 135
 136     # This could just dump the dictionaries as is, but I'd like to
 137     # avoid this so there's some idea of what process-accepted &
 138     # process-new use from process-unchecked
 139
 140     def dump_vars(self, dest_dir):
 141         for i in [ "changes", "dsc", "files", "dsc_files",
 142                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 143             exec "%s = self.pkg.%s;" % (i,i)
 144         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".dak")
 145         dump_file = utils.open_file(dump_filename, 'w')
 146         try:
 147             os.chmod(dump_filename, 0660)
 148         except OSError, e:
 149             if errno.errorcode[e.errno] == 'EPERM':
 150                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE])
 151                 if perms & stat.S_IROTH:
 152                     utils.fubar("%s is world readable and chmod failed." % (dump_filename))
 153             else:
 154                 raise
 155
 156         p = cPickle.Pickler(dump_file, 1)
 157         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 158             exec "%s = {}" % i
 159         ## files
 160         for file in files.keys():
 161             d_files[file] = {}
 162             for i in [ "package", "version", "architecture", "type", "size",
 163                        "md5sum", "component", "location id", "source package",
 164                        "source version", "maintainer", "dbtype", "files id",
 165                        "new", "section", "priority", "othercomponents",
 166                        "pool name", "original component" ]:
 167                 if files[file].has_key(i):
 168                     d_files[file][i] = files[file][i]
 169         ## changes
 170         # Mandatory changes fields
 171         for i in [ "distribution", "source", "architecture", "version",
 172                    "maintainer", "urgency", "fingerprint", "changedby822",
 173                    "changedby2047", "changedbyname", "maintainer822",
 174                    "maintainer2047", "maintainername", "maintaineremail",
 175                    "closes", "changes" ]:
 176             d_changes[i] = changes[i]
 177         # Optional changes fields
 178         for i in [ "changed-by", "filecontents", "format", "process-new note", "adv id", "distribution-version" ]:
 179             if changes.has_key(i):
 180                 d_changes[i] = changes[i]
 181         ## dsc
 182         for i in [ "source", "version", "maintainer", "fingerprint",
 183                    "uploaders", "bts changelog" ]:
 184             if dsc.has_key(i):
 185                 d_dsc[i] = dsc[i]
 186         ## dsc_files
 187         for file in dsc_files.keys():
 188             d_dsc_files[file] = {}
 189             # Mandatory dsc_files fields
 190             for i in [ "size", "md5sum" ]:
 191                 d_dsc_files[file][i] = dsc_files[file][i]
 192             # Optional dsc_files fields
 193             for i in [ "files id" ]:
 194                 if dsc_files[file].has_key(i):
 195                     d_dsc_files[file][i] = dsc_files[file][i]
 196
 197         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 198                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 199             p.dump(i)
 200         dump_file.close()
 201
 202     ###########################################################################
 203
 204     # Set up the per-package template substitution mappings
 205
 206     def update_subst (self, reject_message = ""):
 207         Subst = self.Subst
 208         changes = self.pkg.changes
 209         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
 210         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 211             changes["architecture"] = { "Unknown" : "" }
 212         # and maintainer2047 may not exist.
 213         if not changes.has_key("maintainer2047"):
 214             changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"]
 215
 216         Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys())
 217         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
 218         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "")
 219
 220         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 221         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 222             Subst["__MAINTAINER_FROM__"] = changes["changedby2047"]
 223             Subst["__MAINTAINER_TO__"] = "%s, %s" % (changes["changedby2047"],
 224                                                      changes["maintainer2047"])
 225             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown")
 226         else:
 227             Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"]
 228             Subst["__MAINTAINER_TO__"] = changes["maintainer2047"]
 229             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown")
 230         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 231             Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 232
 233         # Apply any global override of the Maintainer field
 234         if self.Cnf.get("Dinstall::OverrideMaintainer"):
 235             Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"]
 236             Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"]
 237
 238         Subst["__REJECT_MESSAGE__"] = reject_message
 239         Subst["__SOURCE__"] = changes.get("source", "Unknown")
 240         Subst["__VERSION__"] = changes.get("version", "Unknown")
 241
 242     ###########################################################################
 243
 244     def build_summaries(self):
 245         changes = self.pkg.changes
 246         files = self.pkg.files
 247
 248         byhand = summary = new = ""
 249
 250         # changes["distribution"] may not exist in corner cases
 251         # (e.g. unreadable changes files)
 252         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 253             changes["distribution"] = {}
 254
 255         override_summary ="";
 256         file_keys = files.keys()
 257         file_keys.sort()
 258         for file in file_keys:
 259             if files[file].has_key("byhand"):
 260                 byhand = 1
 261                 summary += file + " byhand\n"
 262             elif files[file].has_key("new"):
 263                 new = 1
 264                 summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 265                 if files[file].has_key("othercomponents"):
 266                     summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 267                 if files[file]["type"] == "deb":
 268                     deb_fh = utils.open_file(file)
 269                     summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n'
 270                     deb_fh.close()
 271             else:
 272                 files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"])
 273                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 274                 summary += file + "\n  to " + destination + "\n"
 275                 if files[file]["type"] in ["deb", "udeb", "dsc"]:
 276                     # (queue/unchecked), there we have override entries already, use them
 277                     # (process-new), there we dont have override entries, use the newly generated ones.
 278                     override_prio = files[file].get("override priority", files[file]["priority"])
 279                     override_sect = files[file].get("override section", files[file]["section"])
 280                     override_summary += "%s - %s %s\n" % (file, override_prio, override_sect)
 281
 282         short_summary = summary
 283
 284         # This is for direport's benefit...
 285         f = re_fdnic.sub("\n .\n", changes.get("changes",""))
 286
 287         if byhand or new:
 288             summary += "Changes: " + f
 289
 290         summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
 291
 292         summary += self.announce(short_summary, 0)
 293
 294         return (summary, short_summary)
 295
 296     ###########################################################################
 297
 298     def close_bugs (self, summary, action):
 299         changes = self.pkg.changes
 300         Subst = self.Subst
 301         Cnf = self.Cnf
 302
 303         bugs = changes["closes"].keys()
 304
 305         if not bugs:
 306             return summary
 307
 308         bugs.sort()
 309         if not self.nmu.is_an_nmu(self.pkg):
 310             if changes["distribution"].has_key("experimental"):
 311                 # tag bugs as fixed-in-experimental for uploads to experimental
 312                 summary += "Setting bugs to severity fixed: "
 313                 control_message = ""
 314                 for bug in bugs:
 315                     summary += "%s " % (bug)
 316                     control_message += "tag %s + fixed-in-experimental\n" % (bug)
 317                 if action and control_message != "":
 318                     Subst["__CONTROL_MESSAGE__"] = control_message
 319                     mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.bug-experimental-fixed")
 320                     utils.send_mail (mail_message)
 321                 if action:
 322                     self.Logger.log(["setting bugs to fixed"]+bugs)
 323
 324
 325             else:
 326                 summary += "Closing bugs: "
 327                 for bug in bugs:
 328                     summary += "%s " % (bug)
 329                     if action:
 330                         Subst["__BUG_NUMBER__"] = bug
 331                         if changes["distribution"].has_key("stable"):
 332                             Subst["__STABLE_WARNING__"] = """
 333 Note that this package is not part of the released stable Debian
 334 distribution.  It may have dependencies on other unreleased software,
 335 or other instabilities.  Please take care if you wish to install it.
 336 The update will eventually make its way into the next released Debian
 337 distribution."""
 338                         else:
 339                             Subst["__STABLE_WARNING__"] = ""
 340                             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.bug-close")
 341                             utils.send_mail (mail_message)
 342                 if action:
 343                     self.Logger.log(["closing bugs"]+bugs)
 344
 345         else:                     # NMU
 346             summary += "Setting bugs to severity fixed: "
 347             control_message = ""
 348             for bug in bugs:
 349                 summary += "%s " % (bug)
 350                 control_message += "tag %s + fixed\n" % (bug)
 351             if action and control_message != "":
 352                 Subst["__CONTROL_MESSAGE__"] = control_message
 353                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.bug-nmu-fixed")
 354                 utils.send_mail (mail_message)
 355             if action:
 356                 self.Logger.log(["setting bugs to fixed"]+bugs)
 357         summary += "\n"
 358         return summary
 359
 360     ###########################################################################
 361
 362     def announce (self, short_summary, action):
 363         Subst = self.Subst
 364         Cnf = self.Cnf
 365         changes = self.pkg.changes
 366
 367         # Only do announcements for source uploads with a recent dpkg-dev installed
 368         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 369             return ""
 370
 371         lists_done = {}
 372         summary = ""
 373         Subst["__SHORT_SUMMARY__"] = short_summary
 374
 375         for dist in changes["distribution"].keys():
 376             list = Cnf.Find("Suite::%s::Announce" % (dist))
 377             if list == "" or lists_done.has_key(list):
 378                 continue
 379             lists_done[list] = 1
 380             summary += "Announcing to %s\n" % (list)
 381
 382             if action:
 383                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list
 384                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 385                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"])
 386                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.announce")
 387                 utils.send_mail (mail_message)
 388
 389         if Cnf.FindB("Dinstall::CloseBugs"):
 390             summary = self.close_bugs(summary, action)
 391
 392         return summary
 393
 394     ###########################################################################
 395
 396     def accept (self, summary, short_summary):
 397         Cnf = self.Cnf
 398         Subst = self.Subst
 399         files = self.pkg.files
 400         changes = self.pkg.changes
 401         changes_file = self.pkg.changes_file
 402         dsc = self.pkg.dsc
 403
 404         print "Accepting."
 405         self.Logger.log(["Accepting changes",changes_file])
 406
 407         self.dump_vars(Cnf["Dir::Queue::Accepted"])
 408
 409         # Move all the files into the accepted directory
 410         utils.move(changes_file, Cnf["Dir::Queue::Accepted"])
 411         file_keys = files.keys()
 412         for file in file_keys:
 413             utils.move(file, Cnf["Dir::Queue::Accepted"])
 414             self.accept_bytes += float(files[file]["size"])
 415         self.accept_count += 1
 416
 417         # Send accept mail, announce to lists, close bugs and check for
 418         # override disparities
 419         if not Cnf["Dinstall::Options::No-Mail"]:
 420             Subst["__SUITE__"] = ""
 421             Subst["__SUMMARY__"] = summary
 422             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
 423             utils.send_mail(mail_message)
 424             self.announce(short_summary, 1)
 425
 426
 427         ## Helper stuff for DebBugs Version Tracking
 428         if Cnf.Find("Dir::Queue::BTSVersionTrack"):
 429             # ??? once queue/* is cleared on *.d.o and/or reprocessed
 430             # the conditionalization on dsc["bts changelog"] should be
 431             # dropped.
 432
 433             # Write out the version history from the changelog
 434             if changes["architecture"].has_key("source") and \
 435                dsc.has_key("bts changelog"):
 436
 437                 temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
 438                                                     dotprefix=1, perms=0644)
 439                 version_history = utils.open_file(temp_filename, 'w')
 440                 version_history.write(dsc["bts changelog"])
 441                 version_history.close()
 442                 filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
 443                                       changes_file[:-8]+".versions")
 444                 os.rename(temp_filename, filename)
 445
 446             # Write out the binary -> source mapping.
 447             temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
 448                                                 dotprefix=1, perms=0644)
 449             debinfo = utils.open_file(temp_filename, 'w')
 450             for file in file_keys:
 451                 f = files[file]
 452                 if f["type"] == "deb":
 453                     line = " ".join([f["package"], f["version"],
 454                                      f["architecture"], f["source package"],
 455                                      f["source version"]])
 456                     debinfo.write(line+"\n")
 457             debinfo.close()
 458             filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
 459                                   changes_file[:-8]+".debinfo")
 460             os.rename(temp_filename, filename)
 461
 462         self.queue_build("accepted", Cnf["Dir::Queue::Accepted"])
 463
 464     ###########################################################################
 465
 466     def queue_build (self, queue, path):
 467         Cnf = self.Cnf
 468         Subst = self.Subst
 469         files = self.pkg.files
 470         changes = self.pkg.changes
 471         changes_file = self.pkg.changes_file
 472         dsc = self.pkg.dsc
 473         file_keys = files.keys()
 474
 475         ## Special support to enable clean auto-building of queued packages
 476         queue_id = database.get_or_set_queue_id(queue)
 477
 478         self.projectB.query("BEGIN WORK")
 479         for suite in changes["distribution"].keys():
 480             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
 481                 continue
 482             suite_id = database.get_suite_id(suite)
 483             dest_dir = Cnf["Dir::QueueBuild"]
 484             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
 485                 dest_dir = os.path.join(dest_dir, suite)
 486             for file in file_keys:
 487                 src = os.path.join(path, file)
 488                 dest = os.path.join(dest_dir, file)
 489                 if Cnf.FindB("Dinstall::SecurityQueueBuild"):
 490                     # Copy it since the original won't be readable by www-data
 491                     utils.copy(src, dest)
 492                 else:
 493                     # Create a symlink to it
 494                     os.symlink(src, dest)
 495                 # Add it to the list of packages for later processing by apt-ftparchive
 496                 self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest))
 497             # If the .orig.tar.gz is in the pool, create a symlink to
 498             # it (if one doesn't already exist)
 499             if self.pkg.orig_tar_id:
 500                 # Determine the .orig.tar.gz file name
 501                 for dsc_file in self.pkg.dsc_files.keys():
 502                     if dsc_file.endswith(".orig.tar.gz"):
 503                         filename = dsc_file
 504                 dest = os.path.join(dest_dir, filename)
 505                 # If it doesn't exist, create a symlink
 506                 if not os.path.exists(dest):
 507                     # Find the .orig.tar.gz in the pool
 508                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id))
 509                     ql = q.getresult()
 510                     if not ql:
 511                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id))
 512                     src = os.path.join(ql[0][0], ql[0][1])
 513                     os.symlink(src, dest)
 514                     # Add it to the list of packages for later processing by apt-ftparchive
 515                     self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest))
 516                 # if it does, update things to ensure it's not removed prematurely
 517                 else:
 518                     self.projectB.query("UPDATE queue_build SET in_queue = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id))
 519
 520         self.projectB.query("COMMIT WORK")
 521
 522     ###########################################################################
 523
 524     def check_override (self):
 525         Subst = self.Subst
 526         changes = self.pkg.changes
 527         files = self.pkg.files
 528         Cnf = self.Cnf
 529
 530         # Abandon the check if:
 531         #  a) it's a non-sourceful upload
 532         #  b) override disparity checks have been disabled
 533         #  c) we're not sending mail
 534         if not changes["architecture"].has_key("source") or \
 535            not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 536            Cnf["Dinstall::Options::No-Mail"]:
 537             return
 538
 539         summary = ""
 540         file_keys = files.keys()
 541         file_keys.sort()
 542         for file in file_keys:
 543             if not files[file].has_key("new") and files[file]["type"] == "deb":
 544                 section = files[file]["section"]
 545                 override_section = files[file]["override section"]
 546                 if section.lower() != override_section.lower() and section != "-":
 547                     # Ignore this; it's a common mistake and not worth whining about
 548                     if section.lower() == "non-us/main" and override_section.lower() == "non-us":
 549                         continue
 550                     summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section)
 551                 priority = files[file]["priority"]
 552                 override_priority = files[file]["override priority"]
 553                 if priority != override_priority and priority != "-":
 554                     summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority)
 555
 556         if summary == "":
 557             return
 558
 559         Subst["__SUMMARY__"] = summary
 560         mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/process-unchecked.override-disparity")
 561         utils.send_mail(mail_message)
 562
 563     ###########################################################################
 564
 565     def force_reject (self, files):
 566         """Forcefully move files from the current directory to the
 567            reject directory.  If any file already exists in the reject
 568            directory it will be moved to the morgue to make way for
 569            the new file."""
 570
 571         Cnf = self.Cnf
 572
 573         for file in files:
 574             # Skip any files which don't exist or which we don't have permission to copy.
 575             if os.access(file,os.R_OK) == 0:
 576                 continue
 577             dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file)
 578             try:
 579                 dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
 580             except OSError, e:
 581                 # File exists?  Let's try and move it to the morgue
 582                 if errno.errorcode[e.errno] == 'EEXIST':
 583                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file)
 584                     try:
 585                         morgue_file = utils.find_next_free(morgue_file)
 586                     except utils.tried_too_hard_exc:
 587                         # Something's either gone badly Pete Tong, or
 588                         # someone is trying to exploit us.
 589                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file))
 590                         return
 591                     utils.move(dest_file, morgue_file, perms=0660)
 592                     try:
 593                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
 594                     except OSError, e:
 595                         # Likewise
 596                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file))
 597                         return
 598                 else:
 599                     raise
 600             # If we got here, we own the destination file, so we can
 601             # safely overwrite it.
 602             utils.move(file, dest_file, 1, perms=0660)
 603             os.close(dest_fd)
 604
 605     ###########################################################################
 606
 607     def do_reject (self, manual = 0, reject_message = ""):
 608         # If we weren't given a manual rejection message, spawn an
 609         # editor so the user can add one in...
 610         if manual and not reject_message:
 611             temp_filename = utils.temp_filename()
 612             editor = os.environ.get("EDITOR","vi")
 613             answer = 'E'
 614             while answer == 'E':
 615                 os.system("%s %s" % (editor, temp_filename))
 616                 temp_fh = utils.open_file(temp_filename)
 617                 reject_message = "".join(temp_fh.readlines())
 618                 temp_fh.close()
 619                 print "Reject message:"
 620                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1)
 621                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 622                 answer = "XXX"
 623                 while prompt.find(answer) == -1:
 624                     answer = utils.our_raw_input(prompt)
 625                     m = re_default_answer.search(prompt)
 626                     if answer == "":
 627                         answer = m.group(1)
 628                     answer = answer[:1].upper()
 629             os.unlink(temp_filename)
 630             if answer == 'A':
 631                 return 1
 632             elif answer == 'Q':
 633                 sys.exit(0)
 634
 635         print "Rejecting.\n"
 636
 637         Cnf = self.Cnf
 638         Subst = self.Subst
 639         pkg = self.pkg
 640
 641         reason_filename = pkg.changes_file[:-8] + ".reason"
 642         reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename
 643
 644         # Move all the files into the reject directory
 645         reject_files = pkg.files.keys() + [pkg.changes_file]
 646         self.force_reject(reject_files)
 647
 648         # If we fail here someone is probably trying to exploit the race
 649         # so let's just raise an exception ...
 650         if os.path.exists(reason_filename):
 651             os.unlink(reason_filename)
 652         reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
 653
 654         if not manual:
 655             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
 656             Subst["__MANUAL_REJECT_MESSAGE__"] = ""
 657             Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
 658             os.write(reason_fd, reject_message)
 659             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/queue.rejected")
 660         else:
 661             # Build up the rejection email
 662             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"])
 663
 664             Subst["__REJECTOR_ADDRESS__"] = user_email_address
 665             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
 666             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 667             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/queue.rejected")
 668             # Write the rejection email out as the <foo>.reason file
 669             os.write(reason_fd, reject_mail_message)
 670
 671         os.close(reason_fd)
 672
 673         # Send the rejection mail if appropriate
 674         if not Cnf["Dinstall::Options::No-Mail"]:
 675             utils.send_mail(reject_mail_message)
 676
 677         self.Logger.log(["rejected", pkg.changes_file])
 678         return 0
 679
 680     ################################################################################
 681
 682     # Ensure that source exists somewhere in the archive for the binary
 683     # upload being processed.
 684     #
 685     # (1) exact match                      => 1.0-3
 686     # (2) Bin-only NMU                     => 1.0-3+b1 , 1.0-3.1+b1
 687
 688     def source_exists (self, package, source_version, suites = ["any"]):
 689         okay = 1
 690         for suite in suites:
 691             if suite == "any":
 692                 que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
 693                     (package)
 694             else:
 695                 # source must exist in suite X, or in some other suite that's
 696                 # mapped to X, recursively... silent-maps are counted too,
 697                 # unreleased-maps aren't.
 698                 maps = self.Cnf.ValueList("SuiteMappings")[:]
 699                 maps.reverse()
 700                 maps = [ m.split() for m in maps ]
 701                 maps = [ (x[1], x[2]) for x in maps
 702                                 if x[0] == "map" or x[0] == "silent-map" ]
 703                 s = [suite]
 704                 for x in maps:
 705                         if x[1] in s and x[0] not in s:
 706                                 s.append(x[0])
 707
 708                 que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, " OR ".join(["su.suite_name = '%s'" % a for a in s]))
 709             q = self.projectB.query(que)
 710
 711             # Reduce the query results to a list of version numbers
 712             ql = [ i[0] for i in q.getresult() ]
 713
 714             # Try (1)
 715             if source_version in ql:
 716                 continue
 717
 718             # Try (2)
 719             orig_source_version = re_bin_only_nmu.sub('', source_version)
 720             if orig_source_version in ql:
 721                 continue
 722
 723             # No source found...
 724             okay = 0
 725             break
 726         return okay
 727
 728     ################################################################################
 729
 730     def in_override_p (self, package, component, suite, binary_type, file):
 731         files = self.pkg.files
 732
 733         if binary_type == "": # must be source
 734             type = "dsc"
 735         else:
 736             type = binary_type
 737
 738         # Override suite name; used for example with proposed-updates
 739         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 740             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)]
 741
 742         # Avoid <undef> on unknown distributions
 743         suite_id = database.get_suite_id(suite)
 744         if suite_id == -1:
 745             return None
 746         component_id = database.get_component_id(component)
 747         type_id = database.get_override_type_id(type)
 748
 749         # FIXME: nasty non-US speficic hack
 750         if component.lower().startswith("non-us/"):
 751             component = component[7:]
 752
 753         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 754                            % (package, suite_id, component_id, type_id))
 755         result = q.getresult()
 756         # If checking for a source package fall back on the binary override type
 757         if type == "dsc" and not result:
 758             deb_type_id = database.get_override_type_id("deb")
 759             udeb_type_id = database.get_override_type_id("udeb")
 760             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
 761                                % (package, suite_id, component_id, deb_type_id, udeb_type_id))
 762             result = q.getresult()
 763
 764         # Remember the section and priority so we can check them later if appropriate
 765         if result:
 766             files[file]["override section"] = result[0][0]
 767             files[file]["override priority"] = result[0][1]
 768
 769         return result
 770
 771     ################################################################################
 772
 773     def reject (self, str, prefix="Rejected: "):
 774         if str:
 775             # Unlike other rejects we add new lines first to avoid trailing
 776             # new lines when this message is passed back up to a caller.
 777             if self.reject_message:
 778                 self.reject_message += "\n"
 779             self.reject_message += prefix + str
 780
 781     ################################################################################
 782
 783     def get_anyversion(self, query_result, suite):
 784         anyversion=None
 785         anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
 786         for (v, s) in query_result:
 787             if s in [ x.lower() for x in anysuite ]:
 788                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
 789                     anyversion=v
 790         return anyversion
 791
 792     ################################################################################
 793
 794     def cross_suite_version_check(self, query_result, file, new_version):
 795         """Ensure versions are newer than existing packages in target
 796         suites and that cross-suite version checking rules as
 797         set out in the conf file are satisfied."""
 798
 799         # Check versions for each target suite
 800         for target_suite in self.pkg.changes["distribution"].keys():
 801             must_be_newer_than = [ i.lower for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
 802             must_be_older_than = [ i.lower for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
 803             # Enforce "must be newer than target suite" even if conffile omits it
 804             if target_suite not in must_be_newer_than:
 805                 must_be_newer_than.append(target_suite)
 806             for entry in query_result:
 807                 existent_version = entry[0]
 808                 suite = entry[1]
 809                 if suite in must_be_newer_than and \
 810                    apt_pkg.VersionCompare(new_version, existent_version) < 1:
 811                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
 812                 if suite in must_be_older_than and \
 813                    apt_pkg.VersionCompare(new_version, existent_version) > -1:
 814                     ch = self.pkg.changes
 815                     cansave = 0
 816                     if ch.get('distribution-version', {}).has_key(suite):
 817                         # we really use the other suite, ignoring the conflicting one ...
 818                         addsuite = ch["distribution-version"][suite]
 819
 820                         add_version = self.get_anyversion(query_result, addsuite)
 821                         target_version = self.get_anyversion(query_result, target_suite)
 822
 823                         if not add_version:
 824                             # not add_version can only happen if we map to a suite
 825                             # that doesn't enhance the suite we're propup'ing from.
 826                             # so "propup-ver x a b c; map a d" is a problem only if
 827                             # d doesn't enhance a.
 828                             #
 829                             # i think we could always propagate in this case, rather
 830                             # than complaining. either way, this isn't a REJECT issue
 831                             #
 832                             # And - we really should complain to the dorks who configured dak
 833                             self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ")
 834                             self.pkg.changes.setdefault("propdistribution", {})
 835                             self.pkg.changes["propdistribution"][addsuite] = 1
 836                             cansave = 1
 837                         elif not target_version:
 838                             # not targets_version is true when the package is NEW
 839                             # we could just stick with the "...old version..." REJECT
 840                             # for this, I think.
 841                             self.reject("Won't propogate NEW packages.")
 842                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
 843                             # propogation would be redundant. no need to reject though.
 844                             self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ")
 845                             cansave = 1
 846                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
 847                              apt_pkg.VersionCompare(add_version, target_version) >= 0:
 848                             # propogate!!
 849                             self.reject("Propogating upload to %s" % (addsuite), "Warning: ")
 850                             self.pkg.changes.setdefault("propdistribution", {})
 851                             self.pkg.changes["propdistribution"][addsuite] = 1
 852                             cansave = 1
 853
 854                     if not cansave:
 855                         self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
 856
 857     ################################################################################
 858
 859     def check_binary_against_db(self, file):
 860         self.reject_message = ""
 861         files = self.pkg.files
 862
 863         # Ensure version is sane
 864         q = self.projectB.query("""
 865 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
 866                                      architecture a
 867  WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
 868    AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
 869                                 % (files[file]["package"],
 870                                    files[file]["architecture"]))
 871         self.cross_suite_version_check(q.getresult(), file, files[file]["version"])
 872
 873         # Check for any existing copies of the file
 874         q = self.projectB.query("""
 875 SELECT b.id FROM binaries b, architecture a
 876  WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
 877    AND a.id = b.architecture"""
 878                                 % (files[file]["package"],
 879                                    files[file]["version"],
 880                                    files[file]["architecture"]))
 881         if q.getresult():
 882             self.reject("%s: can not overwrite existing copy already in the archive." % (file))
 883
 884         return self.reject_message
 885
 886     ################################################################################
 887
 888     def check_source_against_db(self, file):
 889         self.reject_message = ""
 890         dsc = self.pkg.dsc
 891
 892         # Ensure version is sane
 893         q = self.projectB.query("""
 894 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
 895  WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")))
 896         self.cross_suite_version_check(q.getresult(), file, dsc.get("version"))
 897
 898         return self.reject_message
 899
 900     ################################################################################
 901
 902     # **WARNING**
 903     # NB: this function can remove entries from the 'files' index [if
 904     # the .orig.tar.gz is a duplicate of the one in the archive]; if
 905     # you're iterating over 'files' and call this function as part of
 906     # the loop, be sure to add a check to the top of the loop to
 907     # ensure you haven't just tried to derefernece the deleted entry.
 908     # **WARNING**
 909
 910     def check_dsc_against_db(self, file):
 911         self.reject_message = ""
 912         files = self.pkg.files
 913         dsc_files = self.pkg.dsc_files
 914         legacy_source_untouchable = self.pkg.legacy_source_untouchable
 915         self.pkg.orig_tar_gz = None
 916
 917         # Try and find all files mentioned in the .dsc.  This has
 918         # to work harder to cope with the multiple possible
 919         # locations of an .orig.tar.gz.
 920         for dsc_file in dsc_files.keys():
 921             found = None
 922             if files.has_key(dsc_file):
 923                 actual_md5 = files[dsc_file]["md5sum"]
 924                 actual_size = int(files[dsc_file]["size"])
 925                 found = "%s in incoming" % (dsc_file)
 926                 # Check the file does not already exist in the archive
 927                 q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file))
 928                 ql = q.getresult()
 929                 # Strip out anything that isn't '%s' or '/%s$'
 930                 for i in ql:
 931                     if i[3] != dsc_file and i[3][-(len(dsc_file)+1):] != '/'+dsc_file:
 932                         ql.remove(i)
 933
 934                 # "[dak] has not broken them.  [dak] has fixed a
 935                 # brokenness.  Your crappy hack exploited a bug in
 936                 # the old dinstall.
 937                 #
 938                 # "(Come on!  I thought it was always obvious that
 939                 # one just doesn't release different files with
 940                 # the same name and version.)"
 941                 #                        -- ajk@ on d-devel@l.d.o
 942
 943                 if ql:
 944                     # Ignore exact matches for .orig.tar.gz
 945                     match = 0
 946                     if dsc_file.endswith(".orig.tar.gz"):
 947                         for i in ql:
 948                             if files.has_key(dsc_file) and \
 949                                int(files[dsc_file]["size"]) == int(i[0]) and \
 950                                files[dsc_file]["md5sum"] == i[1]:
 951                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ")
 952                                 del files[dsc_file]
 953                                 self.pkg.orig_tar_gz = i[2] + i[3]
 954                                 match = 1
 955
 956                     if not match:
 957                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file))
 958             elif dsc_file.endswith(".orig.tar.gz"):
 959                 # Check in the pool
 960                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file))
 961                 ql = q.getresult()
 962                 # Strip out anything that isn't '%s' or '/%s$'
 963                 for i in ql:
 964                     if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
 965                         ql.remove(i)
 966
 967                 if ql:
 968                     # Unfortunately, we may get more than one match here if,
 969                     # for example, the package was in potato but had an -sa
 970                     # upload in woody.  So we need to choose the right one.
 971
 972                     x = ql[0]; # default to something sane in case we don't match any or have only one
 973
 974                     if len(ql) > 1:
 975                         for i in ql:
 976                             old_file = i[0] + i[1]
 977                             old_file_fh = utils.open_file(old_file)
 978                             actual_md5 = apt_pkg.md5sum(old_file_fh)
 979                             old_file_fh.close()
 980                             actual_size = os.stat(old_file)[stat.ST_SIZE]
 981                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 982                                 x = i
 983                             else:
 984                                 legacy_source_untouchable[i[3]] = ""
 985
 986                     old_file = x[0] + x[1]
 987                     old_file_fh = utils.open_file(old_file)
 988                     actual_md5 = apt_pkg.md5sum(old_file_fh)
 989                     old_file_fh.close()
 990                     actual_size = os.stat(old_file)[stat.ST_SIZE]
 991                     found = old_file
 992                     suite_type = x[2]
 993                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 994                     # See install() in process-accepted...
 995                     self.pkg.orig_tar_id = x[3]
 996                     self.pkg.orig_tar_gz = old_file
 997                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 998                         self.pkg.orig_tar_location = "legacy"
 999                     else:
1000                         self.pkg.orig_tar_location = x[4]
1001                 else:
1002                     # Not there? Check the queue directories...
1003
1004                     in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file)
1005                     # See process_it() in 'dak process-unchecked' for explanation of this
1006                     if os.path.exists(in_unchecked):
1007                         return (self.reject_message, in_unchecked)
1008                     else:
1009                         for dir in [ "Accepted", "New", "Byhand" ]:
1010                             in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file)
1011                             if os.path.exists(in_otherdir):
1012                                 in_otherdir_fh = utils.open_file(in_otherdir)
1013                                 actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
1014                                 in_otherdir_fh.close()
1015                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
1016                                 found = in_otherdir
1017                                 self.pkg.orig_tar_gz = in_otherdir
1018
1019                     if not found:
1020                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file))
1021                         self.pkg.orig_tar_gz = -1
1022                         continue
1023             else:
1024                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file))
1025                 continue
1026             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
1027                 self.reject("md5sum for %s doesn't match %s." % (found, file))
1028             if actual_size != int(dsc_files[dsc_file]["size"]):
1029                 self.reject("size for %s doesn't match %s." % (found, file))
1030
1031         return (self.reject_message, None)
1032
1033     def do_query(self, q):
1034         sys.stderr.write("query: \"%s\" ... " % (q))
1035         before = time.time()
1036         r = self.projectB.query(q)
1037         time_diff = time.time()-before
1038         sys.stderr.write("took %.3f seconds.\n" % (time_diff))
1039         return r