daklib/queue.py

   1 #!/usr/bin/env python
   2 # vim:set et sw=4:
   3
   4 """
   5 Queue utility functions for dak
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
   9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ###############################################################################
  28
  29 import cPickle
  30 import errno
  31 import os
  32 import pg
  33 import stat
  34 import sys
  35 import time
  36 import apt_inst
  37 import apt_pkg
  38 import utils
  39 from types import *
  40
  41 from dak_exceptions import *
  42 from changes import *
  43 from regexes import re_default_answer, re_fdnic, re_bin_only_nmu
  44 from config import Config
  45 from dbconn import *
  46 from summarystats import SummaryStats
  47
  48 ###############################################################################
  49
  50 def get_type(f, session=None):
  51     """
  52     Get the file type of C{f}
  53
  54     @type f: dict
  55     @param f: file entry from Changes object
  56
  57     @rtype: string
  58     @return: filetype
  59
  60     """
  61     if session is None:
  62         session = DBConn().session()
  63
  64     # Determine the type
  65     if f.has_key("dbtype"):
  66         file_type = file["dbtype"]
  67     elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
  68         file_type = "dsc"
  69     else:
  70         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))
  71
  72     # Validate the override type
  73     type_id = get_override_type(file_type, session)
  74     if type_id is None:
  75         utils.fubar("invalid type (%s) for new.  Say wha?" % (file_type))
  76
  77     return file_type
  78
  79 ################################################################################
  80
  81 # Determine what parts in a .changes are NEW
  82
  83 def determine_new(changes, files, warn=1):
  84     """
  85     Determine what parts in a C{changes} file are NEW.
  86
  87     @type changes: Upload.Pkg.changes dict
  88     @param changes: Changes dictionary
  89
  90     @type files: Upload.Pkg.files dict
  91     @param files: Files dictionary
  92
  93     @type warn: bool
  94     @param warn: Warn if overrides are added for (old)stable
  95
  96     @rtype: dict
  97     @return: dictionary of NEW components.
  98
  99     """
 100     new = {}
 101
 102     session = DBConn().session()
 103
 104     # Build up a list of potentially new things
 105     for name, f in files.items():
 106         # Skip byhand elements
 107         if f["type"] == "byhand":
 108             continue
 109         pkg = f["package"]
 110         priority = f["priority"]
 111         section = f["section"]
 112         file_type = get_type(f)
 113         component = f["component"]
 114
 115         if file_type == "dsc":
 116             priority = "source"
 117
 118         if not new.has_key(pkg):
 119             new[pkg] = {}
 120             new[pkg]["priority"] = priority
 121             new[pkg]["section"] = section
 122             new[pkg]["type"] = file_type
 123             new[pkg]["component"] = component
 124             new[pkg]["files"] = []
 125         else:
 126             old_type = new[pkg]["type"]
 127             if old_type != file_type:
 128                 # source gets trumped by deb or udeb
 129                 if old_type == "dsc":
 130                     new[pkg]["priority"] = priority
 131                     new[pkg]["section"] = section
 132                     new[pkg]["type"] = file_type
 133                     new[pkg]["component"] = component
 134
 135         new[pkg]["files"].append(name)
 136
 137         if f.has_key("othercomponents"):
 138             new[pkg]["othercomponents"] = f["othercomponents"]
 139
 140     for suite in changes["suite"].keys():
 141         for pkg in new.keys():
 142             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
 143             if len(ql) > 0:
 144                 for file_entry in new[pkg]["files"]:
 145                     if files[file_entry].has_key("new"):
 146                         del files[file_entry]["new"]
 147                 del new[pkg]
 148
 149     if warn:
 150         for s in ['stable', 'oldstable']:
 151             if changes["suite"].has_key(s):
 152                 print "WARNING: overrides will be added for %s!" % s
 153         for pkg in new.keys():
 154             if new[pkg].has_key("othercomponents"):
 155                 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 156
 157     return new
 158
 159 ################################################################################
 160
 161 def check_valid(new):
 162     """
 163     Check if section and priority for NEW packages exist in database.
 164     Additionally does sanity checks:
 165       - debian-installer packages have to be udeb (or source)
 166       - non debian-installer packages can not be udeb
 167       - source priority can only be assigned to dsc file types
 168
 169     @type new: dict
 170     @param new: Dict of new packages with their section, priority and type.
 171
 172     """
 173     for pkg in new.keys():
 174         section_name = new[pkg]["section"]
 175         priority_name = new[pkg]["priority"]
 176         file_type = new[pkg]["type"]
 177
 178         section = get_section(section_name)
 179         if section is None:
 180             new[pkg]["section id"] = -1
 181         else:
 182             new[pkg]["section id"] = section.section_id
 183
 184         priority = get_priority(priority_name)
 185         if priority is None:
 186             new[pkg]["priority id"] = -1
 187         else:
 188             new[pkg]["priority id"] = priority.priority_id
 189
 190         # Sanity checks
 191         di = section_name.find("debian-installer") != -1
 192
 193         # If d-i, we must be udeb and vice-versa
 194         if     (di and file_type not in ("udeb", "dsc")) or \
 195            (not di and file_type == "udeb"):
 196             new[pkg]["section id"] = -1
 197
 198         # If dsc we need to be source and vice-versa
 199         if (priority == "source" and file_type != "dsc") or \
 200            (priority != "source" and file_type == "dsc"):
 201             new[pkg]["priority id"] = -1
 202
 203 ###############################################################################
 204
 205 class Upload(object):
 206     """
 207     Everything that has to do with an upload processed.
 208
 209     """
 210     def __init__(self):
 211         self.pkg = Changes()
 212         self.reset()
 213
 214     ###########################################################################
 215
 216     def reset (self):
 217         """ Reset a number of internal variables."""
 218
 219        # Initialize the substitution template map
 220         cnf = Config()
 221         self.Subst = {}
 222         self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
 223         self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
 224         self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
 225         self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 226
 227         self.reject_message = ""
 228         self.pkg.reset()
 229
 230     ###########################################################################
 231     def update_subst(self, reject_message = ""):
 232         """ Set up the per-package template substitution mappings """
 233
 234         cnf = Config()
 235
 236         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
 237         if not self.pkg.changes.has_key("architecture") or not \
 238            isinstance(changes["architecture"], DictType):
 239             self.pkg.changes["architecture"] = { "Unknown" : "" }
 240
 241         # and maintainer2047 may not exist.
 242         if not self.pkg.changes.has_key("maintainer2047"):
 243             self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
 244
 245         self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
 246         self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
 247         self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
 248
 249         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 250         if self.pkg.changes["architecture"].has_key("source") and \
 251            self.pkg.changes["changedby822"] != "" and \
 252            (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
 253
 254             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
 255             self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], changes["maintainer2047"])
 256             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
 257         else:
 258             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
 259             self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
 260             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
 261
 262         if "sponsoremail" in self.pkg.changes:
 263             self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
 264
 265         if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
 266             self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
 267
 268         # Apply any global override of the Maintainer field
 269         if cnf.get("Dinstall::OverrideMaintainer"):
 270             self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
 271             self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
 272
 273         self.Subst["__REJECT_MESSAGE__"] = self.reject_message
 274         self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
 275         self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
 276
 277     ###########################################################################
 278
 279     def build_summaries(self):
 280         """ Build a summary of changes the upload introduces. """
 281
 282         (byhand, new, summary, override_summary) = self.pkg.file_summary()
 283
 284         short_summary = summary
 285
 286         # This is for direport's benefit...
 287         f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
 288
 289         if byhand or new:
 290             summary += "Changes: " + f
 291
 292         summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
 293
 294         summary += self.announce(short_summary, 0)
 295
 296         return (summary, short_summary)
 297
 298     ###########################################################################
 299
 300     def close_bugs(self, summary, action):
 301         """
 302         Send mail to close bugs as instructed by the closes field in the changes file.
 303         Also add a line to summary if any work was done.
 304
 305         @type summary: string
 306         @param summary: summary text, as given by L{build_summaries}
 307
 308         @type action: bool
 309         @param action: Set to false no real action will be done.
 310
 311         @rtype: string
 312         @return: summary. If action was taken, extended by the list of closed bugs.
 313
 314         """
 315
 316         template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
 317
 318         bugs = self.pkg.changes["closes"].keys()
 319
 320         if not bugs:
 321             return summary
 322
 323         bugs.sort()
 324         summary += "Closing bugs: "
 325         for bug in bugs:
 326             summary += "%s " % (bug)
 327             if action:
 328                 self.Subst["__BUG_NUMBER__"] = bug
 329                 if self.pkg.changes["distribution"].has_key("stable"):
 330                     self.Subst["__STABLE_WARNING__"] = """
 331 Note that this package is not part of the released stable Debian
 332 distribution.  It may have dependencies on other unreleased software,
 333 or other instabilities.  Please take care if you wish to install it.
 334 The update will eventually make its way into the next released Debian
 335 distribution."""
 336                 else:
 337                     self.Subst["__STABLE_WARNING__"] = ""
 338                     mail_message = utils.TemplateSubst(self.Subst, template)
 339                     utils.send_mail(mail_message)
 340
 341                 # Clear up after ourselves
 342                 del self.Subst["__BUG_NUMBER__"]
 343                 del self.Subst["__STABLE_WARNING__"]
 344
 345         if action:
 346             self.Logger.log(["closing bugs"] + bugs)
 347
 348         summary += "\n"
 349
 350         return summary
 351
 352     ###########################################################################
 353
 354     def announce(self, short_summary, action):
 355         """
 356         Send an announce mail about a new upload.
 357
 358         @type short_summary: string
 359         @param short_summary: Short summary text to include in the mail
 360
 361         @type action: bool
 362         @param action: Set to false no real action will be done.
 363
 364         @rtype: string
 365         @return: Textstring about action taken.
 366
 367         """
 368
 369         cnf = Config()
 370         announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
 371
 372         # Only do announcements for source uploads with a recent dpkg-dev installed
 373         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
 374            self.pkg.changes["architecture"].has_key("source"):
 375             return ""
 376
 377         lists_done = {}
 378         summary = ""
 379
 380         self.Subst["__SHORT_SUMMARY__"] = short_summary
 381
 382         for dist in self.pkg.changes["distribution"].keys():
 383             announce_list = Cnf.Find("Suite::%s::Announce" % (dist))
 384             if announce_list == "" or lists_done.has_key(announce_list):
 385                 continue
 386
 387             lists_done[announce_list] = 1
 388             summary += "Announcing to %s\n" % (announce_list)
 389
 390             if action:
 391                 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
 392                 if cnf.get("Dinstall::TrackingServer") and \
 393                    self.pkg.changes["architecture"].has_key("source"):
 394                     trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
 395                     self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
 396
 397                 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
 398                 utils.send_mail(mail_message)
 399
 400                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
 401
 402         if cnf.FindB("Dinstall::CloseBugs"):
 403             summary = self.close_bugs(summary, action)
 404
 405         del self.Subst["__SHORT_SUMMARY__"]
 406
 407         return summary
 408
 409     ###########################################################################
 410
 411     def accept (self, summary, short_summary, targetdir=None):
 412         """
 413         Accept an upload.
 414
 415         This moves all files referenced from the .changes into the I{accepted}
 416         queue, sends the accepted mail, announces to lists, closes bugs and
 417         also checks for override disparities. If enabled it will write out
 418         the version history for the BTS Version Tracking and will finally call
 419         L{queue_build}.
 420
 421         @type summary: string
 422         @param summary: Summary text
 423
 424         @type short_summary: string
 425         @param short_summary: Short summary
 426
 427         """
 428
 429         cnf = Config()
 430         stats = SummaryStats()
 431
 432         accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
 433
 434         if targetdir is None:
 435             targetdir = cnf["Dir::Queue::Accepted"]
 436
 437         print "Accepting."
 438         self.Logger.log(["Accepting changes", self.pkg.changes_file])
 439
 440         self.write_dot_dak(targetdir)
 441
 442         # Move all the files into the accepted directory
 443         utils.move(self.pkg.changes_file, targetdir)
 444
 445         for name, entry in sorted(self.pkg.files.items()):
 446             utils.move(name, targetdir)
 447             stats.accept_bytes += float(entry["size"])
 448
 449         stats.accept_count += 1
 450
 451         # Send accept mail, announce to lists, close bugs and check for
 452         # override disparities
 453         if not cnf["Dinstall::Options::No-Mail"]:
 454             self.Subst["__SUITE__"] = ""
 455             self.Subst["__SUMMARY__"] = summary
 456             mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
 457             utils.send_mail(mail_message)
 458             self.announce(short_summary, 1)
 459
 460         ## Helper stuff for DebBugs Version Tracking
 461         if cnf.Find("Dir::Queue::BTSVersionTrack"):
 462             # ??? once queue/* is cleared on *.d.o and/or reprocessed
 463             # the conditionalization on dsc["bts changelog"] should be
 464             # dropped.
 465
 466             # Write out the version history from the changelog
 467             if self.pkg.changes["architecture"].has_key("source") and \
 468                self.pkg.dsc.has_key("bts changelog"):
 469
 470                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
 471                 version_history = os.fdopen(fd, 'w')
 472                 version_history.write(self.pkg.dsc["bts changelog"])
 473                 version_history.close()
 474                 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
 475                                       self.pkg.changes_file[:-8]+".versions")
 476                 os.rename(temp_filename, filename)
 477                 os.chmod(filename, 0644)
 478
 479             # Write out the binary -> source mapping.
 480             (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
 481             debinfo = os.fdopen(fd, 'w')
 482             for name, entry in sorted(self.pkg.files.items()):
 483                 if entry["type"] == "deb":
 484                     line = " ".join([entry["package"], entry["version"],
 485                                      entry["architecture"], entry["source package"],
 486                                      entry["source version"]])
 487                     debinfo.write(line+"\n")
 488             debinfo.close()
 489             filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
 490                                   self.pkg.changes_file[:-8]+".debinfo")
 491             os.rename(temp_filename, filename)
 492             os.chmod(filename, 0644)
 493
 494         # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
 495         # <Ganneff> we do call queue_build too
 496         # <mhy> well yes, we'd have had to if we were inserting into accepted
 497         # <Ganneff> now. thats database only.
 498         # <mhy> urgh, that's going to get messy
 499         # <Ganneff> so i make the p-n call to it *also* using accepted/
 500         # <mhy> but then the packages will be in the queue_build table without the files being there
 501         # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
 502         # <mhy> ah, good point
 503         # <Ganneff> so it will work out, as unchecked move it over
 504         # <mhy> that's all completely sick
 505         # <Ganneff> yes
 506
 507         # This routine returns None on success or an error on failure
 508         res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
 509         if res:
 510             utils.fubar(res)
 511
 512
 513     def check_override (self):
 514         """
 515         Checks override entries for validity. Mails "Override disparity" warnings,
 516         if that feature is enabled.
 517
 518         Abandons the check if
 519           - override disparity checks are disabled
 520           - mail sending is disabled
 521         """
 522
 523         cnf = Config()
 524
 525         # Abandon the check if:
 526         #  a) override disparity checks have been disabled
 527         #  b) we're not sending mail
 528         if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 529            cnf["Dinstall::Options::No-Mail"]:
 530             return
 531
 532         summary = self.pkg.check_override()
 533
 534         if summary == "":
 535             return
 536
 537         overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
 538
 539         self.Subst["__SUMMARY__"] = summary
 540         mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
 541         utils.send_mail(mail_message)
 542         del self.Subst["__SUMMARY__"]
 543
 544     ###########################################################################
 545     def force_reject(self, reject_files):
 546         """
 547         Forcefully move files from the current directory to the
 548         reject directory.  If any file already exists in the reject
 549         directory it will be moved to the morgue to make way for
 550         the new file.
 551
 552         @type files: dict
 553         @param files: file dictionary
 554
 555         """
 556
 557         cnf = Config()
 558
 559         for file_entry in reject_files:
 560             # Skip any files which don't exist or which we don't have permission to copy.
 561             if os.access(file_entry, os.R_OK) == 0:
 562                 continue
 563
 564             dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
 565
 566             try:
 567                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
 568             except OSError, e:
 569                 # File exists?  Let's try and move it to the morgue
 570                 if e.errno == errno.EEXIST:
 571                     morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
 572                     try:
 573                         morgue_file = utils.find_next_free(morgue_file)
 574                     except NoFreeFilenameError:
 575                         # Something's either gone badly Pete Tong, or
 576                         # someone is trying to exploit us.
 577                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
 578                         return
 579                     utils.move(dest_file, morgue_file, perms=0660)
 580                     try:
 581                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
 582                     except OSError, e:
 583                         # Likewise
 584                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
 585                         return
 586                 else:
 587                     raise
 588             # If we got here, we own the destination file, so we can
 589             # safely overwrite it.
 590             utils.move(file_entry, dest_file, 1, perms=0660)
 591             os.close(dest_fd)
 592
 593     ###########################################################################
 594     def do_reject (self, manual=0, reject_message="", note=""):
 595         """
 596         Reject an upload. If called without a reject message or C{manual} is
 597         true, spawn an editor so the user can write one.
 598
 599         @type manual: bool
 600         @param manual: manual or automated rejection
 601
 602         @type reject_message: string
 603         @param reject_message: A reject message
 604
 605         @return: 0
 606
 607         """
 608         # If we weren't given a manual rejection message, spawn an
 609         # editor so the user can add one in...
 610         if manual and not reject_message:
 611             (fd, temp_filename) = utils.temp_filename()
 612             temp_file = os.fdopen(fd, 'w')
 613             if len(note) > 0:
 614                 for line in note:
 615                     temp_file.write(line)
 616             temp_file.close()
 617             editor = os.environ.get("EDITOR","vi")
 618             answer = 'E'
 619             while answer == 'E':
 620                 os.system("%s %s" % (editor, temp_filename))
 621                 temp_fh = utils.open_file(temp_filename)
 622                 reject_message = "".join(temp_fh.readlines())
 623                 temp_fh.close()
 624                 print "Reject message:"
 625                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1)
 626                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 627                 answer = "XXX"
 628                 while prompt.find(answer) == -1:
 629                     answer = utils.our_raw_input(prompt)
 630                     m = re_default_answer.search(prompt)
 631                     if answer == "":
 632                         answer = m.group(1)
 633                     answer = answer[:1].upper()
 634             os.unlink(temp_filename)
 635             if answer == 'A':
 636                 return 1
 637             elif answer == 'Q':
 638                 sys.exit(0)
 639
 640         print "Rejecting.\n"
 641
 642         cnf = Config()
 643
 644         reason_filename = self.pkg.changes_file[:-8] + ".reason"
 645         reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
 646
 647         # Move all the files into the reject directory
 648         reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
 649         self.force_reject(reject_files)
 650
 651         # If we fail here someone is probably trying to exploit the race
 652         # so let's just raise an exception ...
 653         if os.path.exists(reason_filename):
 654             os.unlink(reason_filename)
 655         reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
 656
 657         rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
 658
 659         if not manual:
 660             self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 661             self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
 662             self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
 663             os.write(reason_fd, reject_message)
 664             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
 665         else:
 666             # Build up the rejection email
 667             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
 668             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
 669             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
 670             self.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 671             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
 672             # Write the rejection email out as the <foo>.reason file
 673             os.write(reason_fd, reject_mail_message)
 674
 675         del self.Subst["__REJECTOR_ADDRESS__"]
 676         del self.Subst["__MANUAL_REJECT_MESSAGE__"]
 677         del self.Subst["__CC__"]
 678
 679         os.close(reason_fd)
 680
 681         # Send the rejection mail if appropriate
 682         if not cnf["Dinstall::Options::No-Mail"]:
 683             utils.send_mail(reject_mail_message)
 684
 685         self.Logger.log(["rejected", pkg.changes_file])
 686
 687         return 0
 688
 689     ################################################################################
 690     def in_override_p(self, package, component, suite, binary_type, file, session=None):
 691         """
 692         Check if a package already has override entries in the DB
 693
 694         @type package: string
 695         @param package: package name
 696
 697         @type component: string
 698         @param component: database id of the component
 699
 700         @type suite: int
 701         @param suite: database id of the suite
 702
 703         @type binary_type: string
 704         @param binary_type: type of the package
 705
 706         @type file: string
 707         @param file: filename we check
 708
 709         @return: the database result. But noone cares anyway.
 710
 711         """
 712
 713         cnf = Config()
 714
 715         if session is None:
 716             session = DBConn().session()
 717
 718         if binary_type == "": # must be source
 719             file_type = "dsc"
 720         else:
 721             file_type = binary_type
 722
 723         # Override suite name; used for example with proposed-updates
 724         if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 725             suite = cnf["Suite::%s::OverrideSuite" % (suite)]
 726
 727         result = get_override(package, suite, component, file_type, session)
 728
 729         # If checking for a source package fall back on the binary override type
 730         if file_type == "dsc" and len(result) < 1:
 731             result = get_override(package, suite, component, ['deb', 'udeb'], session)
 732
 733         # Remember the section and priority so we can check them later if appropriate
 734         if len(result) > 0:
 735             result = result[0]
 736             self.pkg.files[file]["override section"] = result.section.section
 737             self.pkg.files[file]["override priority"] = result.priority.priority
 738             return result
 739
 740         return None
 741
 742     ################################################################################
 743     def reject (self, str, prefix="Rejected: "):
 744         """
 745         Add C{str} to reject_message. Adds C{prefix}, by default "Rejected: "
 746
 747         @type str: string
 748         @param str: Reject text
 749
 750         @type prefix: string
 751         @param prefix: Prefix text, default Rejected:
 752
 753         """
 754         if str:
 755             # Unlike other rejects we add new lines first to avoid trailing
 756             # new lines when this message is passed back up to a caller.
 757             if self.reject_message:
 758                 self.reject_message += "\n"
 759             self.reject_message += prefix + str
 760
 761     ################################################################################
 762     def get_anyversion(self, sv_list, suite):
 763         """
 764         @type sv_list: list
 765         @param sv_list: list of (suite, version) tuples to check
 766
 767         @type suite: string
 768         @param suite: suite name
 769
 770         Description: TODO
 771         """
 772         anyversion = None
 773         anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
 774         for (s, v) in sv_list:
 775             if s in [ x.lower() for x in anysuite ]:
 776                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
 777                     anyversion = v
 778
 779         return anyversion
 780
 781     ################################################################################
 782
 783     def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
 784         """
 785         @type sv_list: list
 786         @param sv_list: list of (suite, version) tuples to check
 787
 788         @type file: string
 789         @param file: XXX
 790
 791         @type new_version: string
 792         @param new_version: XXX
 793
 794         Ensure versions are newer than existing packages in target
 795         suites and that cross-suite version checking rules as
 796         set out in the conf file are satisfied.
 797         """
 798
 799         cnf = Config()
 800
 801         # Check versions for each target suite
 802         for target_suite in self.pkg.changes["distribution"].keys():
 803             must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
 804             must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
 805
 806             # Enforce "must be newer than target suite" even if conffile omits it
 807             if target_suite not in must_be_newer_than:
 808                 must_be_newer_than.append(target_suite)
 809
 810             for (suite, existent_version) in sv_list:
 811                 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
 812
 813                 if suite in must_be_newer_than and sourceful and vercmp < 1:
 814                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
 815
 816                 if suite in must_be_older_than and vercmp > -1:
 817                     cansave = 0
 818
 819                     if self.pkg.changes.get('distribution-version', {}).has_key(suite):
 820                         # we really use the other suite, ignoring the conflicting one ...
 821                         addsuite = self.pkg.changes["distribution-version"][suite]
 822
 823                         add_version = self.get_anyversion(sv_list, addsuite)
 824                         target_version = self.get_anyversion(sv_list, target_suite)
 825
 826                         if not add_version:
 827                             # not add_version can only happen if we map to a suite
 828                             # that doesn't enhance the suite we're propup'ing from.
 829                             # so "propup-ver x a b c; map a d" is a problem only if
 830                             # d doesn't enhance a.
 831                             #
 832                             # i think we could always propagate in this case, rather
 833                             # than complaining. either way, this isn't a REJECT issue
 834                             #
 835                             # And - we really should complain to the dorks who configured dak
 836                             self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ")
 837                             self.pkg.changes.setdefault("propdistribution", {})
 838                             self.pkg.changes["propdistribution"][addsuite] = 1
 839                             cansave = 1
 840                         elif not target_version:
 841                             # not targets_version is true when the package is NEW
 842                             # we could just stick with the "...old version..." REJECT
 843                             # for this, I think.
 844                             self.reject("Won't propogate NEW packages.")
 845                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
 846                             # propogation would be redundant. no need to reject though.
 847                             self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ")
 848                             cansave = 1
 849                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
 850                              apt_pkg.VersionCompare(add_version, target_version) >= 0:
 851                             # propogate!!
 852                             self.reject("Propogating upload to %s" % (addsuite), "Warning: ")
 853                             self.pkg.changes.setdefault("propdistribution", {})
 854                             self.pkg.changes["propdistribution"][addsuite] = 1
 855                             cansave = 1
 856
 857                     if not cansave:
 858                         self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
 859
 860     ################################################################################
 861
 862     def check_binary_against_db(self, file, session=None):
 863         """
 864
 865         """
 866
 867         if session is None:
 868             session = DBConn().session()
 869
 870         self.reject_message = ""
 871
 872         # Ensure version is sane
 873         q = session.query(BinAssociation)
 874         q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
 875         q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
 876
 877         self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
 878                                        file, files[file]["version"], sourceful=False)
 879
 880         # Check for any existing copies of the file
 881         q = session.query(DBBinary).filter_by(files[file]["package"])
 882         q = q.filter_by(version=files[file]["version"])
 883         q = q.join(Architecture).filter_by(arch_string=files[file]["architecture"])
 884
 885         if q.count() > 0:
 886             self.reject("%s: can not overwrite existing copy already in the archive." % (file))
 887
 888         return self.reject_message
 889
 890     ################################################################################
 891
 892     def check_source_against_db(self, file, session=None):
 893         """
 894         """
 895         if session is None:
 896             session = DBConn().session()
 897
 898         self.reject_message = ""
 899         source = self.pkg.dsc.get("source")
 900         version = self.pkg.dsc.get("version")
 901
 902         # Ensure version is sane
 903         q = session.query(SrcAssociation)
 904         q = q.join(DBSource).filter(DBSource.source==source)
 905
 906         self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
 907                                        file, version, sourceful=True)
 908
 909         return self.reject_message
 910
 911     ################################################################################
 912     def check_dsc_against_db(self, file):
 913         """
 914
 915         @warning: NB: this function can remove entries from the 'files' index [if
 916          the .orig.tar.gz is a duplicate of the one in the archive]; if
 917          you're iterating over 'files' and call this function as part of
 918          the loop, be sure to add a check to the top of the loop to
 919          ensure you haven't just tried to dereference the deleted entry.
 920
 921         """
 922         self.reject_message = ""
 923         self.pkg.orig_tar_gz = None
 924
 925         # Try and find all files mentioned in the .dsc.  This has
 926         # to work harder to cope with the multiple possible
 927         # locations of an .orig.tar.gz.
 928         # The ordering on the select is needed to pick the newest orig
 929         # when it exists in multiple places.
 930         for dsc_name, dsc_entry in self.pkg.dsc_files.items():
 931             found = None
 932             if self.pkg.files.has_key(dsc_name):
 933                 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
 934                 actual_size = int(self.pkg.files[dsc_name]["size"])
 935                 found = "%s in incoming" % (dsc_name)
 936
 937                 # Check the file does not already exist in the archive
 938                 ql = get_poolfile_like_name(dsc_name)
 939
 940                 # Strip out anything that isn't '%s' or '/%s$'
 941                 for i in ql:
 942                     if not i.filename.endswith(dsc_name):
 943                         ql.remove(i)
 944
 945                 # "[dak] has not broken them.  [dak] has fixed a
 946                 # brokenness.  Your crappy hack exploited a bug in
 947                 # the old dinstall.
 948                 #
 949                 # "(Come on!  I thought it was always obvious that
 950                 # one just doesn't release different files with
 951                 # the same name and version.)"
 952                 #                        -- ajk@ on d-devel@l.d.o
 953
 954                 if len(ql) > 0:
 955                     # Ignore exact matches for .orig.tar.gz
 956                     match = 0
 957                     if dsc_name.endswith(".orig.tar.gz"):
 958                         for i in ql:
 959                             if self.pkg.files.has_key(dsc_name) and \
 960                                int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
 961                                self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
 962                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_name), "Warning: ")
 963                                 # TODO: Don't delete the entry, just mark it as not needed
 964                                 # This would fix the stupidity of changing something we often iterate over
 965                                 # whilst we're doing it
 966                                 del files[dsc_name]
 967                                 self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename)
 968                                 match = 1
 969
 970                     if not match:
 971                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
 972
 973             elif dsc_name.endswith(".orig.tar.gz"):
 974                 # Check in the pool
 975                 ql = get_poolfile_like_name(dsc_name)
 976
 977                 # Strip out anything that isn't '%s' or '/%s$'
 978                 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
 979                 for i in ql:
 980                     if not i.filename.endswith(dsc_name):
 981                         ql.remove(i)
 982
 983                 if len(ql) > 0:
 984                     # Unfortunately, we may get more than one match here if,
 985                     # for example, the package was in potato but had an -sa
 986                     # upload in woody.  So we need to choose the right one.
 987
 988                     # default to something sane in case we don't match any or have only one
 989                     x = ql[0]
 990
 991                     if len(ql) > 1:
 992                         for i in ql:
 993                             old_file = os.path.join(i.location.path, i.filename)
 994                             old_file_fh = utils.open_file(old_file)
 995                             actual_md5 = apt_pkg.md5sum(old_file_fh)
 996                             old_file_fh.close()
 997                             actual_size = os.stat(old_file)[stat.ST_SIZE]
 998                             if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
 999                                 x = i
1000
1001                     old_file = os.path.join(i.location.path, i.filename)
1002                     old_file_fh = utils.open_file(old_file)
1003                     actual_md5 = apt_pkg.md5sum(old_file_fh)
1004                     old_file_fh.close()
1005                     actual_size = os.stat(old_file)[stat.ST_SIZE]
1006                     found = old_file
1007                     suite_type = f.location.archive_type
1008                     # need this for updating dsc_files in install()
1009                     dsc_entry["files id"] = f.file_id
1010                     # See install() in process-accepted...
1011                     self.pkg.orig_tar_id = f.file_id
1012                     self.pkg.orig_tar_gz = old_file
1013                     self.pkg.orig_tar_location = f.location.location_id
1014                 else:
1015                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
1016                     # Not there? Check the queue directories...
1017                     for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
1018                         in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (directory)], dsc_name)
1019                         if os.path.exists(in_otherdir):
1020                             in_otherdir_fh = utils.open_file(in_otherdir)
1021                             actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
1022                             in_otherdir_fh.close()
1023                             actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
1024                             found = in_otherdir
1025                             self.pkg.orig_tar_gz = in_otherdir
1026
1027                     if not found:
1028                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
1029                         self.pkg.orig_tar_gz = -1
1030                         continue
1031             else:
1032                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
1033                 continue
1034             if actual_md5 != dsc_entry["md5sum"]:
1035                 self.reject("md5sum for %s doesn't match %s." % (found, file))
1036             if actual_size != int(dsc_entry["size"]):
1037                 self.reject("size for %s doesn't match %s." % (found, file))
1038
1039         return (self.reject_message, None)