daklib/queue.py

   1 #!/usr/bin/env python
   2 # vim:set et sw=4:
   3
   4 """
   5 Queue utility functions for dak
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
   9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ###############################################################################
  28
  29 import cPickle
  30 import errno
  31 import os
  32 import pg
  33 import stat
  34 import sys
  35 import time
  36 import apt_inst
  37 import apt_pkg
  38 import utils
  39 import commands
  40 import shutil
  41 import textwrap
  42 from types import *
  43
  44 import yaml
  45
  46 from dak_exceptions import *
  47 from changes import *
  48 from regexes import *
  49 from config import Config
  50 from holding import Holding
  51 from dbconn import *
  52 from summarystats import SummaryStats
  53 from utils import parse_changes
  54 from textutils import fix_maintainer
  55 from binary import Binary
  56
  57 ###############################################################################
  58
  59 def get_type(f, session):
  60     """
  61     Get the file type of C{f}
  62
  63     @type f: dict
  64     @param f: file entry from Changes object
  65
  66     @type session: SQLA Session
  67     @param session: SQL Alchemy session object
  68
  69     @rtype: string
  70     @return: filetype
  71
  72     """
  73     # Determine the type
  74     if f.has_key("dbtype"):
  75         file_type = f["dbtype"]
  76     elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
  77         file_type = "dsc"
  78     else:
  79         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))
  80
  81     # Validate the override type
  82     type_id = get_override_type(file_type, session)
  83     if type_id is None:
  84         utils.fubar("invalid type (%s) for new.  Say wha?" % (file_type))
  85
  86     return file_type
  87
  88 ################################################################################
  89
  90 # Determine what parts in a .changes are NEW
  91
  92 def determine_new(changes, files, warn=1):
  93     """
  94     Determine what parts in a C{changes} file are NEW.
  95
  96     @type changes: Upload.Pkg.changes dict
  97     @param changes: Changes dictionary
  98
  99     @type files: Upload.Pkg.files dict
 100     @param files: Files dictionary
 101
 102     @type warn: bool
 103     @param warn: Warn if overrides are added for (old)stable
 104
 105     @rtype: dict
 106     @return: dictionary of NEW components.
 107
 108     """
 109     new = {}
 110
 111     session = DBConn().session()
 112
 113     # Build up a list of potentially new things
 114     for name, f in files.items():
 115         # Skip byhand elements
 116         if f["type"] == "byhand":
 117             continue
 118         pkg = f["package"]
 119         priority = f["priority"]
 120         section = f["section"]
 121         file_type = get_type(f, session)
 122         component = f["component"]
 123
 124         if file_type == "dsc":
 125             priority = "source"
 126
 127         if not new.has_key(pkg):
 128             new[pkg] = {}
 129             new[pkg]["priority"] = priority
 130             new[pkg]["section"] = section
 131             new[pkg]["type"] = file_type
 132             new[pkg]["component"] = component
 133             new[pkg]["files"] = []
 134         else:
 135             old_type = new[pkg]["type"]
 136             if old_type != file_type:
 137                 # source gets trumped by deb or udeb
 138                 if old_type == "dsc":
 139                     new[pkg]["priority"] = priority
 140                     new[pkg]["section"] = section
 141                     new[pkg]["type"] = file_type
 142                     new[pkg]["component"] = component
 143
 144         new[pkg]["files"].append(name)
 145
 146         if f.has_key("othercomponents"):
 147             new[pkg]["othercomponents"] = f["othercomponents"]
 148
 149     for suite in changes["suite"].keys():
 150         for pkg in new.keys():
 151             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
 152             if len(ql) > 0:
 153                 for file_entry in new[pkg]["files"]:
 154                     if files[file_entry].has_key("new"):
 155                         del files[file_entry]["new"]
 156                 del new[pkg]
 157
 158     if warn:
 159         for s in ['stable', 'oldstable']:
 160             if changes["suite"].has_key(s):
 161                 print "WARNING: overrides will be added for %s!" % s
 162         for pkg in new.keys():
 163             if new[pkg].has_key("othercomponents"):
 164                 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 165
 166     session.close()
 167
 168     return new
 169
 170 ################################################################################
 171
 172 def check_valid(new):
 173     """
 174     Check if section and priority for NEW packages exist in database.
 175     Additionally does sanity checks:
 176       - debian-installer packages have to be udeb (or source)
 177       - non debian-installer packages can not be udeb
 178       - source priority can only be assigned to dsc file types
 179
 180     @type new: dict
 181     @param new: Dict of new packages with their section, priority and type.
 182
 183     """
 184     for pkg in new.keys():
 185         section_name = new[pkg]["section"]
 186         priority_name = new[pkg]["priority"]
 187         file_type = new[pkg]["type"]
 188
 189         section = get_section(section_name)
 190         if section is None:
 191             new[pkg]["section id"] = -1
 192         else:
 193             new[pkg]["section id"] = section.section_id
 194
 195         priority = get_priority(priority_name)
 196         if priority is None:
 197             new[pkg]["priority id"] = -1
 198         else:
 199             new[pkg]["priority id"] = priority.priority_id
 200
 201         # Sanity checks
 202         di = section_name.find("debian-installer") != -1
 203
 204         # If d-i, we must be udeb and vice-versa
 205         if     (di and file_type not in ("udeb", "dsc")) or \
 206            (not di and file_type == "udeb"):
 207             new[pkg]["section id"] = -1
 208
 209         # If dsc we need to be source and vice-versa
 210         if (priority == "source" and file_type != "dsc") or \
 211            (priority != "source" and file_type == "dsc"):
 212             new[pkg]["priority id"] = -1
 213
 214 ###############################################################################
 215
 216 def lookup_uid_from_fingerprint(fpr, session):
 217     uid = None
 218     uid_name = ""
 219     # This is a stupid default, but see the comments below
 220     is_dm = False
 221
 222     user = get_uid_from_fingerprint(fpr, session)
 223
 224     if user is not None:
 225         uid = user.uid
 226         if user.name is None:
 227             uid_name = ''
 228         else:
 229             uid_name = user.name
 230
 231         # Check the relevant fingerprint (which we have to have)
 232         for f in user.fingerprint:
 233             if f.fingerprint == fpr:
 234                 is_dm = f.keyring.debian_maintainer
 235                 break
 236
 237     return (uid, uid_name, is_dm)
 238
 239 ###############################################################################
 240
 241 def check_status(files):
 242     new = byhand = 0
 243     for f in files.keys():
 244         if files[f]["type"] == "byhand":
 245             byhand = 1
 246         elif files[f].has_key("new"):
 247             new = 1
 248     return (new, byhand)
 249
 250 ###############################################################################
 251
 252 # Used by Upload.check_timestamps
 253 class TarTime(object):
 254     def __init__(self, future_cutoff, past_cutoff):
 255         self.reset()
 256         self.future_cutoff = future_cutoff
 257         self.past_cutoff = past_cutoff
 258
 259     def reset(self):
 260         self.future_files = {}
 261         self.ancient_files = {}
 262
 263     def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
 264         if MTime > self.future_cutoff:
 265             self.future_files[Name] = MTime
 266         if MTime < self.past_cutoff:
 267             self.ancient_files[Name] = MTime
 268
 269 ###############################################################################
 270
 271 class Upload(object):
 272     """
 273     Everything that has to do with an upload processed.
 274
 275     """
 276     def __init__(self):
 277         self.logger = None
 278         self.pkg = Changes()
 279         self.reset()
 280
 281     ###########################################################################
 282
 283     def reset (self):
 284         """ Reset a number of internal variables."""
 285
 286         # Initialize the substitution template map
 287         cnf = Config()
 288         self.Subst = {}
 289         self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
 290         self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
 291         self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
 292         self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 293
 294         self.rejects = []
 295         self.warnings = []
 296         self.notes = []
 297
 298         self.pkg.reset()
 299
 300     def package_info(self):
 301         msg = ''
 302
 303         if len(self.rejects) > 0:
 304             msg += "Reject Reasons:\n"
 305             msg += "\n".join(self.rejects)
 306
 307         if len(self.warnings) > 0:
 308             msg += "Warnings:\n"
 309             msg += "\n".join(self.warnings)
 310
 311         if len(self.notes) > 0:
 312             msg += "Notes:\n"
 313             msg += "\n".join(self.notes)
 314
 315         return msg
 316
 317     ###########################################################################
 318     def update_subst(self):
 319         """ Set up the per-package template substitution mappings """
 320
 321         cnf = Config()
 322
 323         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
 324         if not self.pkg.changes.has_key("architecture") or not \
 325            isinstance(self.pkg.changes["architecture"], DictType):
 326             self.pkg.changes["architecture"] = { "Unknown" : "" }
 327
 328         # and maintainer2047 may not exist.
 329         if not self.pkg.changes.has_key("maintainer2047"):
 330             self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
 331
 332         self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
 333         self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
 334         self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
 335
 336         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 337         if self.pkg.changes["architecture"].has_key("source") and \
 338            self.pkg.changes["changedby822"] != "" and \
 339            (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
 340
 341             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
 342             self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
 343             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
 344         else:
 345             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
 346             self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
 347             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
 348
 349         if "sponsoremail" in self.pkg.changes:
 350             self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
 351
 352         if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
 353             self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
 354
 355         # Apply any global override of the Maintainer field
 356         if cnf.get("Dinstall::OverrideMaintainer"):
 357             self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
 358             self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
 359
 360         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
 361         self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
 362         self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
 363
 364     ###########################################################################
 365     def load_changes(self, filename):
 366         """
 367         @rtype: boolean
 368         @rvalue: whether the changes file was valid or not.  We may want to
 369                  reject even if this is True (see what gets put in self.rejects).
 370                  This is simply to prevent us even trying things later which will
 371                  fail because we couldn't properly parse the file.
 372         """
 373         Cnf = Config()
 374         self.pkg.changes_file = filename
 375
 376         # Parse the .changes field into a dictionary
 377         try:
 378             self.pkg.changes.update(parse_changes(filename))
 379         except CantOpenError:
 380             self.rejects.append("%s: can't read file." % (filename))
 381             return False
 382         except ParseChangesError, line:
 383             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 384             return False
 385         except ChangesUnicodeError:
 386             self.rejects.append("%s: changes file not proper utf-8" % (filename))
 387             return False
 388
 389         # Parse the Files field from the .changes into another dictionary
 390         try:
 391             self.pkg.files.update(utils.build_file_list(self.pkg.changes))
 392         except ParseChangesError, line:
 393             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 394             return False
 395         except UnknownFormatError, format:
 396             self.rejects.append("%s: unknown format '%s'." % (filename, format))
 397             return False
 398
 399         # Check for mandatory fields
 400         for i in ("distribution", "source", "binary", "architecture",
 401                   "version", "maintainer", "files", "changes", "description"):
 402             if not self.pkg.changes.has_key(i):
 403                 # Avoid undefined errors later
 404                 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
 405                 return False
 406
 407         # Strip a source version in brackets from the source field
 408         if re_strip_srcver.search(self.pkg.changes["source"]):
 409             self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
 410
 411         # Ensure the source field is a valid package name.
 412         if not re_valid_pkg_name.match(self.pkg.changes["source"]):
 413             self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
 414
 415         # Split multi-value fields into a lower-level dictionary
 416         for i in ("architecture", "distribution", "binary", "closes"):
 417             o = self.pkg.changes.get(i, "")
 418             if o != "":
 419                 del self.pkg.changes[i]
 420
 421             self.pkg.changes[i] = {}
 422
 423             for j in o.split():
 424                 self.pkg.changes[i][j] = 1
 425
 426         # Fix the Maintainer: field to be RFC822/2047 compatible
 427         try:
 428             (self.pkg.changes["maintainer822"],
 429              self.pkg.changes["maintainer2047"],
 430              self.pkg.changes["maintainername"],
 431              self.pkg.changes["maintaineremail"]) = \
 432                    fix_maintainer (self.pkg.changes["maintainer"])
 433         except ParseMaintError, msg:
 434             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
 435                    % (filename, changes["maintainer"], msg))
 436
 437         # ...likewise for the Changed-By: field if it exists.
 438         try:
 439             (self.pkg.changes["changedby822"],
 440              self.pkg.changes["changedby2047"],
 441              self.pkg.changes["changedbyname"],
 442              self.pkg.changes["changedbyemail"]) = \
 443                    fix_maintainer (self.pkg.changes.get("changed-by", ""))
 444         except ParseMaintError, msg:
 445             self.pkg.changes["changedby822"] = ""
 446             self.pkg.changes["changedby2047"] = ""
 447             self.pkg.changes["changedbyname"] = ""
 448             self.pkg.changes["changedbyemail"] = ""
 449
 450             self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
 451                    % (filename, changes["changed-by"], msg))
 452
 453         # Ensure all the values in Closes: are numbers
 454         if self.pkg.changes.has_key("closes"):
 455             for i in self.pkg.changes["closes"].keys():
 456                 if re_isanum.match (i) == None:
 457                     self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
 458
 459         # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
 460         self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
 461         self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
 462
 463         # Check there isn't already a changes file of the same name in one
 464         # of the queue directories.
 465         base_filename = os.path.basename(filename)
 466         for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
 467             if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
 468                 self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
 469
 470         # Check the .changes is non-empty
 471         if not self.pkg.files:
 472             self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
 473             return False
 474
 475         # Changes was syntactically valid even if we'll reject
 476         return True
 477
 478     ###########################################################################
 479
 480     def check_distributions(self):
 481         "Check and map the Distribution field"
 482
 483         Cnf = Config()
 484
 485         # Handle suite mappings
 486         for m in Cnf.ValueList("SuiteMappings"):
 487             args = m.split()
 488             mtype = args[0]
 489             if mtype == "map" or mtype == "silent-map":
 490                 (source, dest) = args[1:3]
 491                 if self.pkg.changes["distribution"].has_key(source):
 492                     del self.pkg.changes["distribution"][source]
 493                     self.pkg.changes["distribution"][dest] = 1
 494                     if mtype != "silent-map":
 495                         self.notes.append("Mapping %s to %s." % (source, dest))
 496                 if self.pkg.changes.has_key("distribution-version"):
 497                     if self.pkg.changes["distribution-version"].has_key(source):
 498                         self.pkg.changes["distribution-version"][source]=dest
 499             elif mtype == "map-unreleased":
 500                 (source, dest) = args[1:3]
 501                 if self.pkg.changes["distribution"].has_key(source):
 502                     for arch in self.pkg.changes["architecture"].keys():
 503                         if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
 504                             self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
 505                             del self.pkg.changes["distribution"][source]
 506                             self.pkg.changes["distribution"][dest] = 1
 507                             break
 508             elif mtype == "ignore":
 509                 suite = args[1]
 510                 if self.pkg.changes["distribution"].has_key(suite):
 511                     del self.pkg.changes["distribution"][suite]
 512                     self.warnings.append("Ignoring %s as a target suite." % (suite))
 513             elif mtype == "reject":
 514                 suite = args[1]
 515                 if self.pkg.changes["distribution"].has_key(suite):
 516                     self.rejects.append("Uploads to %s are not accepted." % (suite))
 517             elif mtype == "propup-version":
 518                 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
 519                 #
 520                 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
 521                 if self.pkg.changes["distribution"].has_key(args[1]):
 522                     self.pkg.changes.setdefault("distribution-version", {})
 523                     for suite in args[2:]:
 524                         self.pkg.changes["distribution-version"][suite] = suite
 525
 526         # Ensure there is (still) a target distribution
 527         if len(self.pkg.changes["distribution"].keys()) < 1:
 528             self.rejects.append("No valid distribution remaining.")
 529
 530         # Ensure target distributions exist
 531         for suite in self.pkg.changes["distribution"].keys():
 532             if not Cnf.has_key("Suite::%s" % (suite)):
 533                 self.rejects.append("Unknown distribution `%s'." % (suite))
 534
 535     ###########################################################################
 536
 537     def binary_file_checks(self, f, session):
 538         cnf = Config()
 539         entry = self.pkg.files[f]
 540
 541         # Extract package control information
 542         deb_file = utils.open_file(f)
 543         try:
 544             control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
 545         except:
 546             self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
 547             deb_file.close()
 548             # Can't continue, none of the checks on control would work.
 549             return
 550
 551         # Check for mandantory "Description:"
 552         deb_file.seek(0)
 553         try:
 554             apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
 555         except:
 556             self.rejects.append("%s: Missing Description in binary package" % (f))
 557             return
 558
 559         deb_file.close()
 560
 561         # Check for mandatory fields
 562         for field in [ "Package", "Architecture", "Version" ]:
 563             if control.Find(field) == None:
 564                 # Can't continue
 565                 self.rejects.append("%s: No %s field in control." % (f, field))
 566                 return
 567
 568         # Ensure the package name matches the one give in the .changes
 569         if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
 570             self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
 571
 572         # Validate the package field
 573         package = control.Find("Package")
 574         if not re_valid_pkg_name.match(package):
 575             self.rejects.append("%s: invalid package name '%s'." % (f, package))
 576
 577         # Validate the version field
 578         version = control.Find("Version")
 579         if not re_valid_version.match(version):
 580             self.rejects.append("%s: invalid version number '%s'." % (f, version))
 581
 582         # Ensure the architecture of the .deb is one we know about.
 583         default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
 584         architecture = control.Find("Architecture")
 585         upload_suite = self.pkg.changes["distribution"].keys()[0]
 586
 587         if      architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
 588             and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
 589             self.rejects.append("Unknown architecture '%s'." % (architecture))
 590
 591         # Ensure the architecture of the .deb is one of the ones
 592         # listed in the .changes.
 593         if not self.pkg.changes["architecture"].has_key(architecture):
 594             self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
 595
 596         # Sanity-check the Depends field
 597         depends = control.Find("Depends")
 598         if depends == '':
 599             self.rejects.append("%s: Depends field is empty." % (f))
 600
 601         # Sanity-check the Provides field
 602         provides = control.Find("Provides")
 603         if provides:
 604             provide = re_spacestrip.sub('', provides)
 605             if provide == '':
 606                 self.rejects.append("%s: Provides field is empty." % (f))
 607             prov_list = provide.split(",")
 608             for prov in prov_list:
 609                 if not re_valid_pkg_name.match(prov):
 610                     self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
 611
 612         # Check the section & priority match those given in the .changes (non-fatal)
 613         if     control.Find("Section") and entry["section"] != "" \
 614            and entry["section"] != control.Find("Section"):
 615             self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
 616                                 (f, control.Find("Section", ""), entry["section"]))
 617         if control.Find("Priority") and entry["priority"] != "" \
 618            and entry["priority"] != control.Find("Priority"):
 619             self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
 620                                 (f, control.Find("Priority", ""), entry["priority"]))
 621
 622         entry["package"] = package
 623         entry["architecture"] = architecture
 624         entry["version"] = version
 625         entry["maintainer"] = control.Find("Maintainer", "")
 626
 627         if f.endswith(".udeb"):
 628             self.pkg.files[f]["dbtype"] = "udeb"
 629         elif f.endswith(".deb"):
 630             self.pkg.files[f]["dbtype"] = "deb"
 631         else:
 632             self.rejects.append("%s is neither a .deb or a .udeb." % (f))
 633
 634         entry["source"] = control.Find("Source", entry["package"])
 635
 636         # Get the source version
 637         source = entry["source"]
 638         source_version = ""
 639
 640         if source.find("(") != -1:
 641             m = re_extract_src_version.match(source)
 642             source = m.group(1)
 643             source_version = m.group(2)
 644
 645         if not source_version:
 646             source_version = self.pkg.files[f]["version"]
 647
 648         entry["source package"] = source
 649         entry["source version"] = source_version
 650
 651         # Ensure the filename matches the contents of the .deb
 652         m = re_isadeb.match(f)
 653
 654         #  package name
 655         file_package = m.group(1)
 656         if entry["package"] != file_package:
 657             self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
 658                                 (f, file_package, entry["dbtype"], entry["package"]))
 659         epochless_version = re_no_epoch.sub('', control.Find("Version"))
 660
 661         #  version
 662         file_version = m.group(2)
 663         if epochless_version != file_version:
 664             self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
 665                                 (f, file_version, entry["dbtype"], epochless_version))
 666
 667         #  architecture
 668         file_architecture = m.group(3)
 669         if entry["architecture"] != file_architecture:
 670             self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
 671                                 (f, file_architecture, entry["dbtype"], entry["architecture"]))
 672
 673         # Check for existent source
 674         source_version = entry["source version"]
 675         source_package = entry["source package"]
 676         if self.pkg.changes["architecture"].has_key("source"):
 677             if source_version != self.pkg.changes["version"]:
 678                 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
 679                                     (source_version, f, self.pkg.changes["version"]))
 680         else:
 681             # Check in the SQL database
 682             if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
 683                 # Check in one of the other directories
 684                 source_epochless_version = re_no_epoch.sub('', source_version)
 685                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 686                 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
 687                     entry["byhand"] = 1
 688                 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
 689                     entry["new"] = 1
 690                 else:
 691                     dsc_file_exists = False
 692                     for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
 693                         if cnf.has_key("Dir::Queue::%s" % (myq)):
 694                             if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
 695                                 dsc_file_exists = True
 696                                 break
 697
 698                     if not dsc_file_exists:
 699                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
 700
 701         # Check the version and for file overwrites
 702         self.check_binary_against_db(f, session)
 703
 704         # Temporarily disable contents generation until we change the table storage layout
 705         #b = Binary(f)
 706         #b.scan_package()
 707         #if len(b.rejects) > 0:
 708         #    for j in b.rejects:
 709         #        self.rejects.append(j)
 710
 711     def source_file_checks(self, f, session):
 712         entry = self.pkg.files[f]
 713
 714         m = re_issource.match(f)
 715         if not m:
 716             return
 717
 718         entry["package"] = m.group(1)
 719         entry["version"] = m.group(2)
 720         entry["type"] = m.group(3)
 721
 722         # Ensure the source package name matches the Source filed in the .changes
 723         if self.pkg.changes["source"] != entry["package"]:
 724             self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
 725
 726         # Ensure the source version matches the version in the .changes file
 727         if entry["type"] == "orig.tar.gz":
 728             changes_version = self.pkg.changes["chopversion2"]
 729         else:
 730             changes_version = self.pkg.changes["chopversion"]
 731
 732         if changes_version != entry["version"]:
 733             self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
 734
 735         # Ensure the .changes lists source in the Architecture field
 736         if not self.pkg.changes["architecture"].has_key("source"):
 737             self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
 738
 739         # Check the signature of a .dsc file
 740         if entry["type"] == "dsc":
 741             # check_signature returns either:
 742             #  (None, [list, of, rejects]) or (signature, [])
 743             (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
 744             for j in rejects:
 745                 self.rejects.append(j)
 746
 747         entry["architecture"] = "source"
 748
 749     def per_suite_file_checks(self, f, suite, session):
 750         cnf = Config()
 751         entry = self.pkg.files[f]
 752         archive = utils.where_am_i()
 753
 754         # Skip byhand
 755         if entry.has_key("byhand"):
 756             return
 757
 758         # Check we have fields we need to do these checks
 759         oktogo = True
 760         for m in ['component', 'package', 'priority', 'size', 'md5sum']:
 761             if not entry.has_key(m):
 762                 self.rejects.append("file '%s' does not have field %s set" % (f, m))
 763                 oktogo = False
 764
 765         if not oktogo:
 766             return
 767
 768         # Handle component mappings
 769         for m in cnf.ValueList("ComponentMappings"):
 770             (source, dest) = m.split()
 771             if entry["component"] == source:
 772                 entry["original component"] = source
 773                 entry["component"] = dest
 774
 775         # Ensure the component is valid for the target suite
 776         if cnf.has_key("Suite:%s::Components" % (suite)) and \
 777            entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
 778             self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
 779             return
 780
 781         # Validate the component
 782         if not get_component(entry["component"], session):
 783             self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
 784             return
 785
 786         # See if the package is NEW
 787         if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
 788             entry["new"] = 1
 789
 790         # Validate the priority
 791         if entry["priority"].find('/') != -1:
 792             self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
 793
 794         # Determine the location
 795         location = cnf["Dir::Pool"]
 796         l = get_location(location, entry["component"], archive, session)
 797         if l is None:
 798             self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
 799             entry["location id"] = -1
 800         else:
 801             entry["location id"] = l.location_id
 802
 803         # Check the md5sum & size against existing files (if any)
 804         entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
 805
 806         found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
 807                                          entry["size"], entry["md5sum"], entry["location id"])
 808
 809         if found is None:
 810             self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
 811         elif found is False and poolfile is not None:
 812             self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
 813         else:
 814             if poolfile is None:
 815                 entry["files id"] = None
 816             else:
 817                 entry["files id"] = poolfile.file_id
 818
 819         # Check for packages that have moved from one component to another
 820         entry['suite'] = suite
 821         res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
 822         if res.rowcount > 0:
 823             entry["othercomponents"] = res.fetchone()[0]
 824
 825     def check_files(self, action=True):
 826         archive = utils.where_am_i()
 827         file_keys = self.pkg.files.keys()
 828         holding = Holding()
 829         cnf = Config()
 830
 831         # XXX: As far as I can tell, this can no longer happen - see
 832         #      comments by AJ in old revisions - mhy
 833         # if reprocess is 2 we've already done this and we're checking
 834         # things again for the new .orig.tar.gz.
 835         # [Yes, I'm fully aware of how disgusting this is]
 836         if action and self.reprocess < 2:
 837             cwd = os.getcwd()
 838             os.chdir(self.pkg.directory)
 839             for f in file_keys:
 840                 ret = holding.copy_to_holding(f)
 841                 if ret is not None:
 842                     # XXX: Should we bail out here or try and continue?
 843                     self.rejects.append(ret)
 844
 845             os.chdir(cwd)
 846
 847         # Check there isn't already a .changes or .dak file of the same name in
 848         # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
 849         # [NB: this check must be done post-suite mapping]
 850         base_filename = os.path.basename(self.pkg.changes_file)
 851         dot_dak_filename = base_filename[:-8] + ".dak"
 852
 853         for suite in self.pkg.changes["distribution"].keys():
 854             copychanges = "Suite::%s::CopyChanges" % (suite)
 855             if cnf.has_key(copychanges) and \
 856                    os.path.exists(os.path.join(cnf[copychanges], base_filename)):
 857                 self.rejects.append("%s: a file with this name already exists in %s" \
 858                            % (base_filename, cnf[copychanges]))
 859
 860             copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
 861             if cnf.has_key(copy_dot_dak) and \
 862                    os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
 863                 self.rejects.append("%s: a file with this name already exists in %s" \
 864                            % (dot_dak_filename, Cnf[copy_dot_dak]))
 865
 866         self.reprocess = 0
 867         has_binaries = False
 868         has_source = False
 869
 870         session = DBConn().session()
 871
 872         for f, entry in self.pkg.files.items():
 873             # Ensure the file does not already exist in one of the accepted directories
 874             for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
 875                 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
 876                 if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
 877                     self.rejects.append("%s file already exists in the %s directory." % (f, d))
 878
 879             if not re_taint_free.match(f):
 880                 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
 881
 882             # Check the file is readable
 883             if os.access(f, os.R_OK) == 0:
 884                 # When running in -n, copy_to_holding() won't have
 885                 # generated the reject_message, so we need to.
 886                 if action:
 887                     if os.path.exists(f):
 888                         self.rejects.append("Can't read `%s'. [permission denied]" % (f))
 889                     else:
 890                         self.rejects.append("Can't read `%s'. [file not found]" % (f))
 891                 entry["type"] = "unreadable"
 892                 continue
 893
 894             # If it's byhand skip remaining checks
 895             if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
 896                 entry["byhand"] = 1
 897                 entry["type"] = "byhand"
 898
 899             # Checks for a binary package...
 900             elif re_isadeb.match(f):
 901                 has_binaries = True
 902                 entry["type"] = "deb"
 903
 904                 # This routine appends to self.rejects/warnings as appropriate
 905                 self.binary_file_checks(f, session)
 906
 907             # Checks for a source package...
 908             elif re_issource.match(f):
 909                 has_source = True
 910
 911                 # This routine appends to self.rejects/warnings as appropriate
 912                 self.source_file_checks(f, session)
 913
 914             # Not a binary or source package?  Assume byhand...
 915             else:
 916                 entry["byhand"] = 1
 917                 entry["type"] = "byhand"
 918
 919             # Per-suite file checks
 920             entry["oldfiles"] = {}
 921             for suite in self.pkg.changes["distribution"].keys():
 922                 self.per_suite_file_checks(f, suite, session)
 923
 924         session.close()
 925
 926         # If the .changes file says it has source, it must have source.
 927         if self.pkg.changes["architecture"].has_key("source"):
 928             if not has_source:
 929                 self.rejects.append("no source found and Architecture line in changes mention source.")
 930
 931             if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
 932                 self.rejects.append("source only uploads are not supported.")
 933
 934     ###########################################################################
 935     def check_dsc(self, action=True):
 936         """Returns bool indicating whether or not the source changes are valid"""
 937         # Ensure there is source to check
 938         if not self.pkg.changes["architecture"].has_key("source"):
 939             return True
 940
 941         # Find the .dsc
 942         dsc_filename = None
 943         for f, entry in self.pkg.files.items():
 944             if entry["type"] == "dsc":
 945                 if dsc_filename:
 946                     self.rejects.append("can not process a .changes file with multiple .dsc's.")
 947                     return False
 948                 else:
 949                     dsc_filename = f
 950
 951         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
 952         if not dsc_filename:
 953             self.rejects.append("source uploads must contain a dsc file")
 954             return False
 955
 956         # Parse the .dsc file
 957         try:
 958             self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
 959         except CantOpenError:
 960             # if not -n copy_to_holding() will have done this for us...
 961             if not action:
 962                 self.rejects.append("%s: can't read file." % (dsc_filename))
 963         except ParseChangesError, line:
 964             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
 965         except InvalidDscError, line:
 966             self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
 967         except ChangesUnicodeError:
 968             self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
 969
 970         # Build up the file list of files mentioned by the .dsc
 971         try:
 972             self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
 973         except NoFilesFieldError:
 974             self.rejects.append("%s: no Files: field." % (dsc_filename))
 975             return False
 976         except UnknownFormatError, format:
 977             self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
 978             return False
 979         except ParseChangesError, line:
 980             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
 981             return False
 982
 983         # Enforce mandatory fields
 984         for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
 985             if not self.pkg.dsc.has_key(i):
 986                 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
 987                 return False
 988
 989         # Validate the source and version fields
 990         if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
 991             self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
 992         if not re_valid_version.match(self.pkg.dsc["version"]):
 993             self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
 994
 995         # Bumping the version number of the .dsc breaks extraction by stable's
 996         # dpkg-source.  So let's not do that...
 997         if self.pkg.dsc["format"] != "1.0":
 998             self.rejects.append("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename))
 999
1000         # Validate the Maintainer field
1001         try:
1002             # We ignore the return value
1003             fix_maintainer(self.pkg.dsc["maintainer"])
1004         except ParseMaintError, msg:
1005             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
1006                                  % (dsc_filename, self.pkg.dsc["maintainer"], msg))
1007
1008         # Validate the build-depends field(s)
1009         for field_name in [ "build-depends", "build-depends-indep" ]:
1010             field = self.pkg.dsc.get(field_name)
1011             if field:
1012                 # Check for broken dpkg-dev lossage...
1013                 if field.startswith("ARRAY"):
1014                     self.rejects.append("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % \
1015                                         (dsc_filename, field_name.title()))
1016
1017                 # Have apt try to parse them...
1018                 try:
1019                     apt_pkg.ParseSrcDepends(field)
1020                 except:
1021                     self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
1022
1023         # Ensure the version number in the .dsc matches the version number in the .changes
1024         epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1025         changes_version = self.pkg.files[dsc_filename]["version"]
1026
1027         if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1028             self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1029
1030         # Ensure there is a .tar.gz in the .dsc file
1031         has_tar = False
1032         for f in self.pkg.dsc_files.keys():
1033             m = re_issource.match(f)
1034             if not m:
1035                 self.rejects.append("%s: %s in Files field not recognised as source." % (dsc_filename, f))
1036                 continue
1037             ftype = m.group(3)
1038             if ftype == "orig.tar.gz" or ftype == "tar.gz":
1039                 has_tar = True
1040
1041         if not has_tar:
1042             self.rejects.append("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
1043
1044         # Ensure source is newer than existing source in target suites
1045         session = DBConn().session()
1046         self.check_source_against_db(dsc_filename, session)
1047         self.check_dsc_against_db(dsc_filename, session)
1048         session.close()
1049
1050         return True
1051
1052     ###########################################################################
1053
1054     def get_changelog_versions(self, source_dir):
1055         """Extracts a the source package and (optionally) grabs the
1056         version history out of debian/changelog for the BTS."""
1057
1058         cnf = Config()
1059
1060         # Find the .dsc (again)
1061         dsc_filename = None
1062         for f in self.pkg.files.keys():
1063             if self.pkg.files[f]["type"] == "dsc":
1064                 dsc_filename = f
1065
1066         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1067         if not dsc_filename:
1068             return
1069
1070         # Create a symlink mirror of the source files in our temporary directory
1071         for f in self.pkg.files.keys():
1072             m = re_issource.match(f)
1073             if m:
1074                 src = os.path.join(source_dir, f)
1075                 # If a file is missing for whatever reason, give up.
1076                 if not os.path.exists(src):
1077                     return
1078                 ftype = m.group(3)
1079                 if ftype == "orig.tar.gz" and self.pkg.orig_tar_gz:
1080                     continue
1081                 dest = os.path.join(os.getcwd(), f)
1082                 os.symlink(src, dest)
1083
1084         # If the orig.tar.gz is not a part of the upload, create a symlink to the
1085         # existing copy.
1086         if self.pkg.orig_tar_gz:
1087             dest = os.path.join(os.getcwd(), os.path.basename(self.pkg.orig_tar_gz))
1088             os.symlink(self.pkg.orig_tar_gz, dest)
1089
1090         # Extract the source
1091         cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1092         (result, output) = commands.getstatusoutput(cmd)
1093         if (result != 0):
1094             self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1095             self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
1096             return
1097
1098         if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1099             return
1100
1101         # Get the upstream version
1102         upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1103         if re_strip_revision.search(upstr_version):
1104             upstr_version = re_strip_revision.sub('', upstr_version)
1105
1106         # Ensure the changelog file exists
1107         changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1108         if not os.path.exists(changelog_filename):
1109             self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1110             return
1111
1112         # Parse the changelog
1113         self.pkg.dsc["bts changelog"] = ""
1114         changelog_file = utils.open_file(changelog_filename)
1115         for line in changelog_file.readlines():
1116             m = re_changelog_versions.match(line)
1117             if m:
1118                 self.pkg.dsc["bts changelog"] += line
1119         changelog_file.close()
1120
1121         # Check we found at least one revision in the changelog
1122         if not self.pkg.dsc["bts changelog"]:
1123             self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1124
1125     def check_source(self):
1126         # XXX: I'm fairly sure reprocess == 2 can never happen
1127         #      AJT disabled the is_incoming check years ago - mhy
1128         #      We should probably scrap or rethink the whole reprocess thing
1129         # Bail out if:
1130         #    a) there's no source
1131         # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
1132         # or c) the orig.tar.gz is MIA
1133         if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
1134            or self.pkg.orig_tar_gz == -1:
1135             return
1136
1137         tmpdir = utils.temp_dirname()
1138
1139         # Move into the temporary directory
1140         cwd = os.getcwd()
1141         os.chdir(tmpdir)
1142
1143         # Get the changelog version history
1144         self.get_changelog_versions(cwd)
1145
1146         # Move back and cleanup the temporary tree
1147         os.chdir(cwd)
1148
1149         try:
1150             shutil.rmtree(tmpdir)
1151         except OSError, e:
1152             if e.errno != errno.EACCES:
1153                 print "foobar"
1154                 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1155
1156             self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1157             # We probably have u-r or u-w directories so chmod everything
1158             # and try again.
1159             cmd = "chmod -R u+rwx %s" % (tmpdir)
1160             result = os.system(cmd)
1161             if result != 0:
1162                 utils.fubar("'%s' failed with result %s." % (cmd, result))
1163             shutil.rmtree(tmpdir)
1164         except Exception, e:
1165             print "foobar2 (%s)" % e
1166             utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1167
1168     ###########################################################################
1169     def ensure_hashes(self):
1170         # Make sure we recognise the format of the Files: field in the .changes
1171         format = self.pkg.changes.get("format", "0.0").split(".", 1)
1172         if len(format) == 2:
1173             format = int(format[0]), int(format[1])
1174         else:
1175             format = int(float(format[0])), 0
1176
1177         # We need to deal with the original changes blob, as the fields we need
1178         # might not be in the changes dict serialised into the .dak anymore.
1179         orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1180
1181         # Copy the checksums over to the current changes dict.  This will keep
1182         # the existing modifications to it intact.
1183         for field in orig_changes:
1184             if field.startswith('checksums-'):
1185                 self.pkg.changes[field] = orig_changes[field]
1186
1187         # Check for unsupported hashes
1188         for j in utils.check_hash_fields(".changes", self.pkg.changes):
1189             self.rejects.append(j)
1190
1191         for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1192             self.rejects.append(j)
1193
1194         # We have to calculate the hash if we have an earlier changes version than
1195         # the hash appears in rather than require it exist in the changes file
1196         for hashname, hashfunc, version in utils.known_hashes:
1197             # TODO: Move _ensure_changes_hash into this class
1198             for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1199                 self.rejects.append(j)
1200             if "source" in self.pkg.changes["architecture"]:
1201                 # TODO: Move _ensure_dsc_hash into this class
1202                 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1203                     self.rejects.append(j)
1204
1205     def check_hashes(self):
1206         for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1207             self.rejects.append(m)
1208
1209         for m in utils.check_size(".changes", self.pkg.files):
1210             self.rejects.append(m)
1211
1212         for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1213             self.rejects.append(m)
1214
1215         for m in utils.check_size(".dsc", self.pkg.dsc_files):
1216             self.rejects.append(m)
1217
1218         self.ensure_hashes()
1219
1220     ###########################################################################
1221     def check_urgency(self):
1222         cnf = Config()
1223         if self.pkg.changes["architecture"].has_key("source"):
1224             if not self.pkg.changes.has_key("urgency"):
1225                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1226             self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1227             if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1228                 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1229                                      (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1230                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1231
1232     ###########################################################################
1233
1234     # Sanity check the time stamps of files inside debs.
1235     # [Files in the near future cause ugly warnings and extreme time
1236     #  travel can cause errors on extraction]
1237
1238     def check_timestamps(self):
1239         Cnf = Config()
1240
1241         future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1242         past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1243         tar = TarTime(future_cutoff, past_cutoff)
1244
1245         for filename, entry in self.pkg.files.items():
1246             if entry["type"] == "deb":
1247                 tar.reset()
1248                 try:
1249                     deb_file = utils.open_file(filename)
1250                     apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1251                     deb_file.seek(0)
1252                     try:
1253                         apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1254                     except SystemError, e:
1255                         # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1256                         if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1257                             raise
1258                         deb_file.seek(0)
1259                         apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1260
1261                     deb_file.close()
1262
1263                     future_files = tar.future_files.keys()
1264                     if future_files:
1265                         num_future_files = len(future_files)
1266                         future_file = future_files[0]
1267                         future_date = tar.future_files[future_file]
1268                         self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1269                                % (filename, num_future_files, future_file, time.ctime(future_date)))
1270
1271                     ancient_files = tar.ancient_files.keys()
1272                     if ancient_files:
1273                         num_ancient_files = len(ancient_files)
1274                         ancient_file = ancient_files[0]
1275                         ancient_date = tar.ancient_files[ancient_file]
1276                         self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1277                                % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1278                 except:
1279                     self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1280
1281     ###########################################################################
1282     def check_transition(self, session):
1283         cnf = Config()
1284
1285         sourcepkg = self.pkg.changes["source"]
1286
1287         # No sourceful upload -> no need to do anything else, direct return
1288         # We also work with unstable uploads, not experimental or those going to some
1289         # proposed-updates queue
1290         if "source" not in self.pkg.changes["architecture"] or \
1291            "unstable" not in self.pkg.changes["distribution"]:
1292             return
1293
1294         # Also only check if there is a file defined (and existant) with
1295         # checks.
1296         transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1297         if transpath == "" or not os.path.exists(transpath):
1298             return
1299
1300         # Parse the yaml file
1301         sourcefile = file(transpath, 'r')
1302         sourcecontent = sourcefile.read()
1303         try:
1304             transitions = yaml.load(sourcecontent)
1305         except yaml.YAMLError, msg:
1306             # This shouldn't happen, there is a wrapper to edit the file which
1307             # checks it, but we prefer to be safe than ending up rejecting
1308             # everything.
1309             utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1310             return
1311
1312         # Now look through all defined transitions
1313         for trans in transitions:
1314             t = transitions[trans]
1315             source = t["source"]
1316             expected = t["new"]
1317
1318             # Will be None if nothing is in testing.
1319             current = get_source_in_suite(source, "testing", session)
1320             if current is not None:
1321                 compare = apt_pkg.VersionCompare(current.version, expected)
1322
1323             if current is None or compare < 0:
1324                 # This is still valid, the current version in testing is older than
1325                 # the new version we wait for, or there is none in testing yet
1326
1327                 # Check if the source we look at is affected by this.
1328                 if sourcepkg in t['packages']:
1329                     # The source is affected, lets reject it.
1330
1331                     rejectmsg = "%s: part of the %s transition.\n\n" % (
1332                         sourcepkg, trans)
1333
1334                     if current is not None:
1335                         currentlymsg = "at version %s" % (current.version)
1336                     else:
1337                         currentlymsg = "not present in testing"
1338
1339                     rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1340
1341                     rejectmsg += "\n".join(textwrap.wrap("""Your package
1342 is part of a testing transition designed to get %s migrated (it is
1343 currently %s, we need version %s).  This transition is managed by the
1344 Release Team, and %s is the Release-Team member responsible for it.
1345 Please mail debian-release@lists.debian.org or contact %s directly if you
1346 need further assistance.  You might want to upload to experimental until this
1347 transition is done."""
1348                             % (source, currentlymsg, expected,t["rm"], t["rm"])))
1349
1350                     self.rejects.append(rejectmsg)
1351                     return
1352
1353     ###########################################################################
1354     def check_signed_by_key(self):
1355         """Ensure the .changes is signed by an authorized uploader."""
1356         session = DBConn().session()
1357
1358         self.check_transition(session)
1359
1360         (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
1361
1362         # match claimed name with actual name:
1363         if uid is None:
1364             # This is fundamentally broken but need us to refactor how we get
1365             # the UIDs/Fingerprints in order for us to fix it properly
1366             uid, uid_email = self.pkg.changes["fingerprint"], uid
1367             may_nmu, may_sponsor = 1, 1
1368             # XXX by default new dds don't have a fingerprint/uid in the db atm,
1369             #     and can't get one in there if we don't allow nmu/sponsorship
1370         elif is_dm is False:
1371             # If is_dm is False, we allow full upload rights
1372             uid_email = "%s@debian.org" % (uid)
1373             may_nmu, may_sponsor = 1, 1
1374         else:
1375             # Assume limited upload rights unless we've discovered otherwise
1376             uid_email = uid
1377             may_nmu, may_sponsor = 0, 0
1378
1379         if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1380             sponsored = 0
1381         elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1382             sponsored = 0
1383             if uid_name == "": sponsored = 1
1384         else:
1385             sponsored = 1
1386             if ("source" in self.pkg.changes["architecture"] and
1387                 uid_email and utils.is_email_alias(uid_email)):
1388                 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1389                 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1390                     self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1391                     self.pkg.changes["sponsoremail"] = uid_email
1392
1393         if sponsored and not may_sponsor:
1394             self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
1395
1396         if not sponsored and not may_nmu:
1397             should_reject = True
1398             highest_sid, highest_version = None, None
1399
1400             # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
1401             #      It ignores higher versions with the dm_upload_allowed flag set to false
1402             #      I'm keeping the existing behaviour for now until I've gone back and
1403             #      checked exactly what the GR says - mhy
1404             for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
1405                 if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
1406                      highest_sid = si.source_id
1407                      highest_version = si.version
1408
1409             if highest_sid is None:
1410                 self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
1411             else:
1412                 for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
1413                     (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1414                     if email == uid_email or name == uid_name:
1415                         should_reject = False
1416                         break
1417
1418             if should_reject is True:
1419                 self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
1420
1421             for b in self.pkg.changes["binary"].keys():
1422                 for suite in self.pkg.changes["distribution"].keys():
1423                     q = session.query(DBSource)
1424                     q = q.join(DBBinary).filter_by(package=b)
1425                     q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1426
1427                     for s in q.all():
1428                         if s.source != self.pkg.changes["source"]:
1429                             self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
1430
1431             for f in self.pkg.files.keys():
1432                 if self.pkg.files[f].has_key("byhand"):
1433                     self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
1434                 if self.pkg.files[f].has_key("new"):
1435                     self.rejects.append("%s may not upload NEW file %s" % (uid, f))
1436
1437         session.close()
1438
1439     ###########################################################################
1440     def build_summaries(self):
1441         """ Build a summary of changes the upload introduces. """
1442
1443         (byhand, new, summary, override_summary) = self.pkg.file_summary()
1444
1445         short_summary = summary
1446
1447         # This is for direport's benefit...
1448         f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1449
1450         if byhand or new:
1451             summary += "Changes: " + f
1452
1453         summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1454
1455         summary += self.announce(short_summary, 0)
1456
1457         return (summary, short_summary)
1458
1459     ###########################################################################
1460
1461     def close_bugs(self, summary, action):
1462         """
1463         Send mail to close bugs as instructed by the closes field in the changes file.
1464         Also add a line to summary if any work was done.
1465
1466         @type summary: string
1467         @param summary: summary text, as given by L{build_summaries}
1468
1469         @type action: bool
1470         @param action: Set to false no real action will be done.
1471
1472         @rtype: string
1473         @return: summary. If action was taken, extended by the list of closed bugs.
1474
1475         """
1476
1477         template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1478
1479         bugs = self.pkg.changes["closes"].keys()
1480
1481         if not bugs:
1482             return summary
1483
1484         bugs.sort()
1485         summary += "Closing bugs: "
1486         for bug in bugs:
1487             summary += "%s " % (bug)
1488             if action:
1489                 self.update_subst()
1490                 self.Subst["__BUG_NUMBER__"] = bug
1491                 if self.pkg.changes["distribution"].has_key("stable"):
1492                     self.Subst["__STABLE_WARNING__"] = """
1493 Note that this package is not part of the released stable Debian
1494 distribution.  It may have dependencies on other unreleased software,
1495 or other instabilities.  Please take care if you wish to install it.
1496 The update will eventually make its way into the next released Debian
1497 distribution."""
1498                 else:
1499                     self.Subst["__STABLE_WARNING__"] = ""
1500                 mail_message = utils.TemplateSubst(self.Subst, template)
1501                 utils.send_mail(mail_message)
1502
1503                 # Clear up after ourselves
1504                 del self.Subst["__BUG_NUMBER__"]
1505                 del self.Subst["__STABLE_WARNING__"]
1506
1507         if action and self.logger:
1508             self.logger.log(["closing bugs"] + bugs)
1509
1510         summary += "\n"
1511
1512         return summary
1513
1514     ###########################################################################
1515
1516     def announce(self, short_summary, action):
1517         """
1518         Send an announce mail about a new upload.
1519
1520         @type short_summary: string
1521         @param short_summary: Short summary text to include in the mail
1522
1523         @type action: bool
1524         @param action: Set to false no real action will be done.
1525
1526         @rtype: string
1527         @return: Textstring about action taken.
1528
1529         """
1530
1531         cnf = Config()
1532         announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1533
1534         # Only do announcements for source uploads with a recent dpkg-dev installed
1535         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1536            self.pkg.changes["architecture"].has_key("source"):
1537             return ""
1538
1539         lists_done = {}
1540         summary = ""
1541
1542         self.Subst["__SHORT_SUMMARY__"] = short_summary
1543
1544         for dist in self.pkg.changes["distribution"].keys():
1545             announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1546             if announce_list == "" or lists_done.has_key(announce_list):
1547                 continue
1548
1549             lists_done[announce_list] = 1
1550             summary += "Announcing to %s\n" % (announce_list)
1551
1552             if action:
1553                 self.update_subst()
1554                 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1555                 if cnf.get("Dinstall::TrackingServer") and \
1556                    self.pkg.changes["architecture"].has_key("source"):
1557                     trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1558                     self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1559
1560                 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1561                 utils.send_mail(mail_message)
1562
1563                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1564
1565         if cnf.FindB("Dinstall::CloseBugs"):
1566             summary = self.close_bugs(summary, action)
1567
1568         del self.Subst["__SHORT_SUMMARY__"]
1569
1570         return summary
1571
1572     ###########################################################################
1573
1574     def accept (self, summary, short_summary, targetdir=None):
1575         """
1576         Accept an upload.
1577
1578         This moves all files referenced from the .changes into the I{accepted}
1579         queue, sends the accepted mail, announces to lists, closes bugs and
1580         also checks for override disparities. If enabled it will write out
1581         the version history for the BTS Version Tracking and will finally call
1582         L{queue_build}.
1583
1584         @type summary: string
1585         @param summary: Summary text
1586
1587         @type short_summary: string
1588         @param short_summary: Short summary
1589
1590         """
1591
1592         cnf = Config()
1593         stats = SummaryStats()
1594
1595         accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
1596
1597         if targetdir is None:
1598             targetdir = cnf["Dir::Queue::Accepted"]
1599
1600         print "Accepting."
1601         if self.logger:
1602             self.logger.log(["Accepting changes", self.pkg.changes_file])
1603
1604         self.pkg.write_dot_dak(targetdir)
1605
1606         # Move all the files into the accepted directory
1607         utils.move(self.pkg.changes_file, targetdir)
1608
1609         for name, entry in sorted(self.pkg.files.items()):
1610             utils.move(name, targetdir)
1611             stats.accept_bytes += float(entry["size"])
1612
1613         stats.accept_count += 1
1614
1615         # Send accept mail, announce to lists, close bugs and check for
1616         # override disparities
1617         if not cnf["Dinstall::Options::No-Mail"]:
1618             self.update_subst()
1619             self.Subst["__SUITE__"] = ""
1620             self.Subst["__SUMMARY__"] = summary
1621             mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
1622             utils.send_mail(mail_message)
1623             self.announce(short_summary, 1)
1624
1625         ## Helper stuff for DebBugs Version Tracking
1626         if cnf.Find("Dir::Queue::BTSVersionTrack"):
1627             # ??? once queue/* is cleared on *.d.o and/or reprocessed
1628             # the conditionalization on dsc["bts changelog"] should be
1629             # dropped.
1630
1631             # Write out the version history from the changelog
1632             if self.pkg.changes["architecture"].has_key("source") and \
1633                self.pkg.dsc.has_key("bts changelog"):
1634
1635                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1636                 version_history = os.fdopen(fd, 'w')
1637                 version_history.write(self.pkg.dsc["bts changelog"])
1638                 version_history.close()
1639                 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1640                                       self.pkg.changes_file[:-8]+".versions")
1641                 os.rename(temp_filename, filename)
1642                 os.chmod(filename, 0644)
1643
1644             # Write out the binary -> source mapping.
1645             (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1646             debinfo = os.fdopen(fd, 'w')
1647             for name, entry in sorted(self.pkg.files.items()):
1648                 if entry["type"] == "deb":
1649                     line = " ".join([entry["package"], entry["version"],
1650                                      entry["architecture"], entry["source package"],
1651                                      entry["source version"]])
1652                     debinfo.write(line+"\n")
1653             debinfo.close()
1654             filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1655                                   self.pkg.changes_file[:-8]+".debinfo")
1656             os.rename(temp_filename, filename)
1657             os.chmod(filename, 0644)
1658
1659         # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
1660         # <Ganneff> we do call queue_build too
1661         # <mhy> well yes, we'd have had to if we were inserting into accepted
1662         # <Ganneff> now. thats database only.
1663         # <mhy> urgh, that's going to get messy
1664         # <Ganneff> so i make the p-n call to it *also* using accepted/
1665         # <mhy> but then the packages will be in the queue_build table without the files being there
1666         # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
1667         # <mhy> ah, good point
1668         # <Ganneff> so it will work out, as unchecked move it over
1669         # <mhy> that's all completely sick
1670         # <Ganneff> yes
1671
1672         # This routine returns None on success or an error on failure
1673         res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
1674         if res:
1675             utils.fubar(res)
1676
1677
1678     def check_override(self):
1679         """
1680         Checks override entries for validity. Mails "Override disparity" warnings,
1681         if that feature is enabled.
1682
1683         Abandons the check if
1684           - override disparity checks are disabled
1685           - mail sending is disabled
1686         """
1687
1688         cnf = Config()
1689
1690         # Abandon the check if:
1691         #  a) override disparity checks have been disabled
1692         #  b) we're not sending mail
1693         if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
1694            cnf["Dinstall::Options::No-Mail"]:
1695             return
1696
1697         summary = self.pkg.check_override()
1698
1699         if summary == "":
1700             return
1701
1702         overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
1703
1704         self.update_subst()
1705         self.Subst["__SUMMARY__"] = summary
1706         mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
1707         utils.send_mail(mail_message)
1708         del self.Subst["__SUMMARY__"]
1709
1710     ###########################################################################
1711
1712     def remove(self, dir=None):
1713         """
1714         Used (for instance) in p-u to remove the package from unchecked
1715         """
1716         if dir is None:
1717             os.chdir(self.pkg.directory)
1718         else:
1719             os.chdir(dir)
1720
1721         for f in self.pkg.files.keys():
1722             os.unlink(f)
1723         os.unlink(self.pkg.changes_file)
1724
1725     ###########################################################################
1726
1727     def move_to_dir (self, dest, perms=0660, changesperms=0664):
1728         """
1729         Move files to dest with certain perms/changesperms
1730         """
1731         utils.move(self.pkg.changes_file, dest, perms=changesperms)
1732         for f in self.pkg.files.keys():
1733             utils.move(f, dest, perms=perms)
1734
1735     ###########################################################################
1736
1737     def force_reject(self, reject_files):
1738         """
1739         Forcefully move files from the current directory to the
1740         reject directory.  If any file already exists in the reject
1741         directory it will be moved to the morgue to make way for
1742         the new file.
1743
1744         @type files: dict
1745         @param files: file dictionary
1746
1747         """
1748
1749         cnf = Config()
1750
1751         for file_entry in reject_files:
1752             # Skip any files which don't exist or which we don't have permission to copy.
1753             if os.access(file_entry, os.R_OK) == 0:
1754                 continue
1755
1756             dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
1757
1758             try:
1759                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
1760             except OSError, e:
1761                 # File exists?  Let's try and move it to the morgue
1762                 if e.errno == errno.EEXIST:
1763                     morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
1764                     try:
1765                         morgue_file = utils.find_next_free(morgue_file)
1766                     except NoFreeFilenameError:
1767                         # Something's either gone badly Pete Tong, or
1768                         # someone is trying to exploit us.
1769                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
1770                         return
1771                     utils.move(dest_file, morgue_file, perms=0660)
1772                     try:
1773                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1774                     except OSError, e:
1775                         # Likewise
1776                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
1777                         return
1778                 else:
1779                     raise
1780             # If we got here, we own the destination file, so we can
1781             # safely overwrite it.
1782             utils.move(file_entry, dest_file, 1, perms=0660)
1783             os.close(dest_fd)
1784
1785     ###########################################################################
1786     def do_reject (self, manual=0, reject_message="", note=""):
1787         """
1788         Reject an upload. If called without a reject message or C{manual} is
1789         true, spawn an editor so the user can write one.
1790
1791         @type manual: bool
1792         @param manual: manual or automated rejection
1793
1794         @type reject_message: string
1795         @param reject_message: A reject message
1796
1797         @return: 0
1798
1799         """
1800         # If we weren't given a manual rejection message, spawn an
1801         # editor so the user can add one in...
1802         if manual and not reject_message:
1803             (fd, temp_filename) = utils.temp_filename()
1804             temp_file = os.fdopen(fd, 'w')
1805             if len(note) > 0:
1806                 for line in note:
1807                     temp_file.write(line)
1808             temp_file.close()
1809             editor = os.environ.get("EDITOR","vi")
1810             answer = 'E'
1811             while answer == 'E':
1812                 os.system("%s %s" % (editor, temp_filename))
1813                 temp_fh = utils.open_file(temp_filename)
1814                 reject_message = "".join(temp_fh.readlines())
1815                 temp_fh.close()
1816                 print "Reject message:"
1817                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1)
1818                 prompt = "[R]eject, Edit, Abandon, Quit ?"
1819                 answer = "XXX"
1820                 while prompt.find(answer) == -1:
1821                     answer = utils.our_raw_input(prompt)
1822                     m = re_default_answer.search(prompt)
1823                     if answer == "":
1824                         answer = m.group(1)
1825                     answer = answer[:1].upper()
1826             os.unlink(temp_filename)
1827             if answer == 'A':
1828                 return 1
1829             elif answer == 'Q':
1830                 sys.exit(0)
1831
1832         print "Rejecting.\n"
1833
1834         cnf = Config()
1835
1836         reason_filename = self.pkg.changes_file[:-8] + ".reason"
1837         reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
1838
1839         # Move all the files into the reject directory
1840         reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
1841         self.force_reject(reject_files)
1842
1843         # If we fail here someone is probably trying to exploit the race
1844         # so let's just raise an exception ...
1845         if os.path.exists(reason_filename):
1846             os.unlink(reason_filename)
1847         reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1848
1849         rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
1850
1851         self.update_subst()
1852         if not manual:
1853             self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
1854             self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
1855             self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
1856             os.write(reason_fd, reject_message)
1857             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1858         else:
1859             # Build up the rejection email
1860             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
1861             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
1862             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
1863             self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
1864             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1865             # Write the rejection email out as the <foo>.reason file
1866             os.write(reason_fd, reject_mail_message)
1867
1868         del self.Subst["__REJECTOR_ADDRESS__"]
1869         del self.Subst["__MANUAL_REJECT_MESSAGE__"]
1870         del self.Subst["__CC__"]
1871
1872         os.close(reason_fd)
1873
1874         # Send the rejection mail if appropriate
1875         if not cnf["Dinstall::Options::No-Mail"]:
1876             utils.send_mail(reject_mail_message)
1877
1878         if self.logger:
1879             self.logger.log(["rejected", self.pkg.changes_file])
1880
1881         return 0
1882
1883     ################################################################################
1884     def in_override_p(self, package, component, suite, binary_type, file, session):
1885         """
1886         Check if a package already has override entries in the DB
1887
1888         @type package: string
1889         @param package: package name
1890
1891         @type component: string
1892         @param component: database id of the component
1893
1894         @type suite: int
1895         @param suite: database id of the suite
1896
1897         @type binary_type: string
1898         @param binary_type: type of the package
1899
1900         @type file: string
1901         @param file: filename we check
1902
1903         @return: the database result. But noone cares anyway.
1904
1905         """
1906
1907         cnf = Config()
1908
1909         if binary_type == "": # must be source
1910             file_type = "dsc"
1911         else:
1912             file_type = binary_type
1913
1914         # Override suite name; used for example with proposed-updates
1915         if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
1916             suite = cnf["Suite::%s::OverrideSuite" % (suite)]
1917
1918         result = get_override(package, suite, component, file_type, session)
1919
1920         # If checking for a source package fall back on the binary override type
1921         if file_type == "dsc" and len(result) < 1:
1922             result = get_override(package, suite, component, ['deb', 'udeb'], session)
1923
1924         # Remember the section and priority so we can check them later if appropriate
1925         if len(result) > 0:
1926             result = result[0]
1927             self.pkg.files[file]["override section"] = result.section.section
1928             self.pkg.files[file]["override priority"] = result.priority.priority
1929             return result
1930
1931         return None
1932
1933     ################################################################################
1934     def get_anyversion(self, sv_list, suite):
1935         """
1936         @type sv_list: list
1937         @param sv_list: list of (suite, version) tuples to check
1938
1939         @type suite: string
1940         @param suite: suite name
1941
1942         Description: TODO
1943         """
1944         Cnf = Config()
1945         anyversion = None
1946         anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
1947         for (s, v) in sv_list:
1948             if s in [ x.lower() for x in anysuite ]:
1949                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
1950                     anyversion = v
1951
1952         return anyversion
1953
1954     ################################################################################
1955
1956     def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
1957         """
1958         @type sv_list: list
1959         @param sv_list: list of (suite, version) tuples to check
1960
1961         @type file: string
1962         @param file: XXX
1963
1964         @type new_version: string
1965         @param new_version: XXX
1966
1967         Ensure versions are newer than existing packages in target
1968         suites and that cross-suite version checking rules as
1969         set out in the conf file are satisfied.
1970         """
1971
1972         cnf = Config()
1973
1974         # Check versions for each target suite
1975         for target_suite in self.pkg.changes["distribution"].keys():
1976             must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
1977             must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
1978
1979             # Enforce "must be newer than target suite" even if conffile omits it
1980             if target_suite not in must_be_newer_than:
1981                 must_be_newer_than.append(target_suite)
1982
1983             for (suite, existent_version) in sv_list:
1984                 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
1985
1986                 if suite in must_be_newer_than and sourceful and vercmp < 1:
1987                     self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
1988
1989                 if suite in must_be_older_than and vercmp > -1:
1990                     cansave = 0
1991
1992                     if self.pkg.changes.get('distribution-version', {}).has_key(suite):
1993                         # we really use the other suite, ignoring the conflicting one ...
1994                         addsuite = self.pkg.changes["distribution-version"][suite]
1995
1996                         add_version = self.get_anyversion(sv_list, addsuite)
1997                         target_version = self.get_anyversion(sv_list, target_suite)
1998
1999                         if not add_version:
2000                             # not add_version can only happen if we map to a suite
2001                             # that doesn't enhance the suite we're propup'ing from.
2002                             # so "propup-ver x a b c; map a d" is a problem only if
2003                             # d doesn't enhance a.
2004                             #
2005                             # i think we could always propagate in this case, rather
2006                             # than complaining. either way, this isn't a REJECT issue
2007                             #
2008                             # And - we really should complain to the dorks who configured dak
2009                             self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
2010                             self.pkg.changes.setdefault("propdistribution", {})
2011                             self.pkg.changes["propdistribution"][addsuite] = 1
2012                             cansave = 1
2013                         elif not target_version:
2014                             # not targets_version is true when the package is NEW
2015                             # we could just stick with the "...old version..." REJECT
2016                             # for this, I think.
2017                             self.rejects.append("Won't propogate NEW packages.")
2018                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2019                             # propogation would be redundant. no need to reject though.
2020                             self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2021                             cansave = 1
2022                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2023                              apt_pkg.VersionCompare(add_version, target_version) >= 0:
2024                             # propogate!!
2025                             self.warnings.append("Propogating upload to %s" % (addsuite))
2026                             self.pkg.changes.setdefault("propdistribution", {})
2027                             self.pkg.changes["propdistribution"][addsuite] = 1
2028                             cansave = 1
2029
2030                     if not cansave:
2031                         self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2032
2033     ################################################################################
2034     def check_binary_against_db(self, file, session):
2035         # Ensure version is sane
2036         q = session.query(BinAssociation)
2037         q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
2038         q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
2039
2040         self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2041                                        file, self.pkg.files[file]["version"], sourceful=False)
2042
2043         # Check for any existing copies of the file
2044         q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
2045         q = q.filter_by(version=self.pkg.files[file]["version"])
2046         q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
2047
2048         if q.count() > 0:
2049             self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
2050
2051     ################################################################################
2052
2053     def check_source_against_db(self, file, session):
2054         """
2055         """
2056         source = self.pkg.dsc.get("source")
2057         version = self.pkg.dsc.get("version")
2058
2059         # Ensure version is sane
2060         q = session.query(SrcAssociation)
2061         q = q.join(DBSource).filter(DBSource.source==source)
2062
2063         self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2064                                        file, version, sourceful=True)
2065
2066     ################################################################################
2067     def check_dsc_against_db(self, file, session):
2068         """
2069
2070         @warning: NB: this function can remove entries from the 'files' index [if
2071          the .orig.tar.gz is a duplicate of the one in the archive]; if
2072          you're iterating over 'files' and call this function as part of
2073          the loop, be sure to add a check to the top of the loop to
2074          ensure you haven't just tried to dereference the deleted entry.
2075
2076         """
2077
2078         Cnf = Config()
2079         self.pkg.orig_tar_gz = None
2080
2081         # Try and find all files mentioned in the .dsc.  This has
2082         # to work harder to cope with the multiple possible
2083         # locations of an .orig.tar.gz.
2084         # The ordering on the select is needed to pick the newest orig
2085         # when it exists in multiple places.
2086         for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2087             found = None
2088             if self.pkg.files.has_key(dsc_name):
2089                 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2090                 actual_size = int(self.pkg.files[dsc_name]["size"])
2091                 found = "%s in incoming" % (dsc_name)
2092
2093                 # Check the file does not already exist in the archive
2094                 ql = get_poolfile_like_name(dsc_name, session)
2095
2096                 # Strip out anything that isn't '%s' or '/%s$'
2097                 for i in ql:
2098                     if not i.filename.endswith(dsc_name):
2099                         ql.remove(i)
2100
2101                 # "[dak] has not broken them.  [dak] has fixed a
2102                 # brokenness.  Your crappy hack exploited a bug in
2103                 # the old dinstall.
2104                 #
2105                 # "(Come on!  I thought it was always obvious that
2106                 # one just doesn't release different files with
2107                 # the same name and version.)"
2108                 #                        -- ajk@ on d-devel@l.d.o
2109
2110                 if len(ql) > 0:
2111                     # Ignore exact matches for .orig.tar.gz
2112                     match = 0
2113                     if dsc_name.endswith(".orig.tar.gz"):
2114                         for i in ql:
2115                             if self.pkg.files.has_key(dsc_name) and \
2116                                int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2117                                self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2118                                 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2119                                 # TODO: Don't delete the entry, just mark it as not needed
2120                                 # This would fix the stupidity of changing something we often iterate over
2121                                 # whilst we're doing it
2122                                 del self.pkg.files[dsc_name]
2123                                 self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename)
2124                                 match = 1
2125
2126                     if not match:
2127                         self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2128
2129             elif dsc_name.endswith(".orig.tar.gz"):
2130                 # Check in the pool
2131                 ql = get_poolfile_like_name(dsc_name, session)
2132
2133                 # Strip out anything that isn't '%s' or '/%s$'
2134                 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2135                 for i in ql:
2136                     if not i.filename.endswith(dsc_name):
2137                         ql.remove(i)
2138
2139                 if len(ql) > 0:
2140                     # Unfortunately, we may get more than one match here if,
2141                     # for example, the package was in potato but had an -sa
2142                     # upload in woody.  So we need to choose the right one.
2143
2144                     # default to something sane in case we don't match any or have only one
2145                     x = ql[0]
2146
2147                     if len(ql) > 1:
2148                         for i in ql:
2149                             old_file = os.path.join(i.location.path, i.filename)
2150                             old_file_fh = utils.open_file(old_file)
2151                             actual_md5 = apt_pkg.md5sum(old_file_fh)
2152                             old_file_fh.close()
2153                             actual_size = os.stat(old_file)[stat.ST_SIZE]
2154                             if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2155                                 x = i
2156
2157                     old_file = os.path.join(i.location.path, i.filename)
2158                     old_file_fh = utils.open_file(old_file)
2159                     actual_md5 = apt_pkg.md5sum(old_file_fh)
2160                     old_file_fh.close()
2161                     actual_size = os.stat(old_file)[stat.ST_SIZE]
2162                     found = old_file
2163                     suite_type = x.location.archive_type
2164                     # need this for updating dsc_files in install()
2165                     dsc_entry["files id"] = x.file_id
2166                     # See install() in process-accepted...
2167                     self.pkg.orig_tar_id = x.file_id
2168                     self.pkg.orig_tar_gz = old_file
2169                     self.pkg.orig_tar_location = x.location.location_id
2170                 else:
2171                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2172                     # Not there? Check the queue directories...
2173                     for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2174                         if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2175                             continue
2176                         in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2177                         if os.path.exists(in_otherdir):
2178                             in_otherdir_fh = utils.open_file(in_otherdir)
2179                             actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2180                             in_otherdir_fh.close()
2181                             actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2182                             found = in_otherdir
2183                             self.pkg.orig_tar_gz = in_otherdir
2184
2185                     if not found:
2186                         self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
2187                         self.pkg.orig_tar_gz = -1
2188                         continue
2189             else:
2190                 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
2191                 continue
2192             if actual_md5 != dsc_entry["md5sum"]:
2193                 self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
2194             if actual_size != int(dsc_entry["size"]):
2195                 self.rejects.append("size for %s doesn't match %s." % (found, file))
2196
2197     ################################################################################
2198     # This is used by process-new and process-holding to recheck a changes file
2199     # at the time we're running.  It mainly wraps various other internal functions
2200     # and is similar to accepted_checks - these should probably be tidied up
2201     # and combined
2202     def recheck(self, session):
2203         cnf = Config()
2204         for f in self.pkg.files.keys():
2205             # The .orig.tar.gz can disappear out from under us is it's a
2206             # duplicate of one in the archive.
2207             if not self.pkg.files.has_key(f):
2208                 continue
2209
2210             entry = self.pkg.files[f]
2211
2212             # Check that the source still exists
2213             if entry["type"] == "deb":
2214                 source_version = entry["source version"]
2215                 source_package = entry["source package"]
2216                 if not self.pkg.changes["architecture"].has_key("source") \
2217                    and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
2218                     source_epochless_version = re_no_epoch.sub('', source_version)
2219                     dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
2220                     found = False
2221                     for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
2222                         if cnf.has_key("Dir::Queue::%s" % (q)):
2223                             if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
2224                                 found = True
2225                     if not found:
2226                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
2227
2228             # Version and file overwrite checks
2229             if entry["type"] == "deb":
2230                 self.check_binary_against_db(f, session)
2231             elif entry["type"] == "dsc":
2232                 self.check_source_against_db(f, session)
2233                 self.check_dsc_against_db(f, session)
2234
2235     ################################################################################
2236     def accepted_checks(self, overwrite_checks, session):
2237         # Recheck anything that relies on the database; since that's not
2238         # frozen between accept and our run time when called from p-a.
2239
2240         # overwrite_checks is set to False when installing to stable/oldstable
2241
2242         propogate={}
2243         nopropogate={}
2244
2245         # Find the .dsc (again)
2246         dsc_filename = None
2247         for f in self.pkg.files.keys():
2248             if self.pkg.files[f]["type"] == "dsc":
2249                 dsc_filename = f
2250
2251         for checkfile in self.pkg.files.keys():
2252             # The .orig.tar.gz can disappear out from under us is it's a
2253             # duplicate of one in the archive.
2254             if not self.pkg.files.has_key(checkfile):
2255                 continue
2256
2257             entry = self.pkg.files[checkfile]
2258
2259             # Check that the source still exists
2260             if entry["type"] == "deb":
2261                 source_version = entry["source version"]
2262                 source_package = entry["source package"]
2263                 if not self.pkg.changes["architecture"].has_key("source") \
2264                    and not source_exists(source_package, source_version,  self.pkg.changes["distribution"].keys()):
2265                     self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2266
2267             # Version and file overwrite checks
2268             if overwrite_checks:
2269                 if entry["type"] == "deb":
2270                     self.check_binary_against_db(checkfile, session)
2271                 elif entry["type"] == "dsc":
2272                     self.check_source_against_db(checkfile, session)
2273                     self.check_dsc_against_db(dsc_filename, session)
2274
2275             # propogate in the case it is in the override tables:
2276             for suite in self.pkg.changes.get("propdistribution", {}).keys():
2277                 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2278                     propogate[suite] = 1
2279                 else:
2280                     nopropogate[suite] = 1
2281
2282         for suite in propogate.keys():
2283             if suite in nopropogate:
2284                 continue
2285             self.pkg.changes["distribution"][suite] = 1
2286
2287         for checkfile in self.pkg.files.keys():
2288             # Check the package is still in the override tables
2289             for suite in self.pkg.changes["distribution"].keys():
2290                 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2291                     self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2292
2293     ################################################################################
2294     # This is not really a reject, but an unaccept, but since a) the code for
2295     # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2296     # extremely rare, for now we'll go with whining at our admin folks...
2297
2298     def do_unaccept(self):
2299         cnf = Config()
2300
2301         self.update_subst()
2302         self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2303         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2304         self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2305         self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
2306         if cnf.has_key("Dinstall::Bcc"):
2307             self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2308
2309         template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2310
2311         reject_mail_message = utils.TemplateSubst(self.Subst, template)
2312
2313         # Write the rejection email out as the <foo>.reason file
2314         reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2315         reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2316
2317         # If we fail here someone is probably trying to exploit the race
2318         # so let's just raise an exception ...
2319         if os.path.exists(reject_filename):
2320             os.unlink(reject_filename)
2321
2322         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2323         os.write(fd, reject_mail_message)
2324         os.close(fd)
2325
2326         utils.send_mail(reject_mail_message)
2327
2328         del self.Subst["__REJECTOR_ADDRESS__"]
2329         del self.Subst["__REJECT_MESSAGE__"]
2330         del self.Subst["__CC__"]
2331
2332     ################################################################################
2333     # If any file of an upload has a recent mtime then chances are good
2334     # the file is still being uploaded.
2335
2336     def upload_too_new(self):
2337         cnf = Config()
2338         too_new = False
2339         # Move back to the original directory to get accurate time stamps
2340         cwd = os.getcwd()
2341         os.chdir(self.pkg.directory)
2342         file_list = self.pkg.files.keys()
2343         file_list.extend(self.pkg.dsc_files.keys())
2344         file_list.append(self.pkg.changes_file)
2345         for f in file_list:
2346             try:
2347                 last_modified = time.time()-os.path.getmtime(f)
2348                 if last_modified < int(cnf["Dinstall::SkipTime"]):
2349                     too_new = True
2350                     break
2351             except:
2352                 pass
2353
2354         os.chdir(cwd)
2355         return too_new