daklib/queue.py

   1 #!/usr/bin/env python
   2 # vim:set et sw=4:
   3
   4 """
   5 Queue utility functions for dak
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
   9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ###############################################################################
  28
  29 import errno
  30 import os
  31 import stat
  32 import sys
  33 import time
  34 import apt_inst
  35 import apt_pkg
  36 import utils
  37 import commands
  38 import shutil
  39 import textwrap
  40 from types import *
  41 from sqlalchemy.sql.expression import desc
  42 from sqlalchemy.orm.exc import NoResultFound
  43
  44 import yaml
  45
  46 from dak_exceptions import *
  47 from changes import *
  48 from regexes import *
  49 from config import Config
  50 from holding import Holding
  51 from urgencylog import UrgencyLog
  52 from dbconn import *
  53 from summarystats import SummaryStats
  54 from utils import parse_changes, check_dsc_files
  55 from textutils import fix_maintainer
  56 from binary import Binary
  57
  58 ###############################################################################
  59
  60 def get_type(f, session):
  61     """
  62     Get the file type of C{f}
  63
  64     @type f: dict
  65     @param f: file entry from Changes object
  66
  67     @type session: SQLA Session
  68     @param session: SQL Alchemy session object
  69
  70     @rtype: string
  71     @return: filetype
  72
  73     """
  74     # Determine the type
  75     if f.has_key("dbtype"):
  76         file_type = f["dbtype"]
  77     elif re_source_ext.match(f["type"]):
  78         file_type = "dsc"
  79     else:
  80         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))
  81
  82     # Validate the override type
  83     type_id = get_override_type(file_type, session)
  84     if type_id is None:
  85         utils.fubar("invalid type (%s) for new.  Say wha?" % (file_type))
  86
  87     return file_type
  88
  89 ################################################################################
  90
  91 # Determine what parts in a .changes are NEW
  92
  93 def determine_new(changes, files, warn=1):
  94     """
  95     Determine what parts in a C{changes} file are NEW.
  96
  97     @type changes: Upload.Pkg.changes dict
  98     @param changes: Changes dictionary
  99
 100     @type files: Upload.Pkg.files dict
 101     @param files: Files dictionary
 102
 103     @type warn: bool
 104     @param warn: Warn if overrides are added for (old)stable
 105
 106     @rtype: dict
 107     @return: dictionary of NEW components.
 108
 109     """
 110     new = {}
 111
 112     session = DBConn().session()
 113
 114     # Build up a list of potentially new things
 115     for name, f in files.items():
 116         # Skip byhand elements
 117         if f["type"] == "byhand":
 118             continue
 119         pkg = f["package"]
 120         priority = f["priority"]
 121         section = f["section"]
 122         file_type = get_type(f, session)
 123         component = f["component"]
 124
 125         if file_type == "dsc":
 126             priority = "source"
 127
 128         if not new.has_key(pkg):
 129             new[pkg] = {}
 130             new[pkg]["priority"] = priority
 131             new[pkg]["section"] = section
 132             new[pkg]["type"] = file_type
 133             new[pkg]["component"] = component
 134             new[pkg]["files"] = []
 135         else:
 136             old_type = new[pkg]["type"]
 137             if old_type != file_type:
 138                 # source gets trumped by deb or udeb
 139                 if old_type == "dsc":
 140                     new[pkg]["priority"] = priority
 141                     new[pkg]["section"] = section
 142                     new[pkg]["type"] = file_type
 143                     new[pkg]["component"] = component
 144
 145         new[pkg]["files"].append(name)
 146
 147         if f.has_key("othercomponents"):
 148             new[pkg]["othercomponents"] = f["othercomponents"]
 149
 150     for suite in changes["suite"].keys():
 151         for pkg in new.keys():
 152             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
 153             if len(ql) > 0:
 154                 for file_entry in new[pkg]["files"]:
 155                     if files[file_entry].has_key("new"):
 156                         del files[file_entry]["new"]
 157                 del new[pkg]
 158
 159     if warn:
 160         for s in ['stable', 'oldstable']:
 161             if changes["suite"].has_key(s):
 162                 print "WARNING: overrides will be added for %s!" % s
 163         for pkg in new.keys():
 164             if new[pkg].has_key("othercomponents"):
 165                 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 166
 167     session.close()
 168
 169     return new
 170
 171 ################################################################################
 172
 173 def check_valid(new):
 174     """
 175     Check if section and priority for NEW packages exist in database.
 176     Additionally does sanity checks:
 177       - debian-installer packages have to be udeb (or source)
 178       - non debian-installer packages can not be udeb
 179       - source priority can only be assigned to dsc file types
 180
 181     @type new: dict
 182     @param new: Dict of new packages with their section, priority and type.
 183
 184     """
 185     for pkg in new.keys():
 186         section_name = new[pkg]["section"]
 187         priority_name = new[pkg]["priority"]
 188         file_type = new[pkg]["type"]
 189
 190         section = get_section(section_name)
 191         if section is None:
 192             new[pkg]["section id"] = -1
 193         else:
 194             new[pkg]["section id"] = section.section_id
 195
 196         priority = get_priority(priority_name)
 197         if priority is None:
 198             new[pkg]["priority id"] = -1
 199         else:
 200             new[pkg]["priority id"] = priority.priority_id
 201
 202         # Sanity checks
 203         di = section_name.find("debian-installer") != -1
 204
 205         # If d-i, we must be udeb and vice-versa
 206         if     (di and file_type not in ("udeb", "dsc")) or \
 207            (not di and file_type == "udeb"):
 208             new[pkg]["section id"] = -1
 209
 210         # If dsc we need to be source and vice-versa
 211         if (priority == "source" and file_type != "dsc") or \
 212            (priority != "source" and file_type == "dsc"):
 213             new[pkg]["priority id"] = -1
 214
 215 ###############################################################################
 216
 217 def check_status(files):
 218     new = byhand = 0
 219     for f in files.keys():
 220         if files[f]["type"] == "byhand":
 221             byhand = 1
 222         elif files[f].has_key("new"):
 223             new = 1
 224     return (new, byhand)
 225
 226 ###############################################################################
 227
 228 # Used by Upload.check_timestamps
 229 class TarTime(object):
 230     def __init__(self, future_cutoff, past_cutoff):
 231         self.reset()
 232         self.future_cutoff = future_cutoff
 233         self.past_cutoff = past_cutoff
 234
 235     def reset(self):
 236         self.future_files = {}
 237         self.ancient_files = {}
 238
 239     def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
 240         if MTime > self.future_cutoff:
 241             self.future_files[Name] = MTime
 242         if MTime < self.past_cutoff:
 243             self.ancient_files[Name] = MTime
 244
 245 ###############################################################################
 246
 247 class Upload(object):
 248     """
 249     Everything that has to do with an upload processed.
 250
 251     """
 252     def __init__(self):
 253         self.logger = None
 254         self.pkg = Changes()
 255         self.reset()
 256
 257     ###########################################################################
 258
 259     def reset (self):
 260         """ Reset a number of internal variables."""
 261
 262         # Initialize the substitution template map
 263         cnf = Config()
 264         self.Subst = {}
 265         self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
 266         self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
 267         self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
 268         self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 269
 270         self.rejects = []
 271         self.warnings = []
 272         self.notes = []
 273
 274         self.pkg.reset()
 275
 276     def package_info(self):
 277         """
 278         Format various messages from this Upload to send to the maintainer.
 279         """
 280
 281         msgs = (
 282             ('Reject Reasons', self.rejects),
 283             ('Warnings', self.warnings),
 284             ('Notes', self.notes),
 285         )
 286
 287         msg = ''
 288         for title, messages in msgs:
 289             if messages:
 290                 msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))
 291         msg += '\n'
 292
 293         return msg
 294
 295     ###########################################################################
 296     def update_subst(self):
 297         """ Set up the per-package template substitution mappings """
 298
 299         cnf = Config()
 300
 301         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
 302         if not self.pkg.changes.has_key("architecture") or not \
 303            isinstance(self.pkg.changes["architecture"], dict):
 304             self.pkg.changes["architecture"] = { "Unknown" : "" }
 305
 306         # and maintainer2047 may not exist.
 307         if not self.pkg.changes.has_key("maintainer2047"):
 308             self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
 309
 310         self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
 311         self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
 312         self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
 313
 314         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 315         if self.pkg.changes["architecture"].has_key("source") and \
 316            self.pkg.changes["changedby822"] != "" and \
 317            (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
 318
 319             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
 320             self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
 321             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
 322         else:
 323             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
 324             self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
 325             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
 326
 327         if "sponsoremail" in self.pkg.changes:
 328             self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
 329
 330         if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
 331             self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
 332
 333         # Apply any global override of the Maintainer field
 334         if cnf.get("Dinstall::OverrideMaintainer"):
 335             self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
 336             self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
 337
 338         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
 339         self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
 340         self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
 341
 342     ###########################################################################
 343     def load_changes(self, filename):
 344         """
 345         @rtype: boolean
 346         @rvalue: whether the changes file was valid or not.  We may want to
 347                  reject even if this is True (see what gets put in self.rejects).
 348                  This is simply to prevent us even trying things later which will
 349                  fail because we couldn't properly parse the file.
 350         """
 351         Cnf = Config()
 352         self.pkg.changes_file = filename
 353
 354         # Parse the .changes field into a dictionary
 355         try:
 356             self.pkg.changes.update(parse_changes(filename))
 357         except CantOpenError:
 358             self.rejects.append("%s: can't read file." % (filename))
 359             return False
 360         except ParseChangesError, line:
 361             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 362             return False
 363         except ChangesUnicodeError:
 364             self.rejects.append("%s: changes file not proper utf-8" % (filename))
 365             return False
 366
 367         # Parse the Files field from the .changes into another dictionary
 368         try:
 369             self.pkg.files.update(utils.build_file_list(self.pkg.changes))
 370         except ParseChangesError, line:
 371             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 372             return False
 373         except UnknownFormatError, format:
 374             self.rejects.append("%s: unknown format '%s'." % (filename, format))
 375             return False
 376
 377         # Check for mandatory fields
 378         for i in ("distribution", "source", "binary", "architecture",
 379                   "version", "maintainer", "files", "changes", "description"):
 380             if not self.pkg.changes.has_key(i):
 381                 # Avoid undefined errors later
 382                 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
 383                 return False
 384
 385         # Strip a source version in brackets from the source field
 386         if re_strip_srcver.search(self.pkg.changes["source"]):
 387             self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
 388
 389         # Ensure the source field is a valid package name.
 390         if not re_valid_pkg_name.match(self.pkg.changes["source"]):
 391             self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
 392
 393         # Split multi-value fields into a lower-level dictionary
 394         for i in ("architecture", "distribution", "binary", "closes"):
 395             o = self.pkg.changes.get(i, "")
 396             if o != "":
 397                 del self.pkg.changes[i]
 398
 399             self.pkg.changes[i] = {}
 400
 401             for j in o.split():
 402                 self.pkg.changes[i][j] = 1
 403
 404         # Fix the Maintainer: field to be RFC822/2047 compatible
 405         try:
 406             (self.pkg.changes["maintainer822"],
 407              self.pkg.changes["maintainer2047"],
 408              self.pkg.changes["maintainername"],
 409              self.pkg.changes["maintaineremail"]) = \
 410                    fix_maintainer (self.pkg.changes["maintainer"])
 411         except ParseMaintError, msg:
 412             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
 413                    % (filename, self.pkg.changes["maintainer"], msg))
 414
 415         # ...likewise for the Changed-By: field if it exists.
 416         try:
 417             (self.pkg.changes["changedby822"],
 418              self.pkg.changes["changedby2047"],
 419              self.pkg.changes["changedbyname"],
 420              self.pkg.changes["changedbyemail"]) = \
 421                    fix_maintainer (self.pkg.changes.get("changed-by", ""))
 422         except ParseMaintError, msg:
 423             self.pkg.changes["changedby822"] = ""
 424             self.pkg.changes["changedby2047"] = ""
 425             self.pkg.changes["changedbyname"] = ""
 426             self.pkg.changes["changedbyemail"] = ""
 427
 428             self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
 429                    % (filename, changes["changed-by"], msg))
 430
 431         # Ensure all the values in Closes: are numbers
 432         if self.pkg.changes.has_key("closes"):
 433             for i in self.pkg.changes["closes"].keys():
 434                 if re_isanum.match (i) == None:
 435                     self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
 436
 437         # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
 438         self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
 439         self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
 440
 441         # Check the .changes is non-empty
 442         if not self.pkg.files:
 443             self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
 444             return False
 445
 446         # Changes was syntactically valid even if we'll reject
 447         return True
 448
 449     ###########################################################################
 450
 451     def check_distributions(self):
 452         "Check and map the Distribution field"
 453
 454         Cnf = Config()
 455
 456         # Handle suite mappings
 457         for m in Cnf.ValueList("SuiteMappings"):
 458             args = m.split()
 459             mtype = args[0]
 460             if mtype == "map" or mtype == "silent-map":
 461                 (source, dest) = args[1:3]
 462                 if self.pkg.changes["distribution"].has_key(source):
 463                     del self.pkg.changes["distribution"][source]
 464                     self.pkg.changes["distribution"][dest] = 1
 465                     if mtype != "silent-map":
 466                         self.notes.append("Mapping %s to %s." % (source, dest))
 467                 if self.pkg.changes.has_key("distribution-version"):
 468                     if self.pkg.changes["distribution-version"].has_key(source):
 469                         self.pkg.changes["distribution-version"][source]=dest
 470             elif mtype == "map-unreleased":
 471                 (source, dest) = args[1:3]
 472                 if self.pkg.changes["distribution"].has_key(source):
 473                     for arch in self.pkg.changes["architecture"].keys():
 474                         if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
 475                             self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
 476                             del self.pkg.changes["distribution"][source]
 477                             self.pkg.changes["distribution"][dest] = 1
 478                             break
 479             elif mtype == "ignore":
 480                 suite = args[1]
 481                 if self.pkg.changes["distribution"].has_key(suite):
 482                     del self.pkg.changes["distribution"][suite]
 483                     self.warnings.append("Ignoring %s as a target suite." % (suite))
 484             elif mtype == "reject":
 485                 suite = args[1]
 486                 if self.pkg.changes["distribution"].has_key(suite):
 487                     self.rejects.append("Uploads to %s are not accepted." % (suite))
 488             elif mtype == "propup-version":
 489                 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
 490                 #
 491                 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
 492                 if self.pkg.changes["distribution"].has_key(args[1]):
 493                     self.pkg.changes.setdefault("distribution-version", {})
 494                     for suite in args[2:]:
 495                         self.pkg.changes["distribution-version"][suite] = suite
 496
 497         # Ensure there is (still) a target distribution
 498         if len(self.pkg.changes["distribution"].keys()) < 1:
 499             self.rejects.append("No valid distribution remaining.")
 500
 501         # Ensure target distributions exist
 502         for suite in self.pkg.changes["distribution"].keys():
 503             if not Cnf.has_key("Suite::%s" % (suite)):
 504                 self.rejects.append("Unknown distribution `%s'." % (suite))
 505
 506     ###########################################################################
 507
 508     def binary_file_checks(self, f, session):
 509         cnf = Config()
 510         entry = self.pkg.files[f]
 511
 512         # Extract package control information
 513         deb_file = utils.open_file(f)
 514         try:
 515             control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
 516         except:
 517             self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
 518             deb_file.close()
 519             # Can't continue, none of the checks on control would work.
 520             return
 521
 522         # Check for mandantory "Description:"
 523         deb_file.seek(0)
 524         try:
 525             apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
 526         except:
 527             self.rejects.append("%s: Missing Description in binary package" % (f))
 528             return
 529
 530         deb_file.close()
 531
 532         # Check for mandatory fields
 533         for field in [ "Package", "Architecture", "Version" ]:
 534             if control.Find(field) == None:
 535                 # Can't continue
 536                 self.rejects.append("%s: No %s field in control." % (f, field))
 537                 return
 538
 539         # Ensure the package name matches the one give in the .changes
 540         if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
 541             self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
 542
 543         # Validate the package field
 544         package = control.Find("Package")
 545         if not re_valid_pkg_name.match(package):
 546             self.rejects.append("%s: invalid package name '%s'." % (f, package))
 547
 548         # Validate the version field
 549         version = control.Find("Version")
 550         if not re_valid_version.match(version):
 551             self.rejects.append("%s: invalid version number '%s'." % (f, version))
 552
 553         # Ensure the architecture of the .deb is one we know about.
 554         default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
 555         architecture = control.Find("Architecture")
 556         upload_suite = self.pkg.changes["distribution"].keys()[0]
 557
 558         if      architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
 559             and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
 560             self.rejects.append("Unknown architecture '%s'." % (architecture))
 561
 562         # Ensure the architecture of the .deb is one of the ones
 563         # listed in the .changes.
 564         if not self.pkg.changes["architecture"].has_key(architecture):
 565             self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
 566
 567         # Sanity-check the Depends field
 568         depends = control.Find("Depends")
 569         if depends == '':
 570             self.rejects.append("%s: Depends field is empty." % (f))
 571
 572         # Sanity-check the Provides field
 573         provides = control.Find("Provides")
 574         if provides:
 575             provide = re_spacestrip.sub('', provides)
 576             if provide == '':
 577                 self.rejects.append("%s: Provides field is empty." % (f))
 578             prov_list = provide.split(",")
 579             for prov in prov_list:
 580                 if not re_valid_pkg_name.match(prov):
 581                     self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
 582
 583         # Check the section & priority match those given in the .changes (non-fatal)
 584         if     control.Find("Section") and entry["section"] != "" \
 585            and entry["section"] != control.Find("Section"):
 586             self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
 587                                 (f, control.Find("Section", ""), entry["section"]))
 588         if control.Find("Priority") and entry["priority"] != "" \
 589            and entry["priority"] != control.Find("Priority"):
 590             self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
 591                                 (f, control.Find("Priority", ""), entry["priority"]))
 592
 593         entry["package"] = package
 594         entry["architecture"] = architecture
 595         entry["version"] = version
 596         entry["maintainer"] = control.Find("Maintainer", "")
 597
 598         if f.endswith(".udeb"):
 599             self.pkg.files[f]["dbtype"] = "udeb"
 600         elif f.endswith(".deb"):
 601             self.pkg.files[f]["dbtype"] = "deb"
 602         else:
 603             self.rejects.append("%s is neither a .deb or a .udeb." % (f))
 604
 605         entry["source"] = control.Find("Source", entry["package"])
 606
 607         # Get the source version
 608         source = entry["source"]
 609         source_version = ""
 610
 611         if source.find("(") != -1:
 612             m = re_extract_src_version.match(source)
 613             source = m.group(1)
 614             source_version = m.group(2)
 615
 616         if not source_version:
 617             source_version = self.pkg.files[f]["version"]
 618
 619         entry["source package"] = source
 620         entry["source version"] = source_version
 621
 622         # Ensure the filename matches the contents of the .deb
 623         m = re_isadeb.match(f)
 624
 625         #  package name
 626         file_package = m.group(1)
 627         if entry["package"] != file_package:
 628             self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
 629                                 (f, file_package, entry["dbtype"], entry["package"]))
 630         epochless_version = re_no_epoch.sub('', control.Find("Version"))
 631
 632         #  version
 633         file_version = m.group(2)
 634         if epochless_version != file_version:
 635             self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
 636                                 (f, file_version, entry["dbtype"], epochless_version))
 637
 638         #  architecture
 639         file_architecture = m.group(3)
 640         if entry["architecture"] != file_architecture:
 641             self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
 642                                 (f, file_architecture, entry["dbtype"], entry["architecture"]))
 643
 644         # Check for existent source
 645         source_version = entry["source version"]
 646         source_package = entry["source package"]
 647         if self.pkg.changes["architecture"].has_key("source"):
 648             if source_version != self.pkg.changes["version"]:
 649                 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
 650                                     (source_version, f, self.pkg.changes["version"]))
 651         else:
 652             # Check in the SQL database
 653             if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
 654                 # Check in one of the other directories
 655                 source_epochless_version = re_no_epoch.sub('', source_version)
 656                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 657                 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
 658                     entry["byhand"] = 1
 659                 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
 660                     entry["new"] = 1
 661                 else:
 662                     dsc_file_exists = False
 663                     for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
 664                         if cnf.has_key("Dir::Queue::%s" % (myq)):
 665                             if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
 666                                 dsc_file_exists = True
 667                                 break
 668
 669                     if not dsc_file_exists:
 670                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
 671
 672         # Check the version and for file overwrites
 673         self.check_binary_against_db(f, session)
 674
 675         # Temporarily disable contents generation until we change the table storage layout
 676         #b = Binary(f)
 677         #b.scan_package()
 678         #if len(b.rejects) > 0:
 679         #    for j in b.rejects:
 680         #        self.rejects.append(j)
 681
 682     def source_file_checks(self, f, session):
 683         entry = self.pkg.files[f]
 684
 685         m = re_issource.match(f)
 686         if not m:
 687             return
 688
 689         entry["package"] = m.group(1)
 690         entry["version"] = m.group(2)
 691         entry["type"] = m.group(3)
 692
 693         # Ensure the source package name matches the Source filed in the .changes
 694         if self.pkg.changes["source"] != entry["package"]:
 695             self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
 696
 697         # Ensure the source version matches the version in the .changes file
 698         if re_is_orig_source.match(f):
 699             changes_version = self.pkg.changes["chopversion2"]
 700         else:
 701             changes_version = self.pkg.changes["chopversion"]
 702
 703         if changes_version != entry["version"]:
 704             self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
 705
 706         # Ensure the .changes lists source in the Architecture field
 707         if not self.pkg.changes["architecture"].has_key("source"):
 708             self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
 709
 710         # Check the signature of a .dsc file
 711         if entry["type"] == "dsc":
 712             # check_signature returns either:
 713             #  (None, [list, of, rejects]) or (signature, [])
 714             (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
 715             for j in rejects:
 716                 self.rejects.append(j)
 717
 718         entry["architecture"] = "source"
 719
 720     def per_suite_file_checks(self, f, suite, session):
 721         cnf = Config()
 722         entry = self.pkg.files[f]
 723         archive = utils.where_am_i()
 724
 725         # Skip byhand
 726         if entry.has_key("byhand"):
 727             return
 728
 729         # Check we have fields we need to do these checks
 730         oktogo = True
 731         for m in ['component', 'package', 'priority', 'size', 'md5sum']:
 732             if not entry.has_key(m):
 733                 self.rejects.append("file '%s' does not have field %s set" % (f, m))
 734                 oktogo = False
 735
 736         if not oktogo:
 737             return
 738
 739         # Handle component mappings
 740         for m in cnf.ValueList("ComponentMappings"):
 741             (source, dest) = m.split()
 742             if entry["component"] == source:
 743                 entry["original component"] = source
 744                 entry["component"] = dest
 745
 746         # Ensure the component is valid for the target suite
 747         if cnf.has_key("Suite:%s::Components" % (suite)) and \
 748            entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
 749             self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
 750             return
 751
 752         # Validate the component
 753         if not get_component(entry["component"], session):
 754             self.rejects.append("file '%s' has unknown component '%s'." % (f, entry["component"]))
 755             return
 756
 757         # See if the package is NEW
 758         if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
 759             entry["new"] = 1
 760
 761         # Validate the priority
 762         if entry["priority"].find('/') != -1:
 763             self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
 764
 765         # Determine the location
 766         location = cnf["Dir::Pool"]
 767         l = get_location(location, entry["component"], archive, session)
 768         if l is None:
 769             self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (entry["component"], archive))
 770             entry["location id"] = -1
 771         else:
 772             entry["location id"] = l.location_id
 773
 774         # Check the md5sum & size against existing files (if any)
 775         entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
 776
 777         found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
 778                                          entry["size"], entry["md5sum"], entry["location id"])
 779
 780         if found is None:
 781             self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
 782         elif found is False and poolfile is not None:
 783             self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
 784         else:
 785             if poolfile is None:
 786                 entry["files id"] = None
 787             else:
 788                 entry["files id"] = poolfile.file_id
 789
 790         # Check for packages that have moved from one component to another
 791         entry['suite'] = suite
 792         res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
 793         if res.rowcount > 0:
 794             entry["othercomponents"] = res.fetchone()[0]
 795
 796     def check_files(self, action=True):
 797         file_keys = self.pkg.files.keys()
 798         holding = Holding()
 799         cnf = Config()
 800
 801         if action:
 802             cwd = os.getcwd()
 803             os.chdir(self.pkg.directory)
 804             for f in file_keys:
 805                 ret = holding.copy_to_holding(f)
 806                 if ret is not None:
 807                     # XXX: Should we bail out here or try and continue?
 808                     self.rejects.append(ret)
 809
 810             os.chdir(cwd)
 811
 812         # check we already know the changes file
 813         # [NB: this check must be done post-suite mapping]
 814         base_filename = os.path.basename(self.pkg.changes_file)
 815
 816         session = DBConn().session()
 817
 818         try:
 819             dbc = session.query(DBChange).filter_by(changesname=base_filename).one()
 820             # if in the pool or in a queue other than unchecked, reject
 821             if (dbc.in_queue is None) \
 822                    or (dbc.in_queue is not None
 823                        and dbc.in_queue.queue_name != 'unchecked'):
 824                 self.rejects.append("%s file already known to dak" % base_filename)
 825         except NoResultFound, e:
 826             # not known, good
 827             pass
 828
 829         has_binaries = False
 830         has_source = False
 831
 832         for f, entry in self.pkg.files.items():
 833             # Ensure the file does not already exist in one of the accepted directories
 834             for d in [ "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
 835                 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
 836                 if os.path.exists(os.path.join(cnf["Dir::Queue::%s" % (d) ], f)):
 837                     self.rejects.append("%s file already exists in the %s directory." % (f, d))
 838
 839             if not re_taint_free.match(f):
 840                 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
 841
 842             # Check the file is readable
 843             if os.access(f, os.R_OK) == 0:
 844                 # When running in -n, copy_to_holding() won't have
 845                 # generated the reject_message, so we need to.
 846                 if action:
 847                     if os.path.exists(f):
 848                         self.rejects.append("Can't read `%s'. [permission denied]" % (f))
 849                     else:
 850                         self.rejects.append("Can't read `%s'. [file not found]" % (f))
 851                 entry["type"] = "unreadable"
 852                 continue
 853
 854             # If it's byhand skip remaining checks
 855             if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
 856                 entry["byhand"] = 1
 857                 entry["type"] = "byhand"
 858
 859             # Checks for a binary package...
 860             elif re_isadeb.match(f):
 861                 has_binaries = True
 862                 entry["type"] = "deb"
 863
 864                 # This routine appends to self.rejects/warnings as appropriate
 865                 self.binary_file_checks(f, session)
 866
 867             # Checks for a source package...
 868             elif re_issource.match(f):
 869                 has_source = True
 870
 871                 # This routine appends to self.rejects/warnings as appropriate
 872                 self.source_file_checks(f, session)
 873
 874             # Not a binary or source package?  Assume byhand...
 875             else:
 876                 entry["byhand"] = 1
 877                 entry["type"] = "byhand"
 878
 879             # Per-suite file checks
 880             entry["oldfiles"] = {}
 881             for suite in self.pkg.changes["distribution"].keys():
 882                 self.per_suite_file_checks(f, suite, session)
 883
 884         session.close()
 885
 886         # If the .changes file says it has source, it must have source.
 887         if self.pkg.changes["architecture"].has_key("source"):
 888             if not has_source:
 889                 self.rejects.append("no source found and Architecture line in changes mention source.")
 890
 891             if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
 892                 self.rejects.append("source only uploads are not supported.")
 893
 894     ###########################################################################
 895     def check_dsc(self, action=True, session=None):
 896         """Returns bool indicating whether or not the source changes are valid"""
 897         # Ensure there is source to check
 898         if not self.pkg.changes["architecture"].has_key("source"):
 899             return True
 900
 901         # Find the .dsc
 902         dsc_filename = None
 903         for f, entry in self.pkg.files.items():
 904             if entry["type"] == "dsc":
 905                 if dsc_filename:
 906                     self.rejects.append("can not process a .changes file with multiple .dsc's.")
 907                     return False
 908                 else:
 909                     dsc_filename = f
 910
 911         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
 912         if not dsc_filename:
 913             self.rejects.append("source uploads must contain a dsc file")
 914             return False
 915
 916         # Parse the .dsc file
 917         try:
 918             self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
 919         except CantOpenError:
 920             # if not -n copy_to_holding() will have done this for us...
 921             if not action:
 922                 self.rejects.append("%s: can't read file." % (dsc_filename))
 923         except ParseChangesError, line:
 924             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
 925         except InvalidDscError, line:
 926             self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
 927         except ChangesUnicodeError:
 928             self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
 929
 930         # Build up the file list of files mentioned by the .dsc
 931         try:
 932             self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
 933         except NoFilesFieldError:
 934             self.rejects.append("%s: no Files: field." % (dsc_filename))
 935             return False
 936         except UnknownFormatError, format:
 937             self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
 938             return False
 939         except ParseChangesError, line:
 940             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
 941             return False
 942
 943         # Enforce mandatory fields
 944         for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
 945             if not self.pkg.dsc.has_key(i):
 946                 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
 947                 return False
 948
 949         # Validate the source and version fields
 950         if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
 951             self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
 952         if not re_valid_version.match(self.pkg.dsc["version"]):
 953             self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
 954
 955         # Only a limited list of source formats are allowed in each suite
 956         for dist in self.pkg.changes["distribution"].keys():
 957             allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
 958             if self.pkg.dsc["format"] not in allowed:
 959                 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
 960
 961         # Validate the Maintainer field
 962         try:
 963             # We ignore the return value
 964             fix_maintainer(self.pkg.dsc["maintainer"])
 965         except ParseMaintError, msg:
 966             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
 967                                  % (dsc_filename, self.pkg.dsc["maintainer"], msg))
 968
 969         # Validate the build-depends field(s)
 970         for field_name in [ "build-depends", "build-depends-indep" ]:
 971             field = self.pkg.dsc.get(field_name)
 972             if field:
 973                 # Have apt try to parse them...
 974                 try:
 975                     apt_pkg.ParseSrcDepends(field)
 976                 except:
 977                     self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
 978
 979         # Ensure the version number in the .dsc matches the version number in the .changes
 980         epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
 981         changes_version = self.pkg.files[dsc_filename]["version"]
 982
 983         if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
 984             self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
 985
 986         # Ensure the Files field contain only what's expected
 987         self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
 988
 989         # Ensure source is newer than existing source in target suites
 990         session = DBConn().session()
 991         self.check_source_against_db(dsc_filename, session)
 992         self.check_dsc_against_db(dsc_filename, session)
 993         session.close()
 994
 995         return True
 996
 997     ###########################################################################
 998
 999     def get_changelog_versions(self, source_dir):
1000         """Extracts a the source package and (optionally) grabs the
1001         version history out of debian/changelog for the BTS."""
1002
1003         cnf = Config()
1004
1005         # Find the .dsc (again)
1006         dsc_filename = None
1007         for f in self.pkg.files.keys():
1008             if self.pkg.files[f]["type"] == "dsc":
1009                 dsc_filename = f
1010
1011         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1012         if not dsc_filename:
1013             return
1014
1015         # Create a symlink mirror of the source files in our temporary directory
1016         for f in self.pkg.files.keys():
1017             m = re_issource.match(f)
1018             if m:
1019                 src = os.path.join(source_dir, f)
1020                 # If a file is missing for whatever reason, give up.
1021                 if not os.path.exists(src):
1022                     return
1023                 ftype = m.group(3)
1024                 if re_is_orig_source.match(f) and self.pkg.orig_files.has_key(f) and \
1025                    self.pkg.orig_files[f].has_key("path"):
1026                     continue
1027                 dest = os.path.join(os.getcwd(), f)
1028                 os.symlink(src, dest)
1029
1030         # If the orig files are not a part of the upload, create symlinks to the
1031         # existing copies.
1032         for orig_file in self.pkg.orig_files.keys():
1033             if not self.pkg.orig_files[orig_file].has_key("path"):
1034                 continue
1035             dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
1036             os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
1037
1038         # Extract the source
1039         cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1040         (result, output) = commands.getstatusoutput(cmd)
1041         if (result != 0):
1042             self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1043             self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "))
1044             return
1045
1046         if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1047             return
1048
1049         # Get the upstream version
1050         upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1051         if re_strip_revision.search(upstr_version):
1052             upstr_version = re_strip_revision.sub('', upstr_version)
1053
1054         # Ensure the changelog file exists
1055         changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1056         if not os.path.exists(changelog_filename):
1057             self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1058             return
1059
1060         # Parse the changelog
1061         self.pkg.dsc["bts changelog"] = ""
1062         changelog_file = utils.open_file(changelog_filename)
1063         for line in changelog_file.readlines():
1064             m = re_changelog_versions.match(line)
1065             if m:
1066                 self.pkg.dsc["bts changelog"] += line
1067         changelog_file.close()
1068
1069         # Check we found at least one revision in the changelog
1070         if not self.pkg.dsc["bts changelog"]:
1071             self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1072
1073     def check_source(self):
1074         # Bail out if:
1075         #    a) there's no source
1076         # or c) the orig files are MIA
1077         if not self.pkg.changes["architecture"].has_key("source") \
1078            or len(self.pkg.orig_files) == 0:
1079             return
1080
1081         tmpdir = utils.temp_dirname()
1082
1083         # Move into the temporary directory
1084         cwd = os.getcwd()
1085         os.chdir(tmpdir)
1086
1087         # Get the changelog version history
1088         self.get_changelog_versions(cwd)
1089
1090         # Move back and cleanup the temporary tree
1091         os.chdir(cwd)
1092
1093         try:
1094             shutil.rmtree(tmpdir)
1095         except OSError, e:
1096             if e.errno != errno.EACCES:
1097                 print "foobar"
1098                 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1099
1100             self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1101             # We probably have u-r or u-w directories so chmod everything
1102             # and try again.
1103             cmd = "chmod -R u+rwx %s" % (tmpdir)
1104             result = os.system(cmd)
1105             if result != 0:
1106                 utils.fubar("'%s' failed with result %s." % (cmd, result))
1107             shutil.rmtree(tmpdir)
1108         except Exception, e:
1109             print "foobar2 (%s)" % e
1110             utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1111
1112     ###########################################################################
1113     def ensure_hashes(self):
1114         # Make sure we recognise the format of the Files: field in the .changes
1115         format = self.pkg.changes.get("format", "0.0").split(".", 1)
1116         if len(format) == 2:
1117             format = int(format[0]), int(format[1])
1118         else:
1119             format = int(float(format[0])), 0
1120
1121         # We need to deal with the original changes blob, as the fields we need
1122         # might not be in the changes dict serialised into the .dak anymore.
1123         orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1124
1125         # Copy the checksums over to the current changes dict.  This will keep
1126         # the existing modifications to it intact.
1127         for field in orig_changes:
1128             if field.startswith('checksums-'):
1129                 self.pkg.changes[field] = orig_changes[field]
1130
1131         # Check for unsupported hashes
1132         for j in utils.check_hash_fields(".changes", self.pkg.changes):
1133             self.rejects.append(j)
1134
1135         for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1136             self.rejects.append(j)
1137
1138         # We have to calculate the hash if we have an earlier changes version than
1139         # the hash appears in rather than require it exist in the changes file
1140         for hashname, hashfunc, version in utils.known_hashes:
1141             # TODO: Move _ensure_changes_hash into this class
1142             for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1143                 self.rejects.append(j)
1144             if "source" in self.pkg.changes["architecture"]:
1145                 # TODO: Move _ensure_dsc_hash into this class
1146                 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1147                     self.rejects.append(j)
1148
1149     def check_hashes(self):
1150         for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1151             self.rejects.append(m)
1152
1153         for m in utils.check_size(".changes", self.pkg.files):
1154             self.rejects.append(m)
1155
1156         for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1157             self.rejects.append(m)
1158
1159         for m in utils.check_size(".dsc", self.pkg.dsc_files):
1160             self.rejects.append(m)
1161
1162         self.ensure_hashes()
1163
1164     ###########################################################################
1165
1166     def ensure_orig(self, target_dir='.', session=None):
1167         """
1168         Ensures that all orig files mentioned in the changes file are present
1169         in target_dir. If they do not exist, they are symlinked into place.
1170
1171         An list containing the symlinks that were created are returned (so they
1172         can be removed).
1173         """
1174
1175         symlinked = []
1176         cnf = Config()
1177
1178         for filename, entry in self.pkg.dsc_files.iteritems():
1179             if not re_is_orig_source.match(filename):
1180                 # File is not an orig; ignore
1181                 continue
1182
1183             if os.path.exists(filename):
1184                 # File exists, no need to continue
1185                 continue
1186
1187             def symlink_if_valid(path):
1188                 f = utils.open_file(path)
1189                 md5sum = apt_pkg.md5sum(f)
1190                 f.close()
1191
1192                 fingerprint = (os.stat(path)[stat.ST_SIZE], md5sum)
1193                 expected = (int(entry['size']), entry['md5sum'])
1194
1195                 if fingerprint != expected:
1196                     return False
1197
1198                 dest = os.path.join(target_dir, filename)
1199
1200                 os.symlink(path, dest)
1201                 symlinked.append(dest)
1202
1203                 return True
1204
1205             session_ = session
1206             if session is None:
1207                 session_ = DBConn().session()
1208
1209             found = False
1210
1211             # Look in the pool
1212             for poolfile in get_poolfile_like_name('/%s' % filename, session_):
1213                 poolfile_path = os.path.join(
1214                     poolfile.location.path, poolfile.filename
1215                 )
1216
1217                 if symlink_if_valid(poolfile_path):
1218                     found = True
1219                     break
1220
1221             if session is None:
1222                 session_.close()
1223
1224             if found:
1225                 continue
1226
1227             # Look in some other queues for the file
1228             queues = ('Accepted', 'New', 'Byhand', 'ProposedUpdates',
1229                 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
1230
1231             for queue in queues:
1232                 if not cnf.get('Dir::Queue::%s' % queue):
1233                     continue
1234
1235                 queuefile_path = os.path.join(
1236                     cnf['Dir::Queue::%s' % queue], filename
1237                 )
1238
1239                 if not os.path.exists(queuefile_path):
1240                     # Does not exist in this queue
1241                     continue
1242
1243                 if symlink_if_valid(queuefile_path):
1244                     break
1245
1246         return symlinked
1247
1248     ###########################################################################
1249
1250     def check_lintian(self):
1251         cnf = Config()
1252
1253         # Don't reject binary uploads
1254         if not self.pkg.changes['architecture'].has_key('source'):
1255             return
1256
1257         # Only check some distributions
1258         valid_dist = False
1259         for dist in ('unstable', 'experimental'):
1260             if dist in self.pkg.changes['distribution']:
1261                 valid_dist = True
1262                 break
1263
1264         if not valid_dist:
1265             return
1266
1267         tagfile = cnf.get("Dinstall::LintianTags")
1268         if tagfile is None:
1269             # We don't have a tagfile, so just don't do anything.
1270             return
1271
1272         # Parse the yaml file
1273         sourcefile = file(tagfile, 'r')
1274         sourcecontent = sourcefile.read()
1275         sourcefile.close()
1276         try:
1277             lintiantags = yaml.load(sourcecontent)['lintian']
1278         except yaml.YAMLError, msg:
1279             utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
1280             return
1281
1282         # Try and find all orig mentioned in the .dsc
1283         symlinked = self.ensure_orig()
1284
1285         # Now setup the input file for lintian. lintian wants "one tag per line" only,
1286         # so put it together like it. We put all types of tags in one file and then sort
1287         # through lintians output later to see if its a fatal tag we detected, or not.
1288         # So we only run lintian once on all tags, even if we might reject on some, but not
1289         # reject on others.
1290         # Additionally build up a set of tags
1291         tags = set()
1292         (fd, temp_filename) = utils.temp_filename()
1293         temptagfile = os.fdopen(fd, 'w')
1294         for tagtype in lintiantags:
1295             for tag in lintiantags[tagtype]:
1296                 temptagfile.write("%s\n" % tag)
1297                 tags.add(tag)
1298         temptagfile.close()
1299
1300         # So now we should look at running lintian at the .changes file, capturing output
1301         # to then parse it.
1302         command = "lintian --show-overrides --tags-from-file %s %s" % (temp_filename, self.pkg.changes_file)
1303         (result, output) = commands.getstatusoutput(command)
1304
1305         # We are done with lintian, remove our tempfile and any symlinks we created
1306         os.unlink(temp_filename)
1307         for symlink in symlinked:
1308             os.unlink(symlink)
1309
1310         if (result == 2):
1311             utils.warn("lintian failed for %s [return code: %s]." % (self.pkg.changes_file, result))
1312             utils.warn(utils.prefix_multi_line_string(output, " [possible output:] "))
1313
1314         if len(output) == 0:
1315             return
1316
1317         def log(*txt):
1318             if self.logger:
1319                 self.logger.log([self.pkg.changes_file, "check_lintian"] + list(txt))
1320
1321         # We have output of lintian, this package isn't clean. Lets parse it and see if we
1322         # are having a victim for a reject.
1323         # W: tzdata: binary-without-manpage usr/sbin/tzconfig
1324         for line in output.split('\n'):
1325             m = re_parse_lintian.match(line)
1326             if m is None:
1327                 continue
1328
1329             etype = m.group(1)
1330             epackage = m.group(2)
1331             etag = m.group(3)
1332             etext = m.group(4)
1333
1334             # So lets check if we know the tag at all.
1335             if etag not in tags:
1336                 continue
1337
1338             if etype == 'O':
1339                 # We know it and it is overriden. Check that override is allowed.
1340                 if etag in lintiantags['warning']:
1341                     # The tag is overriden, and it is allowed to be overriden.
1342                     # Don't add a reject message.
1343                     pass
1344                 elif etag in lintiantags['error']:
1345                     # The tag is overriden - but is not allowed to be
1346                     self.rejects.append("%s: Overriden tag %s found, but this tag may not be overwritten." % (epackage, etag))
1347                     log("ftpmaster does not allow tag to be overridable", etag)
1348             else:
1349                 # Tag is known, it is not overriden, direct reject.
1350                 self.rejects.append("%s: Found lintian output: '%s %s', automatically rejected package." % (epackage, etag, etext))
1351                 # Now tell if they *might* override it.
1352                 if etag in lintiantags['warning']:
1353                     log("auto rejecting", "overridable", etag)
1354                     self.rejects.append("%s: If you have a good reason, you may override this lintian tag." % (epackage))
1355                 else:
1356                     log("auto rejecting", "not overridable", etag)
1357
1358     ###########################################################################
1359     def check_urgency(self):
1360         cnf = Config()
1361         if self.pkg.changes["architecture"].has_key("source"):
1362             if not self.pkg.changes.has_key("urgency"):
1363                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1364             self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1365             if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1366                 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1367                                      (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1368                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1369
1370     ###########################################################################
1371
1372     # Sanity check the time stamps of files inside debs.
1373     # [Files in the near future cause ugly warnings and extreme time
1374     #  travel can cause errors on extraction]
1375
1376     def check_timestamps(self):
1377         Cnf = Config()
1378
1379         future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1380         past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1381         tar = TarTime(future_cutoff, past_cutoff)
1382
1383         for filename, entry in self.pkg.files.items():
1384             if entry["type"] == "deb":
1385                 tar.reset()
1386                 try:
1387                     deb_file = utils.open_file(filename)
1388                     apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1389                     deb_file.seek(0)
1390                     try:
1391                         apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1392                     except SystemError, e:
1393                         # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1394                         if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1395                             raise
1396                         deb_file.seek(0)
1397                         apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1398
1399                     deb_file.close()
1400
1401                     future_files = tar.future_files.keys()
1402                     if future_files:
1403                         num_future_files = len(future_files)
1404                         future_file = future_files[0]
1405                         future_date = tar.future_files[future_file]
1406                         self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1407                                % (filename, num_future_files, future_file, time.ctime(future_date)))
1408
1409                     ancient_files = tar.ancient_files.keys()
1410                     if ancient_files:
1411                         num_ancient_files = len(ancient_files)
1412                         ancient_file = ancient_files[0]
1413                         ancient_date = tar.ancient_files[ancient_file]
1414                         self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1415                                % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1416                 except:
1417                     self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1418
1419     def check_if_upload_is_sponsored(self, uid_email, uid_name):
1420         if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1421             sponsored = False
1422         elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1423             sponsored = False
1424             if uid_name == "":
1425                 sponsored = True
1426         else:
1427             sponsored = True
1428             if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
1429                 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1430                 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1431                     self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1432                         self.pkg.changes["sponsoremail"] = uid_email
1433
1434         return sponsored
1435
1436
1437     ###########################################################################
1438     # check_signed_by_key checks
1439     ###########################################################################
1440
1441     def check_signed_by_key(self):
1442         """Ensure the .changes is signed by an authorized uploader."""
1443         session = DBConn().session()
1444
1445         # First of all we check that the person has proper upload permissions
1446         # and that this upload isn't blocked
1447         fpr = get_fingerprint(self.pkg.changes['fingerprint'], session=session)
1448
1449         if fpr is None:
1450             self.rejects.append("Cannot find fingerprint %s" % self.pkg.changes["fingerprint"])
1451             return
1452
1453         # TODO: Check that import-keyring adds UIDs properly
1454         if not fpr.uid:
1455             self.rejects.append("Cannot find uid for fingerprint %s.  Please contact ftpmaster@debian.org" % fpr.fingerprint)
1456             return
1457
1458         # Check that the fingerprint which uploaded has permission to do so
1459         self.check_upload_permissions(fpr, session)
1460
1461         # Check that this package is not in a transition
1462         self.check_transition(session)
1463
1464         session.close()
1465
1466
1467     def check_upload_permissions(self, fpr, session):
1468         # Check any one-off upload blocks
1469         self.check_upload_blocks(fpr, session)
1470
1471         # Start with DM as a special case
1472         # DM is a special case unfortunately, so we check it first
1473         # (keys with no source access get more access than DMs in one
1474         #  way; DMs can only upload for their packages whether source
1475         #  or binary, whereas keys with no access might be able to
1476         #  upload some binaries)
1477         if fpr.source_acl.access_level == 'dm':
1478             self.check_dm_upload(fpr, session)
1479         else:
1480             # Check source-based permissions for other types
1481             if self.pkg.changes["architecture"].has_key("source"):
1482                 if fpr.source_acl.access_level is None:
1483                     rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
1484                     rej += '\nPlease contact ftpmaster if you think this is incorrect'
1485                     self.rejects.append(rej)
1486                     return
1487             else:
1488                 # If not a DM, we allow full upload rights
1489                 uid_email = "%s@debian.org" % (fpr.uid.uid)
1490                 self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
1491
1492
1493         # Check binary upload permissions
1494         # By this point we know that DMs can't have got here unless they
1495         # are allowed to deal with the package concerned so just apply
1496         # normal checks
1497         if fpr.binary_acl.access_level == 'full':
1498             return
1499
1500         # Otherwise we're in the map case
1501         tmparches = self.pkg.changes["architecture"].copy()
1502         tmparches.pop('source', None)
1503
1504         for bam in fpr.binary_acl_map:
1505             tmparches.pop(bam.architecture.arch_string, None)
1506
1507         if len(tmparches.keys()) > 0:
1508             if fpr.binary_reject:
1509                 rej = ".changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
1510                 rej += "\narchitectures involved are: ", ",".join(tmparches.keys())
1511                 self.rejects.append(rej)
1512             else:
1513                 # TODO: This is where we'll implement reject vs throw away binaries later
1514                 rej = "Uhm.  I'm meant to throw away the binaries now but that's not implemented yet"
1515                 rej += "\nPlease complain to ftpmaster@debian.org as this shouldn't have been turned on"
1516                 rej += "\nFingerprint: %s", (fpr.fingerprint)
1517                 self.rejects.append(rej)
1518
1519
1520     def check_upload_blocks(self, fpr, session):
1521         """Check whether any upload blocks apply to this source, source
1522            version, uid / fpr combination"""
1523
1524         def block_rej_template(fb):
1525             rej = 'Manual upload block in place for package %s' % fb.source
1526             if fb.version is not None:
1527                 rej += ', version %s' % fb.version
1528             return rej
1529
1530         for fb in session.query(UploadBlock).filter_by(source = self.pkg.changes['source']).all():
1531             # version is None if the block applies to all versions
1532             if fb.version is None or fb.version == self.pkg.changes['version']:
1533                 # Check both fpr and uid - either is enough to cause a reject
1534                 if fb.fpr is not None:
1535                     if fb.fpr.fingerprint == fpr.fingerprint:
1536                         self.rejects.append(block_rej_template(fb) + ' for fingerprint %s\nReason: %s' % (fpr.fingerprint, fb.reason))
1537                 if fb.uid is not None:
1538                     if fb.uid == fpr.uid:
1539                         self.rejects.append(block_rej_template(fb) + ' for uid %s\nReason: %s' % (fb.uid.uid, fb.reason))
1540
1541
1542     def check_dm_upload(self, fpr, session):
1543         # Quoth the GR (http://www.debian.org/vote/2007/vote_003):
1544         ## none of the uploaded packages are NEW
1545         rej = False
1546         for f in self.pkg.files.keys():
1547             if self.pkg.files[f].has_key("byhand"):
1548                 self.rejects.append("%s may not upload BYHAND file %s" % (fpr.uid.uid, f))
1549                 rej = True
1550             if self.pkg.files[f].has_key("new"):
1551                 self.rejects.append("%s may not upload NEW file %s" % (fpr.uid.uid, f))
1552                 rej = True
1553
1554         if rej:
1555             return
1556
1557         ## the most recent version of the package uploaded to unstable or
1558         ## experimental includes the field "DM-Upload-Allowed: yes" in the source
1559         ## section of its control file
1560         q = session.query(DBSource).filter_by(source=self.pkg.changes["source"])
1561         q = q.join(SrcAssociation)
1562         q = q.join(Suite).filter(Suite.suite_name.in_(['unstable', 'experimental']))
1563         q = q.order_by(desc('source.version')).limit(1)
1564
1565         r = q.all()
1566
1567         if len(r) != 1:
1568             rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"]
1569             self.rejects.append(rej)
1570             return
1571
1572         r = r[0]
1573         if not r.dm_upload_allowed:
1574             rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version)
1575             self.rejects.append(rej)
1576             return
1577
1578         ## the Maintainer: field of the uploaded .changes file corresponds with
1579         ## the owner of the key used (ie, non-developer maintainers may not sponsor
1580         ## uploads)
1581         if self.check_if_upload_is_sponsored(fpr.uid.uid, fpr.uid.name):
1582             self.rejects.append("%s (%s) is not authorised to sponsor uploads" % (fpr.uid.uid, fpr.fingerprint))
1583
1584         ## the most recent version of the package uploaded to unstable or
1585         ## experimental lists the uploader in the Maintainer: or Uploaders: fields (ie,
1586         ## non-developer maintainers cannot NMU or hijack packages)
1587
1588         # srcuploaders includes the maintainer
1589         accept = False
1590         for sup in r.srcuploaders:
1591             (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1592             # Eww - I hope we never have two people with the same name in Debian
1593             if email == fpr.uid.uid or name == fpr.uid.name:
1594                 accept = True
1595                 break
1596
1597         if not accept:
1598             self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (fpr.uid.uid, self.pkg.changes["source"]))
1599             return
1600
1601         ## none of the packages are being taken over from other source packages
1602         for b in self.pkg.changes["binary"].keys():
1603             for suite in self.pkg.changes["distribution"].keys():
1604                 q = session.query(DBSource)
1605                 q = q.join(DBBinary).filter_by(package=b)
1606                 q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1607
1608                 for s in q.all():
1609                     if s.source != self.pkg.changes["source"]:
1610                         self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite))
1611
1612
1613
1614     def check_transition(self, session):
1615         cnf = Config()
1616
1617         sourcepkg = self.pkg.changes["source"]
1618
1619         # No sourceful upload -> no need to do anything else, direct return
1620         # We also work with unstable uploads, not experimental or those going to some
1621         # proposed-updates queue
1622         if "source" not in self.pkg.changes["architecture"] or \
1623            "unstable" not in self.pkg.changes["distribution"]:
1624             return
1625
1626         # Also only check if there is a file defined (and existant) with
1627         # checks.
1628         transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1629         if transpath == "" or not os.path.exists(transpath):
1630             return
1631
1632         # Parse the yaml file
1633         sourcefile = file(transpath, 'r')
1634         sourcecontent = sourcefile.read()
1635         try:
1636             transitions = yaml.load(sourcecontent)
1637         except yaml.YAMLError, msg:
1638             # This shouldn't happen, there is a wrapper to edit the file which
1639             # checks it, but we prefer to be safe than ending up rejecting
1640             # everything.
1641             utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1642             return
1643
1644         # Now look through all defined transitions
1645         for trans in transitions:
1646             t = transitions[trans]
1647             source = t["source"]
1648             expected = t["new"]
1649
1650             # Will be None if nothing is in testing.
1651             current = get_source_in_suite(source, "testing", session)
1652             if current is not None:
1653                 compare = apt_pkg.VersionCompare(current.version, expected)
1654
1655             if current is None or compare < 0:
1656                 # This is still valid, the current version in testing is older than
1657                 # the new version we wait for, or there is none in testing yet
1658
1659                 # Check if the source we look at is affected by this.
1660                 if sourcepkg in t['packages']:
1661                     # The source is affected, lets reject it.
1662
1663                     rejectmsg = "%s: part of the %s transition.\n\n" % (
1664                         sourcepkg, trans)
1665
1666                     if current is not None:
1667                         currentlymsg = "at version %s" % (current.version)
1668                     else:
1669                         currentlymsg = "not present in testing"
1670
1671                     rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1672
1673                     rejectmsg += "\n".join(textwrap.wrap("""Your package
1674 is part of a testing transition designed to get %s migrated (it is
1675 currently %s, we need version %s).  This transition is managed by the
1676 Release Team, and %s is the Release-Team member responsible for it.
1677 Please mail debian-release@lists.debian.org or contact %s directly if you
1678 need further assistance.  You might want to upload to experimental until this
1679 transition is done."""
1680                             % (source, currentlymsg, expected,t["rm"], t["rm"])))
1681
1682                     self.rejects.append(rejectmsg)
1683                     return
1684
1685     ###########################################################################
1686     # End check_signed_by_key checks
1687     ###########################################################################
1688
1689     def build_summaries(self):
1690         """ Build a summary of changes the upload introduces. """
1691
1692         (byhand, new, summary, override_summary) = self.pkg.file_summary()
1693
1694         short_summary = summary
1695
1696         # This is for direport's benefit...
1697         f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1698
1699         if byhand or new:
1700             summary += "Changes: " + f
1701
1702         summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1703
1704         summary += self.announce(short_summary, 0)
1705
1706         return (summary, short_summary)
1707
1708     ###########################################################################
1709
1710     def close_bugs(self, summary, action):
1711         """
1712         Send mail to close bugs as instructed by the closes field in the changes file.
1713         Also add a line to summary if any work was done.
1714
1715         @type summary: string
1716         @param summary: summary text, as given by L{build_summaries}
1717
1718         @type action: bool
1719         @param action: Set to false no real action will be done.
1720
1721         @rtype: string
1722         @return: summary. If action was taken, extended by the list of closed bugs.
1723
1724         """
1725
1726         template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1727
1728         bugs = self.pkg.changes["closes"].keys()
1729
1730         if not bugs:
1731             return summary
1732
1733         bugs.sort()
1734         summary += "Closing bugs: "
1735         for bug in bugs:
1736             summary += "%s " % (bug)
1737             if action:
1738                 self.update_subst()
1739                 self.Subst["__BUG_NUMBER__"] = bug
1740                 if self.pkg.changes["distribution"].has_key("stable"):
1741                     self.Subst["__STABLE_WARNING__"] = """
1742 Note that this package is not part of the released stable Debian
1743 distribution.  It may have dependencies on other unreleased software,
1744 or other instabilities.  Please take care if you wish to install it.
1745 The update will eventually make its way into the next released Debian
1746 distribution."""
1747                 else:
1748                     self.Subst["__STABLE_WARNING__"] = ""
1749                 mail_message = utils.TemplateSubst(self.Subst, template)
1750                 utils.send_mail(mail_message)
1751
1752                 # Clear up after ourselves
1753                 del self.Subst["__BUG_NUMBER__"]
1754                 del self.Subst["__STABLE_WARNING__"]
1755
1756         if action and self.logger:
1757             self.logger.log(["closing bugs"] + bugs)
1758
1759         summary += "\n"
1760
1761         return summary
1762
1763     ###########################################################################
1764
1765     def announce(self, short_summary, action):
1766         """
1767         Send an announce mail about a new upload.
1768
1769         @type short_summary: string
1770         @param short_summary: Short summary text to include in the mail
1771
1772         @type action: bool
1773         @param action: Set to false no real action will be done.
1774
1775         @rtype: string
1776         @return: Textstring about action taken.
1777
1778         """
1779
1780         cnf = Config()
1781         announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1782
1783         # Only do announcements for source uploads with a recent dpkg-dev installed
1784         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1785            self.pkg.changes["architecture"].has_key("source"):
1786             return ""
1787
1788         lists_done = {}
1789         summary = ""
1790
1791         self.Subst["__SHORT_SUMMARY__"] = short_summary
1792
1793         for dist in self.pkg.changes["distribution"].keys():
1794             announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1795             if announce_list == "" or lists_done.has_key(announce_list):
1796                 continue
1797
1798             lists_done[announce_list] = 1
1799             summary += "Announcing to %s\n" % (announce_list)
1800
1801             if action:
1802                 self.update_subst()
1803                 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1804                 if cnf.get("Dinstall::TrackingServer") and \
1805                    self.pkg.changes["architecture"].has_key("source"):
1806                     trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1807                     self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1808
1809                 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1810                 utils.send_mail(mail_message)
1811
1812                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1813
1814         if cnf.FindB("Dinstall::CloseBugs"):
1815             summary = self.close_bugs(summary, action)
1816
1817         del self.Subst["__SHORT_SUMMARY__"]
1818
1819         return summary
1820
1821     ###########################################################################
1822     @session_wrapper
1823     def accept (self, summary, short_summary, session=None):
1824         """
1825         Accept an upload.
1826
1827         This moves all files referenced from the .changes into the pool,
1828         sends the accepted mail, announces to lists, closes bugs and
1829         also checks for override disparities. If enabled it will write out
1830         the version history for the BTS Version Tracking and will finally call
1831         L{queue_build}.
1832
1833         @type summary: string
1834         @param summary: Summary text
1835
1836         @type short_summary: string
1837         @param short_summary: Short summary
1838         """
1839
1840         cnf = Config()
1841         stats = SummaryStats()
1842
1843         print "Installing."
1844         self.logger.log(["installing changes", self.pkg.changes_file])
1845
1846         # Add the .dsc file to the DB first
1847         for newfile, entry in self.pkg.files.items():
1848             if entry["type"] == "dsc":
1849                 dsc_component, dsc_location_id = add_dsc_to_db(self, newfile, session)
1850
1851         # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
1852         for newfile, entry in self.pkg.files.items():
1853             if entry["type"] == "deb":
1854                 add_deb_to_db(self, newfile, session)
1855
1856         # If this is a sourceful diff only upload that is moving
1857         # cross-component we need to copy the .orig files into the new
1858         # component too for the same reasons as above.
1859         if self.pkg.changes["architecture"].has_key("source"):
1860             for orig_file in self.pkg.orig_files.keys():
1861                 if not self.pkg.orig_files[orig_file].has_key("id"):
1862                     continue # Skip if it's not in the pool
1863                 orig_file_id = self.pkg.orig_files[orig_file]["id"]
1864                 if self.pkg.orig_files[orig_file]["location"] == dsc_location_id:
1865                     continue # Skip if the location didn't change
1866
1867                 # Do the move
1868                 oldf = get_poolfile_by_id(orig_file_id, session)
1869                 old_filename = os.path.join(oldf.location.path, oldf.filename)
1870                 old_dat = {'size': oldf.filesize,   'md5sum': oldf.md5sum,
1871                            'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
1872
1873                 new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
1874
1875                 # TODO: Care about size/md5sum collisions etc
1876                 (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
1877
1878                 if newf is None:
1879                     utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
1880                     newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
1881
1882                     # TODO: Check that there's only 1 here
1883                     source = get_sources_from_name(self.pkg.changes["source"], self.pkg.changes["version"])[0]
1884                     dscf = get_dscfiles(source_id=source.source_id, poolfile_id=orig_file_id, session=session)[0]
1885                     dscf.poolfile_id = newf.file_id
1886                     session.add(dscf)
1887                     session.flush()
1888
1889         # Install the files into the pool
1890         for newfile, entry in self.pkg.files.items():
1891             destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
1892             utils.move(newfile, destination)
1893             self.logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
1894             stats.accept_bytes += float(entry["size"])
1895
1896         # Copy the .changes file across for suite which need it.
1897         copy_changes = {}
1898         for suite_name in self.pkg.changes["distribution"].keys():
1899             if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
1900                 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
1901
1902         for dest in copy_changes.keys():
1903             utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
1904
1905         # We're done - commit the database changes
1906         session.commit()
1907         # Our SQL session will automatically start a new transaction after
1908         # the last commit
1909
1910         # Move the .changes into the 'done' directory
1911         utils.move(self.pkg.changes_file,
1912                    os.path.join(cnf["Dir::Queue::Done"], os.path.basename(self.pkg.changes_file)))
1913
1914         if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"):
1915             UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])
1916
1917         # Send accept mail, announce to lists, close bugs and check for
1918         # override disparities
1919         if not cnf["Dinstall::Options::No-Mail"]:
1920             self.update_subst()
1921             self.Subst["__SUITE__"] = ""
1922             self.Subst["__SUMMARY__"] = summary
1923             mail_message = utils.TemplateSubst(self.Subst,
1924                                                os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted'))
1925             utils.send_mail(mail_message)
1926             self.announce(short_summary, 1)
1927
1928         ## Helper stuff for DebBugs Version Tracking
1929         if cnf.Find("Dir::Queue::BTSVersionTrack"):
1930             # ??? once queue/* is cleared on *.d.o and/or reprocessed
1931             # the conditionalization on dsc["bts changelog"] should be
1932             # dropped.
1933
1934             # Write out the version history from the changelog
1935             if self.pkg.changes["architecture"].has_key("source") and \
1936                self.pkg.dsc.has_key("bts changelog"):
1937
1938                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1939                 version_history = os.fdopen(fd, 'w')
1940                 version_history.write(self.pkg.dsc["bts changelog"])
1941                 version_history.close()
1942                 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1943                                       self.pkg.changes_file[:-8]+".versions")
1944                 os.rename(temp_filename, filename)
1945                 os.chmod(filename, 0644)
1946
1947             # Write out the binary -> source mapping.
1948             (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1949             debinfo = os.fdopen(fd, 'w')
1950             for name, entry in sorted(self.pkg.files.items()):
1951                 if entry["type"] == "deb":
1952                     line = " ".join([entry["package"], entry["version"],
1953                                      entry["architecture"], entry["source package"],
1954                                      entry["source version"]])
1955                     debinfo.write(line+"\n")
1956             debinfo.close()
1957             filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1958                                   self.pkg.changes_file[:-8]+".debinfo")
1959             os.rename(temp_filename, filename)
1960             os.chmod(filename, 0644)
1961
1962         # This routine returns None on success or an error on failure
1963         # TODO: Replace queue copying using the new queue.add_file_from_pool routine
1964         #       and by looking up which queues in suite.copy_queues
1965         #res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
1966         #if res:
1967         #    utils.fubar(res)
1968
1969         session.commit()
1970
1971         # Finally...
1972         stats.accept_count += 1
1973
1974     def check_override(self):
1975         """
1976         Checks override entries for validity. Mails "Override disparity" warnings,
1977         if that feature is enabled.
1978
1979         Abandons the check if
1980           - override disparity checks are disabled
1981           - mail sending is disabled
1982         """
1983
1984         cnf = Config()
1985
1986         # Abandon the check if:
1987         #  a) override disparity checks have been disabled
1988         #  b) we're not sending mail
1989         if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
1990            cnf["Dinstall::Options::No-Mail"]:
1991             return
1992
1993         summary = self.pkg.check_override()
1994
1995         if summary == "":
1996             return
1997
1998         overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
1999
2000         self.update_subst()
2001         self.Subst["__SUMMARY__"] = summary
2002         mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
2003         utils.send_mail(mail_message)
2004         del self.Subst["__SUMMARY__"]
2005
2006     ###########################################################################
2007
2008     def remove(self, from_dir=None):
2009         """
2010         Used (for instance) in p-u to remove the package from unchecked
2011
2012         Also removes the package from holding area.
2013         """
2014         if from_dir is None:
2015             from_dir = self.pkg.directory
2016         h = Holding()
2017
2018         for f in self.pkg.files.keys():
2019             os.unlink(os.path.join(from_dir, f))
2020             if os.path.exists(os.path.join(h.holding_dir, f)):
2021                 os.unlink(os.path.join(h.holding_dir, f))
2022
2023         os.unlink(os.path.join(from_dir, self.pkg.changes_file))
2024         if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
2025             os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
2026
2027     ###########################################################################
2028
2029     def move_to_dir (self, dest, perms=0660, changesperms=0664):
2030         """
2031         Move files to dest with certain perms/changesperms
2032         """
2033         h = Holding()
2034         utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
2035                    dest, perms=changesperms)
2036         for f in self.pkg.files.keys():
2037             utils.move(os.path.join(h.holding_dir, f), dest, perms=perms)
2038
2039     ###########################################################################
2040
2041     def force_reject(self, reject_files):
2042         """
2043         Forcefully move files from the current directory to the
2044         reject directory.  If any file already exists in the reject
2045         directory it will be moved to the morgue to make way for
2046         the new file.
2047
2048         @type files: dict
2049         @param files: file dictionary
2050
2051         """
2052
2053         cnf = Config()
2054
2055         for file_entry in reject_files:
2056             # Skip any files which don't exist or which we don't have permission to copy.
2057             if os.access(file_entry, os.R_OK) == 0:
2058                 continue
2059
2060             dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
2061
2062             try:
2063                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
2064             except OSError, e:
2065                 # File exists?  Let's try and move it to the morgue
2066                 if e.errno == errno.EEXIST:
2067                     morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
2068                     try:
2069                         morgue_file = utils.find_next_free(morgue_file)
2070                     except NoFreeFilenameError:
2071                         # Something's either gone badly Pete Tong, or
2072                         # someone is trying to exploit us.
2073                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
2074                         return
2075                     utils.move(dest_file, morgue_file, perms=0660)
2076                     try:
2077                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2078                     except OSError, e:
2079                         # Likewise
2080                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
2081                         return
2082                 else:
2083                     raise
2084             # If we got here, we own the destination file, so we can
2085             # safely overwrite it.
2086             utils.move(file_entry, dest_file, 1, perms=0660)
2087             os.close(dest_fd)
2088
2089     ###########################################################################
2090     def do_reject (self, manual=0, reject_message="", note=""):
2091         """
2092         Reject an upload. If called without a reject message or C{manual} is
2093         true, spawn an editor so the user can write one.
2094
2095         @type manual: bool
2096         @param manual: manual or automated rejection
2097
2098         @type reject_message: string
2099         @param reject_message: A reject message
2100
2101         @return: 0
2102
2103         """
2104         # If we weren't given a manual rejection message, spawn an
2105         # editor so the user can add one in...
2106         if manual and not reject_message:
2107             (fd, temp_filename) = utils.temp_filename()
2108             temp_file = os.fdopen(fd, 'w')
2109             if len(note) > 0:
2110                 for line in note:
2111                     temp_file.write(line)
2112             temp_file.close()
2113             editor = os.environ.get("EDITOR","vi")
2114             answer = 'E'
2115             while answer == 'E':
2116                 os.system("%s %s" % (editor, temp_filename))
2117                 temp_fh = utils.open_file(temp_filename)
2118                 reject_message = "".join(temp_fh.readlines())
2119                 temp_fh.close()
2120                 print "Reject message:"
2121                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1)
2122                 prompt = "[R]eject, Edit, Abandon, Quit ?"
2123                 answer = "XXX"
2124                 while prompt.find(answer) == -1:
2125                     answer = utils.our_raw_input(prompt)
2126                     m = re_default_answer.search(prompt)
2127                     if answer == "":
2128                         answer = m.group(1)
2129                     answer = answer[:1].upper()
2130             os.unlink(temp_filename)
2131             if answer == 'A':
2132                 return 1
2133             elif answer == 'Q':
2134                 sys.exit(0)
2135
2136         print "Rejecting.\n"
2137
2138         cnf = Config()
2139
2140         reason_filename = self.pkg.changes_file[:-8] + ".reason"
2141         reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2142
2143         # Move all the files into the reject directory
2144         reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
2145         self.force_reject(reject_files)
2146
2147         # If we fail here someone is probably trying to exploit the race
2148         # so let's just raise an exception ...
2149         if os.path.exists(reason_filename):
2150             os.unlink(reason_filename)
2151         reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2152
2153         rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
2154
2155         self.update_subst()
2156         if not manual:
2157             self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2158             self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
2159             self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)"
2160             os.write(reason_fd, reject_message)
2161             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2162         else:
2163             # Build up the rejection email
2164             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
2165             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
2166             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
2167             self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2168             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2169             # Write the rejection email out as the <foo>.reason file
2170             os.write(reason_fd, reject_mail_message)
2171
2172         del self.Subst["__REJECTOR_ADDRESS__"]
2173         del self.Subst["__MANUAL_REJECT_MESSAGE__"]
2174         del self.Subst["__CC__"]
2175
2176         os.close(reason_fd)
2177
2178         # Send the rejection mail if appropriate
2179         if not cnf["Dinstall::Options::No-Mail"]:
2180             utils.send_mail(reject_mail_message)
2181
2182         if self.logger:
2183             self.logger.log(["rejected", self.pkg.changes_file])
2184
2185         return 0
2186
2187     ################################################################################
2188     def in_override_p(self, package, component, suite, binary_type, filename, session):
2189         """
2190         Check if a package already has override entries in the DB
2191
2192         @type package: string
2193         @param package: package name
2194
2195         @type component: string
2196         @param component: database id of the component
2197
2198         @type suite: int
2199         @param suite: database id of the suite
2200
2201         @type binary_type: string
2202         @param binary_type: type of the package
2203
2204         @type filename: string
2205         @param filename: filename we check
2206
2207         @return: the database result. But noone cares anyway.
2208
2209         """
2210
2211         cnf = Config()
2212
2213         if binary_type == "": # must be source
2214             file_type = "dsc"
2215         else:
2216             file_type = binary_type
2217
2218         # Override suite name; used for example with proposed-updates
2219         if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
2220             suite = cnf["Suite::%s::OverrideSuite" % (suite)]
2221
2222         result = get_override(package, suite, component, file_type, session)
2223
2224         # If checking for a source package fall back on the binary override type
2225         if file_type == "dsc" and len(result) < 1:
2226             result = get_override(package, suite, component, ['deb', 'udeb'], session)
2227
2228         # Remember the section and priority so we can check them later if appropriate
2229         if len(result) > 0:
2230             result = result[0]
2231             self.pkg.files[filename]["override section"] = result.section.section
2232             self.pkg.files[filename]["override priority"] = result.priority.priority
2233             return result
2234
2235         return None
2236
2237     ################################################################################
2238     def get_anyversion(self, sv_list, suite):
2239         """
2240         @type sv_list: list
2241         @param sv_list: list of (suite, version) tuples to check
2242
2243         @type suite: string
2244         @param suite: suite name
2245
2246         Description: TODO
2247         """
2248         Cnf = Config()
2249         anyversion = None
2250         anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
2251         for (s, v) in sv_list:
2252             if s in [ x.lower() for x in anysuite ]:
2253                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
2254                     anyversion = v
2255
2256         return anyversion
2257
2258     ################################################################################
2259
2260     def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
2261         """
2262         @type sv_list: list
2263         @param sv_list: list of (suite, version) tuples to check
2264
2265         @type filename: string
2266         @param filename: XXX
2267
2268         @type new_version: string
2269         @param new_version: XXX
2270
2271         Ensure versions are newer than existing packages in target
2272         suites and that cross-suite version checking rules as
2273         set out in the conf file are satisfied.
2274         """
2275
2276         cnf = Config()
2277
2278         # Check versions for each target suite
2279         for target_suite in self.pkg.changes["distribution"].keys():
2280             must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
2281             must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
2282
2283             # Enforce "must be newer than target suite" even if conffile omits it
2284             if target_suite not in must_be_newer_than:
2285                 must_be_newer_than.append(target_suite)
2286
2287             for (suite, existent_version) in sv_list:
2288                 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
2289
2290                 if suite in must_be_newer_than and sourceful and vercmp < 1:
2291                     self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2292
2293                 if suite in must_be_older_than and vercmp > -1:
2294                     cansave = 0
2295
2296                     if self.pkg.changes.get('distribution-version', {}).has_key(suite):
2297                         # we really use the other suite, ignoring the conflicting one ...
2298                         addsuite = self.pkg.changes["distribution-version"][suite]
2299
2300                         add_version = self.get_anyversion(sv_list, addsuite)
2301                         target_version = self.get_anyversion(sv_list, target_suite)
2302
2303                         if not add_version:
2304                             # not add_version can only happen if we map to a suite
2305                             # that doesn't enhance the suite we're propup'ing from.
2306                             # so "propup-ver x a b c; map a d" is a problem only if
2307                             # d doesn't enhance a.
2308                             #
2309                             # i think we could always propagate in this case, rather
2310                             # than complaining. either way, this isn't a REJECT issue
2311                             #
2312                             # And - we really should complain to the dorks who configured dak
2313                             self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
2314                             self.pkg.changes.setdefault("propdistribution", {})
2315                             self.pkg.changes["propdistribution"][addsuite] = 1
2316                             cansave = 1
2317                         elif not target_version:
2318                             # not targets_version is true when the package is NEW
2319                             # we could just stick with the "...old version..." REJECT
2320                             # for this, I think.
2321                             self.rejects.append("Won't propogate NEW packages.")
2322                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2323                             # propogation would be redundant. no need to reject though.
2324                             self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2325                             cansave = 1
2326                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2327                              apt_pkg.VersionCompare(add_version, target_version) >= 0:
2328                             # propogate!!
2329                             self.warnings.append("Propogating upload to %s" % (addsuite))
2330                             self.pkg.changes.setdefault("propdistribution", {})
2331                             self.pkg.changes["propdistribution"][addsuite] = 1
2332                             cansave = 1
2333
2334                     if not cansave:
2335                         self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2336
2337     ################################################################################
2338     def check_binary_against_db(self, filename, session):
2339         # Ensure version is sane
2340         q = session.query(BinAssociation)
2341         q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[filename]["package"])
2342         q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[filename]["architecture"], 'all']))
2343
2344         self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2345                                        filename, self.pkg.files[filename]["version"], sourceful=False)
2346
2347         # Check for any existing copies of the file
2348         q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"])
2349         q = q.filter_by(version=self.pkg.files[filename]["version"])
2350         q = q.join(Architecture).filter_by(arch_string=self.pkg.files[filename]["architecture"])
2351
2352         if q.count() > 0:
2353             self.rejects.append("%s: can not overwrite existing copy already in the archive." % filename)
2354
2355     ################################################################################
2356
2357     def check_source_against_db(self, filename, session):
2358         """
2359         """
2360         source = self.pkg.dsc.get("source")
2361         version = self.pkg.dsc.get("version")
2362
2363         # Ensure version is sane
2364         q = session.query(SrcAssociation)
2365         q = q.join(DBSource).filter(DBSource.source==source)
2366
2367         self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2368                                        filename, version, sourceful=True)
2369
2370     ################################################################################
2371     def check_dsc_against_db(self, filename, session):
2372         """
2373
2374         @warning: NB: this function can remove entries from the 'files' index [if
2375          the orig tarball is a duplicate of the one in the archive]; if
2376          you're iterating over 'files' and call this function as part of
2377          the loop, be sure to add a check to the top of the loop to
2378          ensure you haven't just tried to dereference the deleted entry.
2379
2380         """
2381
2382         Cnf = Config()
2383         self.pkg.orig_files = {} # XXX: do we need to clear it?
2384         orig_files = self.pkg.orig_files
2385
2386         # Try and find all files mentioned in the .dsc.  This has
2387         # to work harder to cope with the multiple possible
2388         # locations of an .orig.tar.gz.
2389         # The ordering on the select is needed to pick the newest orig
2390         # when it exists in multiple places.
2391         for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2392             found = None
2393             if self.pkg.files.has_key(dsc_name):
2394                 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2395                 actual_size = int(self.pkg.files[dsc_name]["size"])
2396                 found = "%s in incoming" % (dsc_name)
2397
2398                 # Check the file does not already exist in the archive
2399                 ql = get_poolfile_like_name(dsc_name, session)
2400
2401                 # Strip out anything that isn't '%s' or '/%s$'
2402                 for i in ql:
2403                     if not i.filename.endswith(dsc_name):
2404                         ql.remove(i)
2405
2406                 # "[dak] has not broken them.  [dak] has fixed a
2407                 # brokenness.  Your crappy hack exploited a bug in
2408                 # the old dinstall.
2409                 #
2410                 # "(Come on!  I thought it was always obvious that
2411                 # one just doesn't release different files with
2412                 # the same name and version.)"
2413                 #                        -- ajk@ on d-devel@l.d.o
2414
2415                 if len(ql) > 0:
2416                     # Ignore exact matches for .orig.tar.gz
2417                     match = 0
2418                     if re_is_orig_source.match(dsc_name):
2419                         for i in ql:
2420                             if self.pkg.files.has_key(dsc_name) and \
2421                                int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2422                                self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2423                                 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2424                                 # TODO: Don't delete the entry, just mark it as not needed
2425                                 # This would fix the stupidity of changing something we often iterate over
2426                                 # whilst we're doing it
2427                                 del self.pkg.files[dsc_name]
2428                                 dsc_entry["files id"] = i.file_id
2429                                 if not orig_files.has_key(dsc_name):
2430                                     orig_files[dsc_name] = {}
2431                                 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
2432                                 match = 1
2433
2434                     if not match:
2435                         self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2436
2437             elif re_is_orig_source.match(dsc_name):
2438                 # Check in the pool
2439                 ql = get_poolfile_like_name(dsc_name, session)
2440
2441                 # Strip out anything that isn't '%s' or '/%s$'
2442                 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2443                 for i in ql:
2444                     if not i.filename.endswith(dsc_name):
2445                         ql.remove(i)
2446
2447                 if len(ql) > 0:
2448                     # Unfortunately, we may get more than one match here if,
2449                     # for example, the package was in potato but had an -sa
2450                     # upload in woody.  So we need to choose the right one.
2451
2452                     # default to something sane in case we don't match any or have only one
2453                     x = ql[0]
2454
2455                     if len(ql) > 1:
2456                         for i in ql:
2457                             old_file = os.path.join(i.location.path, i.filename)
2458                             old_file_fh = utils.open_file(old_file)
2459                             actual_md5 = apt_pkg.md5sum(old_file_fh)
2460                             old_file_fh.close()
2461                             actual_size = os.stat(old_file)[stat.ST_SIZE]
2462                             if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2463                                 x = i
2464
2465                     old_file = os.path.join(i.location.path, i.filename)
2466                     old_file_fh = utils.open_file(old_file)
2467                     actual_md5 = apt_pkg.md5sum(old_file_fh)
2468                     old_file_fh.close()
2469                     actual_size = os.stat(old_file)[stat.ST_SIZE]
2470                     found = old_file
2471                     suite_type = x.location.archive_type
2472                     # need this for updating dsc_files in install()
2473                     dsc_entry["files id"] = x.file_id
2474                     # See install() in process-accepted...
2475                     if not orig_files.has_key(dsc_name):
2476                         orig_files[dsc_name] = {}
2477                     orig_files[dsc_name]["id"] = x.file_id
2478                     orig_files[dsc_name]["path"] = old_file
2479                     orig_files[dsc_name]["location"] = x.location.location_id
2480                 else:
2481                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2482                     # Not there? Check the queue directories...
2483                     for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2484                         if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2485                             continue
2486                         in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2487                         if os.path.exists(in_otherdir):
2488                             in_otherdir_fh = utils.open_file(in_otherdir)
2489                             actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2490                             in_otherdir_fh.close()
2491                             actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2492                             found = in_otherdir
2493                             if not orig_files.has_key(dsc_name):
2494                                 orig_files[dsc_name] = {}
2495                             orig_files[dsc_name]["path"] = in_otherdir
2496
2497                     if not found:
2498                         self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (filename, dsc_name))
2499                         continue
2500             else:
2501                 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (filename, dsc_name))
2502                 continue
2503             if actual_md5 != dsc_entry["md5sum"]:
2504                 self.rejects.append("md5sum for %s doesn't match %s." % (found, filename))
2505             if actual_size != int(dsc_entry["size"]):
2506                 self.rejects.append("size for %s doesn't match %s." % (found, filename))
2507
2508     ################################################################################
2509     # This is used by process-new and process-holding to recheck a changes file
2510     # at the time we're running.  It mainly wraps various other internal functions
2511     # and is similar to accepted_checks - these should probably be tidied up
2512     # and combined
2513     def recheck(self, session):
2514         cnf = Config()
2515         for f in self.pkg.files.keys():
2516             # The .orig.tar.gz can disappear out from under us is it's a
2517             # duplicate of one in the archive.
2518             if not self.pkg.files.has_key(f):
2519                 continue
2520
2521             entry = self.pkg.files[f]
2522
2523             # Check that the source still exists
2524             if entry["type"] == "deb":
2525                 source_version = entry["source version"]
2526                 source_package = entry["source package"]
2527                 if not self.pkg.changes["architecture"].has_key("source") \
2528                    and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
2529                     source_epochless_version = re_no_epoch.sub('', source_version)
2530                     dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
2531                     found = False
2532                     for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
2533                         if cnf.has_key("Dir::Queue::%s" % (q)):
2534                             if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
2535                                 found = True
2536                     if not found:
2537                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
2538
2539             # Version and file overwrite checks
2540             if entry["type"] == "deb":
2541                 self.check_binary_against_db(f, session)
2542             elif entry["type"] == "dsc":
2543                 self.check_source_against_db(f, session)
2544                 self.check_dsc_against_db(f, session)
2545
2546     ################################################################################
2547     def accepted_checks(self, overwrite_checks, session):
2548         # Recheck anything that relies on the database; since that's not
2549         # frozen between accept and our run time when called from p-a.
2550
2551         # overwrite_checks is set to False when installing to stable/oldstable
2552
2553         propogate={}
2554         nopropogate={}
2555
2556         # Find the .dsc (again)
2557         dsc_filename = None
2558         for f in self.pkg.files.keys():
2559             if self.pkg.files[f]["type"] == "dsc":
2560                 dsc_filename = f
2561
2562         for checkfile in self.pkg.files.keys():
2563             # The .orig.tar.gz can disappear out from under us is it's a
2564             # duplicate of one in the archive.
2565             if not self.pkg.files.has_key(checkfile):
2566                 continue
2567
2568             entry = self.pkg.files[checkfile]
2569
2570             # Check that the source still exists
2571             if entry["type"] == "deb":
2572                 source_version = entry["source version"]
2573                 source_package = entry["source package"]
2574                 if not self.pkg.changes["architecture"].has_key("source") \
2575                    and not source_exists(source_package, source_version,  self.pkg.changes["distribution"].keys()):
2576                     self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2577
2578             # Version and file overwrite checks
2579             if overwrite_checks:
2580                 if entry["type"] == "deb":
2581                     self.check_binary_against_db(checkfile, session)
2582                 elif entry["type"] == "dsc":
2583                     self.check_source_against_db(checkfile, session)
2584                     self.check_dsc_against_db(dsc_filename, session)
2585
2586             # propogate in the case it is in the override tables:
2587             for suite in self.pkg.changes.get("propdistribution", {}).keys():
2588                 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2589                     propogate[suite] = 1
2590                 else:
2591                     nopropogate[suite] = 1
2592
2593         for suite in propogate.keys():
2594             if suite in nopropogate:
2595                 continue
2596             self.pkg.changes["distribution"][suite] = 1
2597
2598         for checkfile in self.pkg.files.keys():
2599             # Check the package is still in the override tables
2600             for suite in self.pkg.changes["distribution"].keys():
2601                 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2602                     self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2603
2604     ################################################################################
2605     # This is not really a reject, but an unaccept, but since a) the code for
2606     # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2607     # extremely rare, for now we'll go with whining at our admin folks...
2608
2609     def do_unaccept(self):
2610         cnf = Config()
2611
2612         self.update_subst()
2613         self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2614         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2615         self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2616         self.Subst["__BCC__"] = "X-DAK: dak process-accepted"
2617         if cnf.has_key("Dinstall::Bcc"):
2618             self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2619
2620         template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2621
2622         reject_mail_message = utils.TemplateSubst(self.Subst, template)
2623
2624         # Write the rejection email out as the <foo>.reason file
2625         reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2626         reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2627
2628         # If we fail here someone is probably trying to exploit the race
2629         # so let's just raise an exception ...
2630         if os.path.exists(reject_filename):
2631             os.unlink(reject_filename)
2632
2633         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2634         os.write(fd, reject_mail_message)
2635         os.close(fd)
2636
2637         utils.send_mail(reject_mail_message)
2638
2639         del self.Subst["__REJECTOR_ADDRESS__"]
2640         del self.Subst["__REJECT_MESSAGE__"]
2641         del self.Subst["__CC__"]
2642
2643     ################################################################################
2644     # If any file of an upload has a recent mtime then chances are good
2645     # the file is still being uploaded.
2646
2647     def upload_too_new(self):
2648         cnf = Config()
2649         too_new = False
2650         # Move back to the original directory to get accurate time stamps
2651         cwd = os.getcwd()
2652         os.chdir(self.pkg.directory)
2653         file_list = self.pkg.files.keys()
2654         file_list.extend(self.pkg.dsc_files.keys())
2655         file_list.append(self.pkg.changes_file)
2656         for f in file_list:
2657             try:
2658                 last_modified = time.time()-os.path.getmtime(f)
2659                 if last_modified < int(cnf["Dinstall::SkipTime"]):
2660                     too_new = True
2661                     break
2662             except:
2663                 pass
2664
2665         os.chdir(cwd)
2666         return too_new