daklib/queue.py

   1 #!/usr/bin/env python
   2 # vim:set et sw=4:
   3
   4 """
   5 Queue utility functions for dak
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
   9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ###############################################################################
  28
  29 import errno
  30 import os
  31 import pg
  32 import stat
  33 import sys
  34 import time
  35 import apt_inst
  36 import apt_pkg
  37 import utils
  38 import commands
  39 import shutil
  40 import textwrap
  41 from types import *
  42
  43 import yaml
  44
  45 from dak_exceptions import *
  46 from changes import *
  47 from regexes import *
  48 from config import Config
  49 from holding import Holding
  50 from dbconn import *
  51 from summarystats import SummaryStats
  52 from utils import parse_changes, check_dsc_files
  53 from textutils import fix_maintainer
  54 from binary import Binary
  55
  56 ###############################################################################
  57
  58 def get_type(f, session):
  59     """
  60     Get the file type of C{f}
  61
  62     @type f: dict
  63     @param f: file entry from Changes object
  64
  65     @type session: SQLA Session
  66     @param session: SQL Alchemy session object
  67
  68     @rtype: string
  69     @return: filetype
  70
  71     """
  72     # Determine the type
  73     if f.has_key("dbtype"):
  74         file_type = file["dbtype"]
  75     elif re_source_ext.match(f["type"]):
  76         file_type = "dsc"
  77     else:
  78         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))
  79
  80     # Validate the override type
  81     type_id = get_override_type(file_type, session)
  82     if type_id is None:
  83         utils.fubar("invalid type (%s) for new.  Say wha?" % (file_type))
  84
  85     return file_type
  86
  87 ################################################################################
  88
  89 # Determine what parts in a .changes are NEW
  90
  91 def determine_new(changes, files, warn=1):
  92     """
  93     Determine what parts in a C{changes} file are NEW.
  94
  95     @type changes: Upload.Pkg.changes dict
  96     @param changes: Changes dictionary
  97
  98     @type files: Upload.Pkg.files dict
  99     @param files: Files dictionary
 100
 101     @type warn: bool
 102     @param warn: Warn if overrides are added for (old)stable
 103
 104     @rtype: dict
 105     @return: dictionary of NEW components.
 106
 107     """
 108     new = {}
 109
 110     session = DBConn().session()
 111
 112     # Build up a list of potentially new things
 113     for name, f in files.items():
 114         # Skip byhand elements
 115         if f["type"] == "byhand":
 116             continue
 117         pkg = f["package"]
 118         priority = f["priority"]
 119         section = f["section"]
 120         file_type = get_type(f, session)
 121         component = f["component"]
 122
 123         if file_type == "dsc":
 124             priority = "source"
 125
 126         if not new.has_key(pkg):
 127             new[pkg] = {}
 128             new[pkg]["priority"] = priority
 129             new[pkg]["section"] = section
 130             new[pkg]["type"] = file_type
 131             new[pkg]["component"] = component
 132             new[pkg]["files"] = []
 133         else:
 134             old_type = new[pkg]["type"]
 135             if old_type != file_type:
 136                 # source gets trumped by deb or udeb
 137                 if old_type == "dsc":
 138                     new[pkg]["priority"] = priority
 139                     new[pkg]["section"] = section
 140                     new[pkg]["type"] = file_type
 141                     new[pkg]["component"] = component
 142
 143         new[pkg]["files"].append(name)
 144
 145         if f.has_key("othercomponents"):
 146             new[pkg]["othercomponents"] = f["othercomponents"]
 147
 148     for suite in changes["suite"].keys():
 149         for pkg in new.keys():
 150             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
 151             if len(ql) > 0:
 152                 for file_entry in new[pkg]["files"]:
 153                     if files[file_entry].has_key("new"):
 154                         del files[file_entry]["new"]
 155                 del new[pkg]
 156
 157     if warn:
 158         for s in ['stable', 'oldstable']:
 159             if changes["suite"].has_key(s):
 160                 print "WARNING: overrides will be added for %s!" % s
 161         for pkg in new.keys():
 162             if new[pkg].has_key("othercomponents"):
 163                 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 164
 165     session.close()
 166
 167     return new
 168
 169 ################################################################################
 170
 171 def check_valid(new):
 172     """
 173     Check if section and priority for NEW packages exist in database.
 174     Additionally does sanity checks:
 175       - debian-installer packages have to be udeb (or source)
 176       - non debian-installer packages can not be udeb
 177       - source priority can only be assigned to dsc file types
 178
 179     @type new: dict
 180     @param new: Dict of new packages with their section, priority and type.
 181
 182     """
 183     for pkg in new.keys():
 184         section_name = new[pkg]["section"]
 185         priority_name = new[pkg]["priority"]
 186         file_type = new[pkg]["type"]
 187
 188         section = get_section(section_name)
 189         if section is None:
 190             new[pkg]["section id"] = -1
 191         else:
 192             new[pkg]["section id"] = section.section_id
 193
 194         priority = get_priority(priority_name)
 195         if priority is None:
 196             new[pkg]["priority id"] = -1
 197         else:
 198             new[pkg]["priority id"] = priority.priority_id
 199
 200         # Sanity checks
 201         di = section_name.find("debian-installer") != -1
 202
 203         # If d-i, we must be udeb and vice-versa
 204         if     (di and file_type not in ("udeb", "dsc")) or \
 205            (not di and file_type == "udeb"):
 206             new[pkg]["section id"] = -1
 207
 208         # If dsc we need to be source and vice-versa
 209         if (priority == "source" and file_type != "dsc") or \
 210            (priority != "source" and file_type == "dsc"):
 211             new[pkg]["priority id"] = -1
 212
 213 ###############################################################################
 214
 215 def lookup_uid_from_fingerprint(fpr, session):
 216     uid = None
 217     uid_name = ""
 218     # This is a stupid default, but see the comments below
 219     is_dm = False
 220
 221     user = get_uid_from_fingerprint(fpr, session)
 222
 223     if user is not None:
 224         uid = user.uid
 225         if user.name is None:
 226             uid_name = ''
 227         else:
 228             uid_name = user.name
 229
 230         # Check the relevant fingerprint (which we have to have)
 231         for f in user.fingerprint:
 232             if f.fingerprint == fpr:
 233                 is_dm = f.keyring.debian_maintainer
 234                 break
 235
 236     return (uid, uid_name, is_dm)
 237
 238 ###############################################################################
 239
 240 # Used by Upload.check_timestamps
 241 class TarTime(object):
 242     def __init__(self, future_cutoff, past_cutoff):
 243         self.reset()
 244         self.future_cutoff = future_cutoff
 245         self.past_cutoff = past_cutoff
 246
 247     def reset(self):
 248         self.future_files = {}
 249         self.ancient_files = {}
 250
 251     def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
 252         if MTime > self.future_cutoff:
 253             self.future_files[Name] = MTime
 254         if MTime < self.past_cutoff:
 255             self.ancient_files[Name] = MTime
 256
 257 ###############################################################################
 258
 259 class Upload(object):
 260     """
 261     Everything that has to do with an upload processed.
 262
 263     """
 264     def __init__(self):
 265         self.logger = None
 266         self.pkg = Changes()
 267         self.reset()
 268
 269     ###########################################################################
 270
 271     def reset (self):
 272         """ Reset a number of internal variables."""
 273
 274         # Initialize the substitution template map
 275         cnf = Config()
 276         self.Subst = {}
 277         self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
 278         self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
 279         self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
 280         self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 281
 282         self.rejects = []
 283         self.warnings = []
 284         self.notes = []
 285
 286         self.pkg.reset()
 287
 288     def package_info(self):
 289         msg = ''
 290
 291         if len(self.rejects) > 0:
 292             msg += "Reject Reasons:\n"
 293             msg += "\n".join(self.rejects)
 294
 295         if len(self.warnings) > 0:
 296             msg += "Warnings:\n"
 297             msg += "\n".join(self.warnings)
 298
 299         if len(self.notes) > 0:
 300             msg += "Notes:\n"
 301             msg += "\n".join(self.notes)
 302
 303         return msg
 304
 305     ###########################################################################
 306     def update_subst(self):
 307         """ Set up the per-package template substitution mappings """
 308
 309         cnf = Config()
 310
 311         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
 312         if not self.pkg.changes.has_key("architecture") or not \
 313            isinstance(self.pkg.changes["architecture"], DictType):
 314             self.pkg.changes["architecture"] = { "Unknown" : "" }
 315
 316         # and maintainer2047 may not exist.
 317         if not self.pkg.changes.has_key("maintainer2047"):
 318             self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
 319
 320         self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
 321         self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
 322         self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
 323
 324         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 325         if self.pkg.changes["architecture"].has_key("source") and \
 326            self.pkg.changes["changedby822"] != "" and \
 327            (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
 328
 329             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
 330             self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
 331             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
 332         else:
 333             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
 334             self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
 335             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
 336
 337         if "sponsoremail" in self.pkg.changes:
 338             self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
 339
 340         if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
 341             self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
 342
 343         # Apply any global override of the Maintainer field
 344         if cnf.get("Dinstall::OverrideMaintainer"):
 345             self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
 346             self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
 347
 348         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
 349         self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
 350         self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
 351
 352     ###########################################################################
 353     def load_changes(self, filename):
 354         """
 355         @rtype: boolean
 356         @rvalue: whether the changes file was valid or not.  We may want to
 357                  reject even if this is True (see what gets put in self.rejects).
 358                  This is simply to prevent us even trying things later which will
 359                  fail because we couldn't properly parse the file.
 360         """
 361         Cnf = Config()
 362         self.pkg.changes_file = filename
 363
 364         # Parse the .changes field into a dictionary
 365         try:
 366             self.pkg.changes.update(parse_changes(filename))
 367         except CantOpenError:
 368             self.rejects.append("%s: can't read file." % (filename))
 369             return False
 370         except ParseChangesError, line:
 371             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 372             return False
 373         except ChangesUnicodeError:
 374             self.rejects.append("%s: changes file not proper utf-8" % (filename))
 375             return False
 376
 377         # Parse the Files field from the .changes into another dictionary
 378         try:
 379             self.pkg.files.update(utils.build_file_list(self.pkg.changes))
 380         except ParseChangesError, line:
 381             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 382             return False
 383         except UnknownFormatError, format:
 384             self.rejects.append("%s: unknown format '%s'." % (filename, format))
 385             return False
 386
 387         # Check for mandatory fields
 388         for i in ("distribution", "source", "binary", "architecture",
 389                   "version", "maintainer", "files", "changes", "description"):
 390             if not self.pkg.changes.has_key(i):
 391                 # Avoid undefined errors later
 392                 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
 393                 return False
 394
 395         # Strip a source version in brackets from the source field
 396         if re_strip_srcver.search(self.pkg.changes["source"]):
 397             self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
 398
 399         # Ensure the source field is a valid package name.
 400         if not re_valid_pkg_name.match(self.pkg.changes["source"]):
 401             self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
 402
 403         # Split multi-value fields into a lower-level dictionary
 404         for i in ("architecture", "distribution", "binary", "closes"):
 405             o = self.pkg.changes.get(i, "")
 406             if o != "":
 407                 del self.pkg.changes[i]
 408
 409             self.pkg.changes[i] = {}
 410
 411             for j in o.split():
 412                 self.pkg.changes[i][j] = 1
 413
 414         # Fix the Maintainer: field to be RFC822/2047 compatible
 415         try:
 416             (self.pkg.changes["maintainer822"],
 417              self.pkg.changes["maintainer2047"],
 418              self.pkg.changes["maintainername"],
 419              self.pkg.changes["maintaineremail"]) = \
 420                    fix_maintainer (self.pkg.changes["maintainer"])
 421         except ParseMaintError, msg:
 422             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
 423                    % (filename, changes["maintainer"], msg))
 424
 425         # ...likewise for the Changed-By: field if it exists.
 426         try:
 427             (self.pkg.changes["changedby822"],
 428              self.pkg.changes["changedby2047"],
 429              self.pkg.changes["changedbyname"],
 430              self.pkg.changes["changedbyemail"]) = \
 431                    fix_maintainer (self.pkg.changes.get("changed-by", ""))
 432         except ParseMaintError, msg:
 433             self.pkg.changes["changedby822"] = ""
 434             self.pkg.changes["changedby2047"] = ""
 435             self.pkg.changes["changedbyname"] = ""
 436             self.pkg.changes["changedbyemail"] = ""
 437
 438             self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
 439                    % (filename, changes["changed-by"], msg))
 440
 441         # Ensure all the values in Closes: are numbers
 442         if self.pkg.changes.has_key("closes"):
 443             for i in self.pkg.changes["closes"].keys():
 444                 if re_isanum.match (i) == None:
 445                     self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
 446
 447         # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
 448         self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
 449         self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
 450
 451         # Check there isn't already a changes file of the same name in one
 452         # of the queue directories.
 453         base_filename = os.path.basename(filename)
 454         for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
 455             if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
 456                 self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
 457
 458         # Check the .changes is non-empty
 459         if not self.pkg.files:
 460             self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
 461             return False
 462
 463         # Changes was syntactically valid even if we'll reject
 464         return True
 465
 466     ###########################################################################
 467
 468     def check_distributions(self):
 469         "Check and map the Distribution field"
 470
 471         Cnf = Config()
 472
 473         # Handle suite mappings
 474         for m in Cnf.ValueList("SuiteMappings"):
 475             args = m.split()
 476             mtype = args[0]
 477             if mtype == "map" or mtype == "silent-map":
 478                 (source, dest) = args[1:3]
 479                 if self.pkg.changes["distribution"].has_key(source):
 480                     del self.pkg.changes["distribution"][source]
 481                     self.pkg.changes["distribution"][dest] = 1
 482                     if mtype != "silent-map":
 483                         self.notes.append("Mapping %s to %s." % (source, dest))
 484                 if self.pkg.changes.has_key("distribution-version"):
 485                     if self.pkg.changes["distribution-version"].has_key(source):
 486                         self.pkg.changes["distribution-version"][source]=dest
 487             elif mtype == "map-unreleased":
 488                 (source, dest) = args[1:3]
 489                 if self.pkg.changes["distribution"].has_key(source):
 490                     for arch in self.pkg.changes["architecture"].keys():
 491                         if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
 492                             self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
 493                             del self.pkg.changes["distribution"][source]
 494                             self.pkg.changes["distribution"][dest] = 1
 495                             break
 496             elif mtype == "ignore":
 497                 suite = args[1]
 498                 if self.pkg.changes["distribution"].has_key(suite):
 499                     del self.pkg.changes["distribution"][suite]
 500                     self.warnings.append("Ignoring %s as a target suite." % (suite))
 501             elif mtype == "reject":
 502                 suite = args[1]
 503                 if self.pkg.changes["distribution"].has_key(suite):
 504                     self.rejects.append("Uploads to %s are not accepted." % (suite))
 505             elif mtype == "propup-version":
 506                 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
 507                 #
 508                 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
 509                 if self.pkg.changes["distribution"].has_key(args[1]):
 510                     self.pkg.changes.setdefault("distribution-version", {})
 511                     for suite in args[2:]:
 512                         self.pkg.changes["distribution-version"][suite] = suite
 513
 514         # Ensure there is (still) a target distribution
 515         if len(self.pkg.changes["distribution"].keys()) < 1:
 516             self.rejects.append("No valid distribution remaining.")
 517
 518         # Ensure target distributions exist
 519         for suite in self.pkg.changes["distribution"].keys():
 520             if not Cnf.has_key("Suite::%s" % (suite)):
 521                 self.rejects.append("Unknown distribution `%s'." % (suite))
 522
 523     ###########################################################################
 524
 525     def binary_file_checks(self, f, session):
 526         cnf = Config()
 527         entry = self.pkg.files[f]
 528
 529         # Extract package control information
 530         deb_file = utils.open_file(f)
 531         try:
 532             control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
 533         except:
 534             self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
 535             deb_file.close()
 536             # Can't continue, none of the checks on control would work.
 537             return
 538
 539         # Check for mandantory "Description:"
 540         deb_file.seek(0)
 541         try:
 542             apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
 543         except:
 544             self.rejects.append("%s: Missing Description in binary package" % (f))
 545             return
 546
 547         deb_file.close()
 548
 549         # Check for mandatory fields
 550         for field in [ "Package", "Architecture", "Version" ]:
 551             if control.Find(field) == None:
 552                 # Can't continue
 553                 self.rejects.append("%s: No %s field in control." % (f, field))
 554                 return
 555
 556         # Ensure the package name matches the one give in the .changes
 557         if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
 558             self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
 559
 560         # Validate the package field
 561         package = control.Find("Package")
 562         if not re_valid_pkg_name.match(package):
 563             self.rejects.append("%s: invalid package name '%s'." % (f, package))
 564
 565         # Validate the version field
 566         version = control.Find("Version")
 567         if not re_valid_version.match(version):
 568             self.rejects.append("%s: invalid version number '%s'." % (f, version))
 569
 570         # Ensure the architecture of the .deb is one we know about.
 571         default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
 572         architecture = control.Find("Architecture")
 573         upload_suite = self.pkg.changes["distribution"].keys()[0]
 574
 575         if      architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
 576             and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
 577             self.rejects.append("Unknown architecture '%s'." % (architecture))
 578
 579         # Ensure the architecture of the .deb is one of the ones
 580         # listed in the .changes.
 581         if not self.pkg.changes["architecture"].has_key(architecture):
 582             self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
 583
 584         # Sanity-check the Depends field
 585         depends = control.Find("Depends")
 586         if depends == '':
 587             self.rejects.append("%s: Depends field is empty." % (f))
 588
 589         # Sanity-check the Provides field
 590         provides = control.Find("Provides")
 591         if provides:
 592             provide = re_spacestrip.sub('', provides)
 593             if provide == '':
 594                 self.rejects.append("%s: Provides field is empty." % (f))
 595             prov_list = provide.split(",")
 596             for prov in prov_list:
 597                 if not re_valid_pkg_name.match(prov):
 598                     self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
 599
 600         # Check the section & priority match those given in the .changes (non-fatal)
 601         if     control.Find("Section") and entry["section"] != "" \
 602            and entry["section"] != control.Find("Section"):
 603             self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
 604                                 (f, control.Find("Section", ""), entry["section"]))
 605         if control.Find("Priority") and entry["priority"] != "" \
 606            and entry["priority"] != control.Find("Priority"):
 607             self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
 608                                 (f, control.Find("Priority", ""), entry["priority"]))
 609
 610         entry["package"] = package
 611         entry["architecture"] = architecture
 612         entry["version"] = version
 613         entry["maintainer"] = control.Find("Maintainer", "")
 614
 615         if f.endswith(".udeb"):
 616             self.pkg.files[f]["dbtype"] = "udeb"
 617         elif f.endswith(".deb"):
 618             self.pkg.files[f]["dbtype"] = "deb"
 619         else:
 620             self.rejects.append("%s is neither a .deb or a .udeb." % (f))
 621
 622         entry["source"] = control.Find("Source", entry["package"])
 623
 624         # Get the source version
 625         source = entry["source"]
 626         source_version = ""
 627
 628         if source.find("(") != -1:
 629             m = re_extract_src_version.match(source)
 630             source = m.group(1)
 631             source_version = m.group(2)
 632
 633         if not source_version:
 634             source_version = self.pkg.files[f]["version"]
 635
 636         entry["source package"] = source
 637         entry["source version"] = source_version
 638
 639         # Ensure the filename matches the contents of the .deb
 640         m = re_isadeb.match(f)
 641
 642         #  package name
 643         file_package = m.group(1)
 644         if entry["package"] != file_package:
 645             self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
 646                                 (f, file_package, entry["dbtype"], entry["package"]))
 647         epochless_version = re_no_epoch.sub('', control.Find("Version"))
 648
 649         #  version
 650         file_version = m.group(2)
 651         if epochless_version != file_version:
 652             self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
 653                                 (f, file_version, entry["dbtype"], epochless_version))
 654
 655         #  architecture
 656         file_architecture = m.group(3)
 657         if entry["architecture"] != file_architecture:
 658             self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
 659                                 (f, file_architecture, entry["dbtype"], entry["architecture"]))
 660
 661         # Check for existent source
 662         source_version = entry["source version"]
 663         source_package = entry["source package"]
 664         if self.pkg.changes["architecture"].has_key("source"):
 665             if source_version != self.pkg.changes["version"]:
 666                 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
 667                                     (source_version, f, self.pkg.changes["version"]))
 668         else:
 669             # Check in the SQL database
 670             if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
 671                 # Check in one of the other directories
 672                 source_epochless_version = re_no_epoch.sub('', source_version)
 673                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 674                 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
 675                     entry["byhand"] = 1
 676                 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
 677                     entry["new"] = 1
 678                 else:
 679                     dsc_file_exists = False
 680                     for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
 681                         if cnf.has_key("Dir::Queue::%s" % (myq)):
 682                             if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
 683                                 dsc_file_exists = True
 684                                 break
 685
 686                     if not dsc_file_exists:
 687                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
 688
 689         # Check the version and for file overwrites
 690         self.check_binary_against_db(f, session)
 691
 692         # Temporarily disable contents generation until we change the table storage layout
 693         #b = Binary(f)
 694         #b.scan_package()
 695         #if len(b.rejects) > 0:
 696         #    for j in b.rejects:
 697         #        self.rejects.append(j)
 698
 699     def source_file_checks(self, f, session):
 700         entry = self.pkg.files[f]
 701
 702         m = re_issource.match(f)
 703         if not m:
 704             return
 705
 706         entry["package"] = m.group(1)
 707         entry["version"] = m.group(2)
 708         entry["type"] = m.group(3)
 709
 710         # Ensure the source package name matches the Source filed in the .changes
 711         if self.pkg.changes["source"] != entry["package"]:
 712             self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
 713
 714         # Ensure the source version matches the version in the .changes file
 715         if re_is_orig_source.match(f):
 716             changes_version = self.pkg.changes["chopversion2"]
 717         else:
 718             changes_version = self.pkg.changes["chopversion"]
 719
 720         if changes_version != entry["version"]:
 721             self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
 722
 723         # Ensure the .changes lists source in the Architecture field
 724         if not self.pkg.changes["architecture"].has_key("source"):
 725             self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
 726
 727         # Check the signature of a .dsc file
 728         if entry["type"] == "dsc":
 729             # check_signature returns either:
 730             #  (None, [list, of, rejects]) or (signature, [])
 731             (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
 732             for j in rejects:
 733                 self.rejects.append(j)
 734
 735         entry["architecture"] = "source"
 736
 737     def per_suite_file_checks(self, f, suite, session):
 738         cnf = Config()
 739         entry = self.pkg.files[f]
 740         archive = utils.where_am_i()
 741
 742         # Skip byhand
 743         if entry.has_key("byhand"):
 744             return
 745
 746         # Check we have fields we need to do these checks
 747         oktogo = True
 748         for m in ['component', 'package', 'priority', 'size', 'md5sum']:
 749             if not entry.has_key(m):
 750                 self.rejects.append("file '%s' does not have field %s set" % (f, m))
 751                 oktogo = False
 752
 753         if not oktogo:
 754             return
 755
 756         # Handle component mappings
 757         for m in cnf.ValueList("ComponentMappings"):
 758             (source, dest) = m.split()
 759             if entry["component"] == source:
 760                 entry["original component"] = source
 761                 entry["component"] = dest
 762
 763         # Ensure the component is valid for the target suite
 764         if cnf.has_key("Suite:%s::Components" % (suite)) and \
 765            entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
 766             self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
 767             return
 768
 769         # Validate the component
 770         if not get_component(entry["component"], session):
 771             self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
 772             return
 773
 774         # See if the package is NEW
 775         if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
 776             entry["new"] = 1
 777
 778         # Validate the priority
 779         if entry["priority"].find('/') != -1:
 780             self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
 781
 782         # Determine the location
 783         location = cnf["Dir::Pool"]
 784         l = get_location(location, entry["component"], archive, session)
 785         if l is None:
 786             self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
 787             entry["location id"] = -1
 788         else:
 789             entry["location id"] = l.location_id
 790
 791         # Check the md5sum & size against existing files (if any)
 792         entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
 793
 794         found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
 795                                          entry["size"], entry["md5sum"], entry["location id"])
 796
 797         if found is None:
 798             self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
 799         elif found is False and poolfile is not None:
 800             self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
 801         else:
 802             if poolfile is None:
 803                 entry["files id"] = None
 804             else:
 805                 entry["files id"] = poolfile.file_id
 806
 807         # Check for packages that have moved from one component to another
 808         entry['suite'] = suite
 809         res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
 810         if res.rowcount > 0:
 811             entry["othercomponents"] = res.fetchone()[0]
 812
 813     def check_files(self, action=True):
 814         archive = utils.where_am_i()
 815         file_keys = self.pkg.files.keys()
 816         holding = Holding()
 817         cnf = Config()
 818
 819         # XXX: As far as I can tell, this can no longer happen - see
 820         #      comments by AJ in old revisions - mhy
 821         # if reprocess is 2 we've already done this and we're checking
 822         # things again for the new .orig.tar.gz.
 823         # [Yes, I'm fully aware of how disgusting this is]
 824         if action and self.reprocess < 2:
 825             cwd = os.getcwd()
 826             os.chdir(self.pkg.directory)
 827             for f in file_keys:
 828                 ret = holding.copy_to_holding(f)
 829                 if ret is not None:
 830                     # XXX: Should we bail out here or try and continue?
 831                     self.rejects.append(ret)
 832
 833             os.chdir(cwd)
 834
 835         # Check there isn't already a .changes or .dak file of the same name in
 836         # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
 837         # [NB: this check must be done post-suite mapping]
 838         base_filename = os.path.basename(self.pkg.changes_file)
 839         dot_dak_filename = base_filename[:-8] + ".dak"
 840
 841         for suite in self.pkg.changes["distribution"].keys():
 842             copychanges = "Suite::%s::CopyChanges" % (suite)
 843             if cnf.has_key(copychanges) and \
 844                    os.path.exists(os.path.join(cnf[copychanges], base_filename)):
 845                 self.rejects.append("%s: a file with this name already exists in %s" \
 846                            % (base_filename, cnf[copychanges]))
 847
 848             copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
 849             if cnf.has_key(copy_dot_dak) and \
 850                    os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
 851                 self.rejects.append("%s: a file with this name already exists in %s" \
 852                            % (dot_dak_filename, Cnf[copy_dot_dak]))
 853
 854         self.reprocess = 0
 855         has_binaries = False
 856         has_source = False
 857
 858         session = DBConn().session()
 859
 860         for f, entry in self.pkg.files.items():
 861             # Ensure the file does not already exist in one of the accepted directories
 862             for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
 863                 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
 864                 if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
 865                     self.rejects.append("%s file already exists in the %s directory." % (f, d))
 866
 867             if not re_taint_free.match(f):
 868                 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
 869
 870             # Check the file is readable
 871             if os.access(f, os.R_OK) == 0:
 872                 # When running in -n, copy_to_holding() won't have
 873                 # generated the reject_message, so we need to.
 874                 if action:
 875                     if os.path.exists(f):
 876                         self.rejects.append("Can't read `%s'. [permission denied]" % (f))
 877                     else:
 878                         self.rejects.append("Can't read `%s'. [file not found]" % (f))
 879                 entry["type"] = "unreadable"
 880                 continue
 881
 882             # If it's byhand skip remaining checks
 883             if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
 884                 entry["byhand"] = 1
 885                 entry["type"] = "byhand"
 886
 887             # Checks for a binary package...
 888             elif re_isadeb.match(f):
 889                 has_binaries = True
 890                 entry["type"] = "deb"
 891
 892                 # This routine appends to self.rejects/warnings as appropriate
 893                 self.binary_file_checks(f, session)
 894
 895             # Checks for a source package...
 896             elif re_issource.match(f):
 897                 has_source = True
 898
 899                 # This routine appends to self.rejects/warnings as appropriate
 900                 self.source_file_checks(f, session)
 901
 902             # Not a binary or source package?  Assume byhand...
 903             else:
 904                 entry["byhand"] = 1
 905                 entry["type"] = "byhand"
 906
 907             # Per-suite file checks
 908             entry["oldfiles"] = {}
 909             for suite in self.pkg.changes["distribution"].keys():
 910                 self.per_suite_file_checks(f, suite, session)
 911
 912         session.close()
 913
 914         # If the .changes file says it has source, it must have source.
 915         if self.pkg.changes["architecture"].has_key("source"):
 916             if not has_source:
 917                 self.rejects.append("no source found and Architecture line in changes mention source.")
 918
 919             if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
 920                 self.rejects.append("source only uploads are not supported.")
 921
 922     ###########################################################################
 923     def check_dsc(self, action=True, session=None):
 924         """Returns bool indicating whether or not the source changes are valid"""
 925         # Ensure there is source to check
 926         if not self.pkg.changes["architecture"].has_key("source"):
 927             return True
 928
 929         # Find the .dsc
 930         dsc_filename = None
 931         for f, entry in self.pkg.files.items():
 932             if entry["type"] == "dsc":
 933                 if dsc_filename:
 934                     self.rejects.append("can not process a .changes file with multiple .dsc's.")
 935                     return False
 936                 else:
 937                     dsc_filename = f
 938
 939         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
 940         if not dsc_filename:
 941             self.rejects.append("source uploads must contain a dsc file")
 942             return False
 943
 944         # Parse the .dsc file
 945         try:
 946             self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
 947         except CantOpenError:
 948             # if not -n copy_to_holding() will have done this for us...
 949             if not action:
 950                 self.rejects.append("%s: can't read file." % (dsc_filename))
 951         except ParseChangesError, line:
 952             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
 953         except InvalidDscError, line:
 954             self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
 955         except ChangesUnicodeError:
 956             self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
 957
 958         # Build up the file list of files mentioned by the .dsc
 959         try:
 960             self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
 961         except NoFilesFieldError:
 962             self.rejects.append("%s: no Files: field." % (dsc_filename))
 963             return False
 964         except UnknownFormatError, format:
 965             self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
 966             return False
 967         except ParseChangesError, line:
 968             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
 969             return False
 970
 971         # Enforce mandatory fields
 972         for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
 973             if not self.pkg.dsc.has_key(i):
 974                 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
 975                 return False
 976
 977         # Validate the source and version fields
 978         if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
 979             self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
 980         if not re_valid_version.match(self.pkg.dsc["version"]):
 981             self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
 982
 983         # Only a limited list of source formats are allowed in each suite
 984         for dist in self.pkg.changes["distribution"].keys():
 985             allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
 986             if self.pkg.dsc["format"] not in allowed:
 987                 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
 988
 989         # Validate the Maintainer field
 990         try:
 991             # We ignore the return value
 992             fix_maintainer(self.pkg.dsc["maintainer"])
 993         except ParseMaintError, msg:
 994             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
 995                                  % (dsc_filename, self.pkg.dsc["maintainer"], msg))
 996
 997         # Validate the build-depends field(s)
 998         for field_name in [ "build-depends", "build-depends-indep" ]:
 999             field = self.pkg.dsc.get(field_name)
1000             if field:
1001                 # Check for broken dpkg-dev lossage...
1002                 if field.startswith("ARRAY"):
1003                     self.rejects.append("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % \
1004                                         (dsc_filename, field_name.title()))
1005
1006                 # Have apt try to parse them...
1007                 try:
1008                     apt_pkg.ParseSrcDepends(field)
1009                 except:
1010                     self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
1011
1012         # Ensure the version number in the .dsc matches the version number in the .changes
1013         epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1014         changes_version = self.pkg.files[dsc_filename]["version"]
1015
1016         if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1017             self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1018
1019         # Ensure the Files field contain only what's expected
1020         self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
1021
1022         # Ensure source is newer than existing source in target suites
1023         session = DBConn().session()
1024         self.check_source_against_db(dsc_filename, session)
1025         self.check_dsc_against_db(dsc_filename, session)
1026         session.close()
1027
1028         return True
1029
1030     ###########################################################################
1031
1032     def get_changelog_versions(self, source_dir):
1033         """Extracts a the source package and (optionally) grabs the
1034         version history out of debian/changelog for the BTS."""
1035
1036         cnf = Config()
1037
1038         # Find the .dsc (again)
1039         dsc_filename = None
1040         for f in self.pkg.files.keys():
1041             if self.pkg.files[f]["type"] == "dsc":
1042                 dsc_filename = f
1043
1044         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1045         if not dsc_filename:
1046             return
1047
1048         # Create a symlink mirror of the source files in our temporary directory
1049         for f in self.pkg.files.keys():
1050             m = re_issource.match(f)
1051             if m:
1052                 src = os.path.join(source_dir, f)
1053                 # If a file is missing for whatever reason, give up.
1054                 if not os.path.exists(src):
1055                     return
1056                 ftype = m.group(3)
1057                 if re_is_orig_source.match(f) and pkg.orig_files.has_key(f) and \
1058                    pkg.orig_files[f].has_key("path"):
1059                     continue
1060                 dest = os.path.join(os.getcwd(), f)
1061                 os.symlink(src, dest)
1062
1063         # If the orig files are not a part of the upload, create symlinks to the
1064         # existing copies.
1065         for orig_file in self.pkg.orig_files.keys():
1066             if not self.pkg.orig_files[orig_file].has_key("path"):
1067                 continue
1068             dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
1069             os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
1070
1071         # Extract the source
1072         cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1073         (result, output) = commands.getstatusoutput(cmd)
1074         if (result != 0):
1075             self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1076             self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
1077             return
1078
1079         if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1080             return
1081
1082         # Get the upstream version
1083         upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1084         if re_strip_revision.search(upstr_version):
1085             upstr_version = re_strip_revision.sub('', upstr_version)
1086
1087         # Ensure the changelog file exists
1088         changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1089         if not os.path.exists(changelog_filename):
1090             self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1091             return
1092
1093         # Parse the changelog
1094         self.pkg.dsc["bts changelog"] = ""
1095         changelog_file = utils.open_file(changelog_filename)
1096         for line in changelog_file.readlines():
1097             m = re_changelog_versions.match(line)
1098             if m:
1099                 self.pkg.dsc["bts changelog"] += line
1100         changelog_file.close()
1101
1102         # Check we found at least one revision in the changelog
1103         if not self.pkg.dsc["bts changelog"]:
1104             self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1105
1106     def check_source(self):
1107         # XXX: I'm fairly sure reprocess == 2 can never happen
1108         #      AJT disabled the is_incoming check years ago - mhy
1109         #      We should probably scrap or rethink the whole reprocess thing
1110         # Bail out if:
1111         #    a) there's no source
1112         # or b) reprocess is 2 - we will do this check next time when orig
1113         #       tarball is in 'files'
1114         # or c) the orig files are MIA
1115         if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
1116            or len(self.pkg.orig_files) == 0:
1117             return
1118
1119         tmpdir = utils.temp_dirname()
1120
1121         # Move into the temporary directory
1122         cwd = os.getcwd()
1123         os.chdir(tmpdir)
1124
1125         # Get the changelog version history
1126         self.get_changelog_versions(cwd)
1127
1128         # Move back and cleanup the temporary tree
1129         os.chdir(cwd)
1130
1131         try:
1132             shutil.rmtree(tmpdir)
1133         except OSError, e:
1134             if e.errno != errno.EACCES:
1135                 print "foobar"
1136                 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1137
1138             self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1139             # We probably have u-r or u-w directories so chmod everything
1140             # and try again.
1141             cmd = "chmod -R u+rwx %s" % (tmpdir)
1142             result = os.system(cmd)
1143             if result != 0:
1144                 utils.fubar("'%s' failed with result %s." % (cmd, result))
1145             shutil.rmtree(tmpdir)
1146         except Exception, e:
1147             print "foobar2 (%s)" % e
1148             utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1149
1150     ###########################################################################
1151     def ensure_hashes(self):
1152         # Make sure we recognise the format of the Files: field in the .changes
1153         format = self.pkg.changes.get("format", "0.0").split(".", 1)
1154         if len(format) == 2:
1155             format = int(format[0]), int(format[1])
1156         else:
1157             format = int(float(format[0])), 0
1158
1159         # We need to deal with the original changes blob, as the fields we need
1160         # might not be in the changes dict serialised into the .dak anymore.
1161         orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1162
1163         # Copy the checksums over to the current changes dict.  This will keep
1164         # the existing modifications to it intact.
1165         for field in orig_changes:
1166             if field.startswith('checksums-'):
1167                 self.pkg.changes[field] = orig_changes[field]
1168
1169         # Check for unsupported hashes
1170         for j in utils.check_hash_fields(".changes", self.pkg.changes):
1171             self.rejects.append(j)
1172
1173         for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1174             self.rejects.append(j)
1175
1176         # We have to calculate the hash if we have an earlier changes version than
1177         # the hash appears in rather than require it exist in the changes file
1178         for hashname, hashfunc, version in utils.known_hashes:
1179             # TODO: Move _ensure_changes_hash into this class
1180             for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1181                 self.rejects.append(j)
1182             if "source" in self.pkg.changes["architecture"]:
1183                 # TODO: Move _ensure_dsc_hash into this class
1184                 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1185                     self.rejects.append(j)
1186
1187     def check_hashes(self):
1188         for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1189             self.rejects.append(m)
1190
1191         for m in utils.check_size(".changes", self.pkg.files):
1192             self.rejects.append(m)
1193
1194         for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1195             self.rejects.append(m)
1196
1197         for m in utils.check_size(".dsc", self.pkg.dsc_files):
1198             self.rejects.append(m)
1199
1200         self.ensure_hashes()
1201
1202     ###########################################################################
1203     def check_urgency(self):
1204         cnf = Config()
1205         if self.pkg.changes["architecture"].has_key("source"):
1206             if not self.pkg.changes.has_key("urgency"):
1207                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1208             self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1209             if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1210                 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1211                                      (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1212                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1213
1214     ###########################################################################
1215
1216     # Sanity check the time stamps of files inside debs.
1217     # [Files in the near future cause ugly warnings and extreme time
1218     #  travel can cause errors on extraction]
1219
1220     def check_timestamps(self):
1221         Cnf = Config()
1222
1223         future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1224         past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1225         tar = TarTime(future_cutoff, past_cutoff)
1226
1227         for filename, entry in self.pkg.files.items():
1228             if entry["type"] == "deb":
1229                 tar.reset()
1230                 try:
1231                     deb_file = utils.open_file(filename)
1232                     apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1233                     deb_file.seek(0)
1234                     try:
1235                         apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1236                     except SystemError, e:
1237                         # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1238                         if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1239                             raise
1240                         deb_file.seek(0)
1241                         apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1242
1243                     deb_file.close()
1244
1245                     future_files = tar.future_files.keys()
1246                     if future_files:
1247                         num_future_files = len(future_files)
1248                         future_file = future_files[0]
1249                         future_date = tar.future_files[future_file]
1250                         self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1251                                % (filename, num_future_files, future_file, time.ctime(future_date)))
1252
1253                     ancient_files = tar.ancient_files.keys()
1254                     if ancient_files:
1255                         num_ancient_files = len(ancient_files)
1256                         ancient_file = ancient_files[0]
1257                         ancient_date = tar.ancient_files[ancient_file]
1258                         self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1259                                % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1260                 except:
1261                     self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1262
1263     ###########################################################################
1264     def check_transition(self, session):
1265         cnf = Config()
1266
1267         sourcepkg = self.pkg.changes["source"]
1268
1269         # No sourceful upload -> no need to do anything else, direct return
1270         # We also work with unstable uploads, not experimental or those going to some
1271         # proposed-updates queue
1272         if "source" not in self.pkg.changes["architecture"] or \
1273            "unstable" not in self.pkg.changes["distribution"]:
1274             return
1275
1276         # Also only check if there is a file defined (and existant) with
1277         # checks.
1278         transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1279         if transpath == "" or not os.path.exists(transpath):
1280             return
1281
1282         # Parse the yaml file
1283         sourcefile = file(transpath, 'r')
1284         sourcecontent = sourcefile.read()
1285         try:
1286             transitions = yaml.load(sourcecontent)
1287         except yaml.YAMLError, msg:
1288             # This shouldn't happen, there is a wrapper to edit the file which
1289             # checks it, but we prefer to be safe than ending up rejecting
1290             # everything.
1291             utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1292             return
1293
1294         # Now look through all defined transitions
1295         for trans in transitions:
1296             t = transitions[trans]
1297             source = t["source"]
1298             expected = t["new"]
1299
1300             # Will be None if nothing is in testing.
1301             current = get_source_in_suite(source, "testing", session)
1302             if current is not None:
1303                 compare = apt_pkg.VersionCompare(current.version, expected)
1304
1305             if current is None or compare < 0:
1306                 # This is still valid, the current version in testing is older than
1307                 # the new version we wait for, or there is none in testing yet
1308
1309                 # Check if the source we look at is affected by this.
1310                 if sourcepkg in t['packages']:
1311                     # The source is affected, lets reject it.
1312
1313                     rejectmsg = "%s: part of the %s transition.\n\n" % (
1314                         sourcepkg, trans)
1315
1316                     if current is not None:
1317                         currentlymsg = "at version %s" % (current.version)
1318                     else:
1319                         currentlymsg = "not present in testing"
1320
1321                     rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1322
1323                     rejectmsg += "\n".join(textwrap.wrap("""Your package
1324 is part of a testing transition designed to get %s migrated (it is
1325 currently %s, we need version %s).  This transition is managed by the
1326 Release Team, and %s is the Release-Team member responsible for it.
1327 Please mail debian-release@lists.debian.org or contact %s directly if you
1328 need further assistance.  You might want to upload to experimental until this
1329 transition is done."""
1330                             % (source, currentlymsg, expected,t["rm"], t["rm"])))
1331
1332                     self.rejects.append(rejectmsg)
1333                     return
1334
1335     ###########################################################################
1336     def check_signed_by_key(self):
1337         """Ensure the .changes is signed by an authorized uploader."""
1338         session = DBConn().session()
1339
1340         self.check_transition(session)
1341
1342         (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
1343
1344         # match claimed name with actual name:
1345         if uid is None:
1346             # This is fundamentally broken but need us to refactor how we get
1347             # the UIDs/Fingerprints in order for us to fix it properly
1348             uid, uid_email = self.pkg.changes["fingerprint"], uid
1349             may_nmu, may_sponsor = 1, 1
1350             # XXX by default new dds don't have a fingerprint/uid in the db atm,
1351             #     and can't get one in there if we don't allow nmu/sponsorship
1352         elif is_dm is False:
1353             # If is_dm is False, we allow full upload rights
1354             uid_email = "%s@debian.org" % (uid)
1355             may_nmu, may_sponsor = 1, 1
1356         else:
1357             # Assume limited upload rights unless we've discovered otherwise
1358             uid_email = uid
1359             may_nmu, may_sponsor = 0, 0
1360
1361         if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1362             sponsored = 0
1363         elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1364             sponsored = 0
1365             if uid_name == "": sponsored = 1
1366         else:
1367             sponsored = 1
1368             if ("source" in self.pkg.changes["architecture"] and
1369                 uid_email and utils.is_email_alias(uid_email)):
1370                 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1371                 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1372                     self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1373                     self.pkg.changes["sponsoremail"] = uid_email
1374
1375         if sponsored and not may_sponsor:
1376             self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
1377
1378         if not sponsored and not may_nmu:
1379             should_reject = True
1380             highest_sid, highest_version = None, None
1381
1382             # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
1383             #      It ignores higher versions with the dm_upload_allowed flag set to false
1384             #      I'm keeping the existing behaviour for now until I've gone back and
1385             #      checked exactly what the GR says - mhy
1386             for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
1387                 if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
1388                      highest_sid = si.source_id
1389                      highest_version = si.version
1390
1391             if highest_sid is None:
1392                 self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
1393             else:
1394                 for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
1395                     (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1396                     if email == uid_email or name == uid_name:
1397                         should_reject = False
1398                         break
1399
1400             if should_reject is True:
1401                 self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
1402
1403             for b in self.pkg.changes["binary"].keys():
1404                 for suite in self.pkg.changes["distribution"].keys():
1405                     q = session.query(DBSource)
1406                     q = q.join(DBBinary).filter_by(package=b)
1407                     q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1408
1409                     for s in q.all():
1410                         if s.source != self.pkg.changes["source"]:
1411                             self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
1412
1413             for f in self.pkg.files.keys():
1414                 if self.pkg.files[f].has_key("byhand"):
1415                     self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
1416                 if self.pkg.files[f].has_key("new"):
1417                     self.rejects.append("%s may not upload NEW file %s" % (uid, f))
1418
1419         session.close()
1420
1421     ###########################################################################
1422     def build_summaries(self):
1423         """ Build a summary of changes the upload introduces. """
1424
1425         (byhand, new, summary, override_summary) = self.pkg.file_summary()
1426
1427         short_summary = summary
1428
1429         # This is for direport's benefit...
1430         f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1431
1432         if byhand or new:
1433             summary += "Changes: " + f
1434
1435         summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1436
1437         summary += self.announce(short_summary, 0)
1438
1439         return (summary, short_summary)
1440
1441     ###########################################################################
1442
1443     def close_bugs(self, summary, action):
1444         """
1445         Send mail to close bugs as instructed by the closes field in the changes file.
1446         Also add a line to summary if any work was done.
1447
1448         @type summary: string
1449         @param summary: summary text, as given by L{build_summaries}
1450
1451         @type action: bool
1452         @param action: Set to false no real action will be done.
1453
1454         @rtype: string
1455         @return: summary. If action was taken, extended by the list of closed bugs.
1456
1457         """
1458
1459         template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1460
1461         bugs = self.pkg.changes["closes"].keys()
1462
1463         if not bugs:
1464             return summary
1465
1466         bugs.sort()
1467         summary += "Closing bugs: "
1468         for bug in bugs:
1469             summary += "%s " % (bug)
1470             if action:
1471                 self.update_subst()
1472                 self.Subst["__BUG_NUMBER__"] = bug
1473                 if self.pkg.changes["distribution"].has_key("stable"):
1474                     self.Subst["__STABLE_WARNING__"] = """
1475 Note that this package is not part of the released stable Debian
1476 distribution.  It may have dependencies on other unreleased software,
1477 or other instabilities.  Please take care if you wish to install it.
1478 The update will eventually make its way into the next released Debian
1479 distribution."""
1480                 else:
1481                     self.Subst["__STABLE_WARNING__"] = ""
1482                 mail_message = utils.TemplateSubst(self.Subst, template)
1483                 utils.send_mail(mail_message)
1484
1485                 # Clear up after ourselves
1486                 del self.Subst["__BUG_NUMBER__"]
1487                 del self.Subst["__STABLE_WARNING__"]
1488
1489         if action and self.logger:
1490             self.logger.log(["closing bugs"] + bugs)
1491
1492         summary += "\n"
1493
1494         return summary
1495
1496     ###########################################################################
1497
1498     def announce(self, short_summary, action):
1499         """
1500         Send an announce mail about a new upload.
1501
1502         @type short_summary: string
1503         @param short_summary: Short summary text to include in the mail
1504
1505         @type action: bool
1506         @param action: Set to false no real action will be done.
1507
1508         @rtype: string
1509         @return: Textstring about action taken.
1510
1511         """
1512
1513         cnf = Config()
1514         announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1515
1516         # Only do announcements for source uploads with a recent dpkg-dev installed
1517         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1518            self.pkg.changes["architecture"].has_key("source"):
1519             return ""
1520
1521         lists_done = {}
1522         summary = ""
1523
1524         self.Subst["__SHORT_SUMMARY__"] = short_summary
1525
1526         for dist in self.pkg.changes["distribution"].keys():
1527             announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1528             if announce_list == "" or lists_done.has_key(announce_list):
1529                 continue
1530
1531             lists_done[announce_list] = 1
1532             summary += "Announcing to %s\n" % (announce_list)
1533
1534             if action:
1535                 self.update_subst()
1536                 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1537                 if cnf.get("Dinstall::TrackingServer") and \
1538                    self.pkg.changes["architecture"].has_key("source"):
1539                     trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1540                     self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1541
1542                 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1543                 utils.send_mail(mail_message)
1544
1545                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1546
1547         if cnf.FindB("Dinstall::CloseBugs"):
1548             summary = self.close_bugs(summary, action)
1549
1550         del self.Subst["__SHORT_SUMMARY__"]
1551
1552         return summary
1553
1554     ###########################################################################
1555
1556     def accept (self, summary, short_summary, targetdir=None):
1557         """
1558         Accept an upload.
1559
1560         This moves all files referenced from the .changes into the I{accepted}
1561         queue, sends the accepted mail, announces to lists, closes bugs and
1562         also checks for override disparities. If enabled it will write out
1563         the version history for the BTS Version Tracking and will finally call
1564         L{queue_build}.
1565
1566         @type summary: string
1567         @param summary: Summary text
1568
1569         @type short_summary: string
1570         @param short_summary: Short summary
1571
1572         """
1573
1574         cnf = Config()
1575         stats = SummaryStats()
1576
1577         accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
1578
1579         if targetdir is None:
1580             targetdir = cnf["Dir::Queue::Accepted"]
1581
1582         print "Accepting."
1583         if self.logger:
1584             self.logger.log(["Accepting changes", self.pkg.changes_file])
1585
1586         self.pkg.write_dot_dak(targetdir)
1587
1588         # Move all the files into the accepted directory
1589         utils.move(self.pkg.changes_file, targetdir)
1590
1591         for name, entry in sorted(self.pkg.files.items()):
1592             utils.move(name, targetdir)
1593             stats.accept_bytes += float(entry["size"])
1594
1595         stats.accept_count += 1
1596
1597         # Send accept mail, announce to lists, close bugs and check for
1598         # override disparities
1599         if not cnf["Dinstall::Options::No-Mail"]:
1600             self.update_subst()
1601             self.Subst["__SUITE__"] = ""
1602             self.Subst["__SUMMARY__"] = summary
1603             mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
1604             utils.send_mail(mail_message)
1605             self.announce(short_summary, 1)
1606
1607         ## Helper stuff for DebBugs Version Tracking
1608         if cnf.Find("Dir::Queue::BTSVersionTrack"):
1609             # ??? once queue/* is cleared on *.d.o and/or reprocessed
1610             # the conditionalization on dsc["bts changelog"] should be
1611             # dropped.
1612
1613             # Write out the version history from the changelog
1614             if self.pkg.changes["architecture"].has_key("source") and \
1615                self.pkg.dsc.has_key("bts changelog"):
1616
1617                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1618                 version_history = os.fdopen(fd, 'w')
1619                 version_history.write(self.pkg.dsc["bts changelog"])
1620                 version_history.close()
1621                 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1622                                       self.pkg.changes_file[:-8]+".versions")
1623                 os.rename(temp_filename, filename)
1624                 os.chmod(filename, 0644)
1625
1626             # Write out the binary -> source mapping.
1627             (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1628             debinfo = os.fdopen(fd, 'w')
1629             for name, entry in sorted(self.pkg.files.items()):
1630                 if entry["type"] == "deb":
1631                     line = " ".join([entry["package"], entry["version"],
1632                                      entry["architecture"], entry["source package"],
1633                                      entry["source version"]])
1634                     debinfo.write(line+"\n")
1635             debinfo.close()
1636             filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1637                                   self.pkg.changes_file[:-8]+".debinfo")
1638             os.rename(temp_filename, filename)
1639             os.chmod(filename, 0644)
1640
1641         # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
1642         # <Ganneff> we do call queue_build too
1643         # <mhy> well yes, we'd have had to if we were inserting into accepted
1644         # <Ganneff> now. thats database only.
1645         # <mhy> urgh, that's going to get messy
1646         # <Ganneff> so i make the p-n call to it *also* using accepted/
1647         # <mhy> but then the packages will be in the queue_build table without the files being there
1648         # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
1649         # <mhy> ah, good point
1650         # <Ganneff> so it will work out, as unchecked move it over
1651         # <mhy> that's all completely sick
1652         # <Ganneff> yes
1653
1654         # This routine returns None on success or an error on failure
1655         res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
1656         if res:
1657             utils.fubar(res)
1658
1659
1660     def check_override(self):
1661         """
1662         Checks override entries for validity. Mails "Override disparity" warnings,
1663         if that feature is enabled.
1664
1665         Abandons the check if
1666           - override disparity checks are disabled
1667           - mail sending is disabled
1668         """
1669
1670         cnf = Config()
1671
1672         # Abandon the check if:
1673         #  a) override disparity checks have been disabled
1674         #  b) we're not sending mail
1675         if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
1676            cnf["Dinstall::Options::No-Mail"]:
1677             return
1678
1679         summary = self.pkg.check_override()
1680
1681         if summary == "":
1682             return
1683
1684         overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
1685
1686         self.update_subst()
1687         self.Subst["__SUMMARY__"] = summary
1688         mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
1689         utils.send_mail(mail_message)
1690         del self.Subst["__SUMMARY__"]
1691
1692     ###########################################################################
1693
1694     def remove(self, dir=None):
1695         """
1696         Used (for instance) in p-u to remove the package from unchecked
1697         """
1698         if dir is None:
1699             os.chdir(self.pkg.directory)
1700         else:
1701             os.chdir(dir)
1702
1703         for f in self.pkg.files.keys():
1704             os.unlink(f)
1705         os.unlink(self.pkg.changes_file)
1706
1707     ###########################################################################
1708
1709     def move_to_dir (self, dest, perms=0660, changesperms=0664):
1710         """
1711         Move files to dest with certain perms/changesperms
1712         """
1713         utils.move(self.pkg.changes_file, dest, perms=changesperms)
1714         for f in self.pkg.files.keys():
1715             utils.move(f, dest, perms=perms)
1716
1717     ###########################################################################
1718
1719     def force_reject(self, reject_files):
1720         """
1721         Forcefully move files from the current directory to the
1722         reject directory.  If any file already exists in the reject
1723         directory it will be moved to the morgue to make way for
1724         the new file.
1725
1726         @type files: dict
1727         @param files: file dictionary
1728
1729         """
1730
1731         cnf = Config()
1732
1733         for file_entry in reject_files:
1734             # Skip any files which don't exist or which we don't have permission to copy.
1735             if os.access(file_entry, os.R_OK) == 0:
1736                 continue
1737
1738             dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
1739
1740             try:
1741                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
1742             except OSError, e:
1743                 # File exists?  Let's try and move it to the morgue
1744                 if e.errno == errno.EEXIST:
1745                     morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
1746                     try:
1747                         morgue_file = utils.find_next_free(morgue_file)
1748                     except NoFreeFilenameError:
1749                         # Something's either gone badly Pete Tong, or
1750                         # someone is trying to exploit us.
1751                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
1752                         return
1753                     utils.move(dest_file, morgue_file, perms=0660)
1754                     try:
1755                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1756                     except OSError, e:
1757                         # Likewise
1758                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
1759                         return
1760                 else:
1761                     raise
1762             # If we got here, we own the destination file, so we can
1763             # safely overwrite it.
1764             utils.move(file_entry, dest_file, 1, perms=0660)
1765             os.close(dest_fd)
1766
1767     ###########################################################################
1768     def do_reject (self, manual=0, reject_message="", note=""):
1769         """
1770         Reject an upload. If called without a reject message or C{manual} is
1771         true, spawn an editor so the user can write one.
1772
1773         @type manual: bool
1774         @param manual: manual or automated rejection
1775
1776         @type reject_message: string
1777         @param reject_message: A reject message
1778
1779         @return: 0
1780
1781         """
1782         # If we weren't given a manual rejection message, spawn an
1783         # editor so the user can add one in...
1784         if manual and not reject_message:
1785             (fd, temp_filename) = utils.temp_filename()
1786             temp_file = os.fdopen(fd, 'w')
1787             if len(note) > 0:
1788                 for line in note:
1789                     temp_file.write(line)
1790             temp_file.close()
1791             editor = os.environ.get("EDITOR","vi")
1792             answer = 'E'
1793             while answer == 'E':
1794                 os.system("%s %s" % (editor, temp_filename))
1795                 temp_fh = utils.open_file(temp_filename)
1796                 reject_message = "".join(temp_fh.readlines())
1797                 temp_fh.close()
1798                 print "Reject message:"
1799                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1)
1800                 prompt = "[R]eject, Edit, Abandon, Quit ?"
1801                 answer = "XXX"
1802                 while prompt.find(answer) == -1:
1803                     answer = utils.our_raw_input(prompt)
1804                     m = re_default_answer.search(prompt)
1805                     if answer == "":
1806                         answer = m.group(1)
1807                     answer = answer[:1].upper()
1808             os.unlink(temp_filename)
1809             if answer == 'A':
1810                 return 1
1811             elif answer == 'Q':
1812                 sys.exit(0)
1813
1814         print "Rejecting.\n"
1815
1816         cnf = Config()
1817
1818         reason_filename = self.pkg.changes_file[:-8] + ".reason"
1819         reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
1820
1821         # Move all the files into the reject directory
1822         reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
1823         self.force_reject(reject_files)
1824
1825         # If we fail here someone is probably trying to exploit the race
1826         # so let's just raise an exception ...
1827         if os.path.exists(reason_filename):
1828             os.unlink(reason_filename)
1829         reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1830
1831         rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
1832
1833         self.update_subst()
1834         if not manual:
1835             self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
1836             self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
1837             self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
1838             os.write(reason_fd, reject_message)
1839             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1840         else:
1841             # Build up the rejection email
1842             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
1843             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
1844             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
1845             self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
1846             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1847             # Write the rejection email out as the <foo>.reason file
1848             os.write(reason_fd, reject_mail_message)
1849
1850         del self.Subst["__REJECTOR_ADDRESS__"]
1851         del self.Subst["__MANUAL_REJECT_MESSAGE__"]
1852         del self.Subst["__CC__"]
1853
1854         os.close(reason_fd)
1855
1856         # Send the rejection mail if appropriate
1857         if not cnf["Dinstall::Options::No-Mail"]:
1858             utils.send_mail(reject_mail_message)
1859
1860         if self.logger:
1861             self.logger.log(["rejected", self.pkg.changes_file])
1862
1863         return 0
1864
1865     ################################################################################
1866     def in_override_p(self, package, component, suite, binary_type, file, session):
1867         """
1868         Check if a package already has override entries in the DB
1869
1870         @type package: string
1871         @param package: package name
1872
1873         @type component: string
1874         @param component: database id of the component
1875
1876         @type suite: int
1877         @param suite: database id of the suite
1878
1879         @type binary_type: string
1880         @param binary_type: type of the package
1881
1882         @type file: string
1883         @param file: filename we check
1884
1885         @return: the database result. But noone cares anyway.
1886
1887         """
1888
1889         cnf = Config()
1890
1891         if binary_type == "": # must be source
1892             file_type = "dsc"
1893         else:
1894             file_type = binary_type
1895
1896         # Override suite name; used for example with proposed-updates
1897         if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
1898             suite = cnf["Suite::%s::OverrideSuite" % (suite)]
1899
1900         result = get_override(package, suite, component, file_type, session)
1901
1902         # If checking for a source package fall back on the binary override type
1903         if file_type == "dsc" and len(result) < 1:
1904             result = get_override(package, suite, component, ['deb', 'udeb'], session)
1905
1906         # Remember the section and priority so we can check them later if appropriate
1907         if len(result) > 0:
1908             result = result[0]
1909             self.pkg.files[file]["override section"] = result.section.section
1910             self.pkg.files[file]["override priority"] = result.priority.priority
1911             return result
1912
1913         return None
1914
1915     ################################################################################
1916     def get_anyversion(self, sv_list, suite):
1917         """
1918         @type sv_list: list
1919         @param sv_list: list of (suite, version) tuples to check
1920
1921         @type suite: string
1922         @param suite: suite name
1923
1924         Description: TODO
1925         """
1926         Cnf = Config()
1927         anyversion = None
1928         anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
1929         for (s, v) in sv_list:
1930             if s in [ x.lower() for x in anysuite ]:
1931                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
1932                     anyversion = v
1933
1934         return anyversion
1935
1936     ################################################################################
1937
1938     def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
1939         """
1940         @type sv_list: list
1941         @param sv_list: list of (suite, version) tuples to check
1942
1943         @type file: string
1944         @param file: XXX
1945
1946         @type new_version: string
1947         @param new_version: XXX
1948
1949         Ensure versions are newer than existing packages in target
1950         suites and that cross-suite version checking rules as
1951         set out in the conf file are satisfied.
1952         """
1953
1954         cnf = Config()
1955
1956         # Check versions for each target suite
1957         for target_suite in self.pkg.changes["distribution"].keys():
1958             must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
1959             must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
1960
1961             # Enforce "must be newer than target suite" even if conffile omits it
1962             if target_suite not in must_be_newer_than:
1963                 must_be_newer_than.append(target_suite)
1964
1965             for (suite, existent_version) in sv_list:
1966                 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
1967
1968                 if suite in must_be_newer_than and sourceful and vercmp < 1:
1969                     self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
1970
1971                 if suite in must_be_older_than and vercmp > -1:
1972                     cansave = 0
1973
1974                     if self.pkg.changes.get('distribution-version', {}).has_key(suite):
1975                         # we really use the other suite, ignoring the conflicting one ...
1976                         addsuite = self.pkg.changes["distribution-version"][suite]
1977
1978                         add_version = self.get_anyversion(sv_list, addsuite)
1979                         target_version = self.get_anyversion(sv_list, target_suite)
1980
1981                         if not add_version:
1982                             # not add_version can only happen if we map to a suite
1983                             # that doesn't enhance the suite we're propup'ing from.
1984                             # so "propup-ver x a b c; map a d" is a problem only if
1985                             # d doesn't enhance a.
1986                             #
1987                             # i think we could always propagate in this case, rather
1988                             # than complaining. either way, this isn't a REJECT issue
1989                             #
1990                             # And - we really should complain to the dorks who configured dak
1991                             self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
1992                             self.pkg.changes.setdefault("propdistribution", {})
1993                             self.pkg.changes["propdistribution"][addsuite] = 1
1994                             cansave = 1
1995                         elif not target_version:
1996                             # not targets_version is true when the package is NEW
1997                             # we could just stick with the "...old version..." REJECT
1998                             # for this, I think.
1999                             self.rejects.append("Won't propogate NEW packages.")
2000                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2001                             # propogation would be redundant. no need to reject though.
2002                             self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2003                             cansave = 1
2004                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2005                              apt_pkg.VersionCompare(add_version, target_version) >= 0:
2006                             # propogate!!
2007                             self.warnings.append("Propogating upload to %s" % (addsuite))
2008                             self.pkg.changes.setdefault("propdistribution", {})
2009                             self.pkg.changes["propdistribution"][addsuite] = 1
2010                             cansave = 1
2011
2012                     if not cansave:
2013                         self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2014
2015     ################################################################################
2016     def check_binary_against_db(self, file, session):
2017         # Ensure version is sane
2018         q = session.query(BinAssociation)
2019         q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
2020         q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
2021
2022         self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2023                                        file, self.pkg.files[file]["version"], sourceful=False)
2024
2025         # Check for any existing copies of the file
2026         q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
2027         q = q.filter_by(version=self.pkg.files[file]["version"])
2028         q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
2029
2030         if q.count() > 0:
2031             self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
2032
2033     ################################################################################
2034
2035     def check_source_against_db(self, file, session):
2036         """
2037         """
2038         source = self.pkg.dsc.get("source")
2039         version = self.pkg.dsc.get("version")
2040
2041         # Ensure version is sane
2042         q = session.query(SrcAssociation)
2043         q = q.join(DBSource).filter(DBSource.source==source)
2044
2045         self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2046                                        file, version, sourceful=True)
2047
2048     ################################################################################
2049     def check_dsc_against_db(self, file, session):
2050         """
2051
2052         @warning: NB: this function can remove entries from the 'files' index [if
2053          the orig tarball is a duplicate of the one in the archive]; if
2054          you're iterating over 'files' and call this function as part of
2055          the loop, be sure to add a check to the top of the loop to
2056          ensure you haven't just tried to dereference the deleted entry.
2057
2058         """
2059
2060         Cnf = Config()
2061         self.pkg.orig_files = {} # XXX: do we need to clear it?
2062         orig_files = self.pkg.orig_files
2063
2064         # Try and find all files mentioned in the .dsc.  This has
2065         # to work harder to cope with the multiple possible
2066         # locations of an .orig.tar.gz.
2067         # The ordering on the select is needed to pick the newest orig
2068         # when it exists in multiple places.
2069         for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2070             found = None
2071             if self.pkg.files.has_key(dsc_name):
2072                 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2073                 actual_size = int(self.pkg.files[dsc_name]["size"])
2074                 found = "%s in incoming" % (dsc_name)
2075
2076                 # Check the file does not already exist in the archive
2077                 ql = get_poolfile_like_name(dsc_name, session)
2078
2079                 # Strip out anything that isn't '%s' or '/%s$'
2080                 for i in ql:
2081                     if not i.filename.endswith(dsc_name):
2082                         ql.remove(i)
2083
2084                 # "[dak] has not broken them.  [dak] has fixed a
2085                 # brokenness.  Your crappy hack exploited a bug in
2086                 # the old dinstall.
2087                 #
2088                 # "(Come on!  I thought it was always obvious that
2089                 # one just doesn't release different files with
2090                 # the same name and version.)"
2091                 #                        -- ajk@ on d-devel@l.d.o
2092
2093                 if len(ql) > 0:
2094                     # Ignore exact matches for .orig.tar.gz
2095                     match = 0
2096                     if re_is_orig_source.match(dsc_name):
2097                         for i in ql:
2098                             if self.pkg.files.has_key(dsc_name) and \
2099                                int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2100                                self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2101                                 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2102                                 # TODO: Don't delete the entry, just mark it as not needed
2103                                 # This would fix the stupidity of changing something we often iterate over
2104                                 # whilst we're doing it
2105                                 del self.pkg.files[dsc_name]
2106                                 if not orig_files.has_key(dsc_name):
2107                                     orig_files[dsc_name] = {}
2108                                 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
2109                                 match = 1
2110
2111                     if not match:
2112                         self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2113
2114             elif re_is_orig_source.match(dsc_name):
2115                 # Check in the pool
2116                 ql = get_poolfile_like_name(dsc_name, session)
2117
2118                 # Strip out anything that isn't '%s' or '/%s$'
2119                 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2120                 for i in ql:
2121                     if not i.filename.endswith(dsc_name):
2122                         ql.remove(i)
2123
2124                 if len(ql) > 0:
2125                     # Unfortunately, we may get more than one match here if,
2126                     # for example, the package was in potato but had an -sa
2127                     # upload in woody.  So we need to choose the right one.
2128
2129                     # default to something sane in case we don't match any or have only one
2130                     x = ql[0]
2131
2132                     if len(ql) > 1:
2133                         for i in ql:
2134                             old_file = os.path.join(i.location.path, i.filename)
2135                             old_file_fh = utils.open_file(old_file)
2136                             actual_md5 = apt_pkg.md5sum(old_file_fh)
2137                             old_file_fh.close()
2138                             actual_size = os.stat(old_file)[stat.ST_SIZE]
2139                             if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2140                                 x = i
2141
2142                     old_file = os.path.join(i.location.path, i.filename)
2143                     old_file_fh = utils.open_file(old_file)
2144                     actual_md5 = apt_pkg.md5sum(old_file_fh)
2145                     old_file_fh.close()
2146                     actual_size = os.stat(old_file)[stat.ST_SIZE]
2147                     found = old_file
2148                     suite_type = x.location.archive_type
2149                     # need this for updating dsc_files in install()
2150                     dsc_entry["files id"] = x.file_id
2151                     # See install() in process-accepted...
2152                     if not orig_files.has_key(dsc_name):
2153                         orig_files[dsc_name] = {}
2154                     orig_files[dsc_name]["id"] = x.file_id
2155                     orig_files[dsc_name]["path"] = old_file
2156                     orig_files[dsc_name]["location"] = x.location.location_id
2157                 else:
2158                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2159                     # Not there? Check the queue directories...
2160                     for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2161                         if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2162                             continue
2163                         in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2164                         if os.path.exists(in_otherdir):
2165                             in_otherdir_fh = utils.open_file(in_otherdir)
2166                             actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2167                             in_otherdir_fh.close()
2168                             actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2169                             found = in_otherdir
2170                             if not orig_files.has_key(dsc_name):
2171                                 orig_files[dsc_name] = {}
2172                             orig_files[dsc_name]["path"] = in_otherdir
2173
2174                     if not found:
2175                         self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
2176                         continue
2177             else:
2178                 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
2179                 continue
2180             if actual_md5 != dsc_entry["md5sum"]:
2181                 self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
2182             if actual_size != int(dsc_entry["size"]):
2183                 self.rejects.append("size for %s doesn't match %s." % (found, file))
2184
2185     ################################################################################
2186     def accepted_checks(self, overwrite_checks, session):
2187         # Recheck anything that relies on the database; since that's not
2188         # frozen between accept and our run time when called from p-a.
2189
2190         # overwrite_checks is set to False when installing to stable/oldstable
2191
2192         propogate={}
2193         nopropogate={}
2194
2195         # Find the .dsc (again)
2196         dsc_filename = None
2197         for f in self.pkg.files.keys():
2198             if self.pkg.files[f]["type"] == "dsc":
2199                 dsc_filename = f
2200
2201         for checkfile in self.pkg.files.keys():
2202             # The .orig.tar.gz can disappear out from under us is it's a
2203             # duplicate of one in the archive.
2204             if not self.pkg.files.has_key(checkfile):
2205                 continue
2206
2207             entry = self.pkg.files[checkfile]
2208
2209             # Check that the source still exists
2210             if entry["type"] == "deb":
2211                 source_version = entry["source version"]
2212                 source_package = entry["source package"]
2213                 if not self.pkg.changes["architecture"].has_key("source") \
2214                    and not source_exists(source_package, source_version,  self.pkg.changes["distribution"].keys()):
2215                     self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2216
2217             # Version and file overwrite checks
2218             if overwrite_checks:
2219                 if entry["type"] == "deb":
2220                     self.check_binary_against_db(checkfile, session)
2221                 elif entry["type"] == "dsc":
2222                     self.check_source_against_db(checkfile, session)
2223                     self.check_dsc_against_db(dsc_filename, session)
2224
2225             # propogate in the case it is in the override tables:
2226             for suite in self.pkg.changes.get("propdistribution", {}).keys():
2227                 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2228                     propogate[suite] = 1
2229                 else:
2230                     nopropogate[suite] = 1
2231
2232         for suite in propogate.keys():
2233             if suite in nopropogate:
2234                 continue
2235             self.pkg.changes["distribution"][suite] = 1
2236
2237         for checkfile in self.pkg.files.keys():
2238             # Check the package is still in the override tables
2239             for suite in self.pkg.changes["distribution"].keys():
2240                 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2241                     self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2242
2243     ################################################################################
2244     # This is not really a reject, but an unaccept, but since a) the code for
2245     # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2246     # extremely rare, for now we'll go with whining at our admin folks...
2247
2248     def do_unaccept(self):
2249         cnf = Config()
2250
2251         self.update_subst()
2252         self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2253         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2254         self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2255         self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
2256         if cnf.has_key("Dinstall::Bcc"):
2257             self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2258
2259         template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2260
2261         reject_mail_message = utils.TemplateSubst(self.Subst, template)
2262
2263         # Write the rejection email out as the <foo>.reason file
2264         reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2265         reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2266
2267         # If we fail here someone is probably trying to exploit the race
2268         # so let's just raise an exception ...
2269         if os.path.exists(reject_filename):
2270             os.unlink(reject_filename)
2271
2272         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2273         os.write(fd, reject_mail_message)
2274         os.close(fd)
2275
2276         utils.send_mail(reject_mail_message)
2277
2278         del self.Subst["__REJECTOR_ADDRESS__"]
2279         del self.Subst["__REJECT_MESSAGE__"]
2280         del self.Subst["__CC__"]
2281
2282     ################################################################################
2283     # If any file of an upload has a recent mtime then chances are good
2284     # the file is still being uploaded.
2285
2286     def upload_too_new(self):
2287         cnf = Config()
2288         too_new = False
2289         # Move back to the original directory to get accurate time stamps
2290         cwd = os.getcwd()
2291         os.chdir(self.pkg.directory)
2292         file_list = self.pkg.files.keys()
2293         file_list.extend(self.pkg.dsc_files.keys())
2294         file_list.append(self.pkg.changes_file)
2295         for f in file_list:
2296             try:
2297                 last_modified = time.time()-os.path.getmtime(f)
2298                 if last_modified < int(cnf["Dinstall::SkipTime"]):
2299                     too_new = True
2300                     break
2301             except:
2302                 pass
2303
2304         os.chdir(cwd)
2305         return too_new