daklib/queue.py

   1 #!/usr/bin/env python
   2 # vim:set et sw=4:
   3
   4 """
   5 Queue utility functions for dak
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
   9 @copyright: 2009, 2010  Joerg Jaspert <joerg@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ###############################################################################
  28
  29 import errno
  30 import os
  31 import stat
  32 import sys
  33 import time
  34 import apt_inst
  35 import apt_pkg
  36 import utils
  37 import commands
  38 import shutil
  39 import textwrap
  40 from types import *
  41 from sqlalchemy.sql.expression import desc
  42 from sqlalchemy.orm.exc import NoResultFound
  43
  44 import yaml
  45
  46 from dak_exceptions import *
  47 from changes import *
  48 from regexes import *
  49 from config import Config
  50 from holding import Holding
  51 from urgencylog import UrgencyLog
  52 from dbconn import *
  53 from summarystats import SummaryStats
  54 from utils import parse_changes, check_dsc_files, build_package_set
  55 from textutils import fix_maintainer
  56 from lintian import parse_lintian_output, generate_reject_messages
  57 from contents import UnpackedSource
  58
  59 # suppress some deprecation warnings in squeeze related to apt_pkg
  60 # module
  61 import warnings
  62 warnings.filterwarnings('ignore', \
  63     "apt_pkg.ParseSection\(\) is deprecated. Please see apt_pkg\.TagSection\(\) for the replacement\.", \
  64     DeprecationWarning)
  65 warnings.filterwarnings('ignore', \
  66     "Attribute 'Find' of the 'apt_pkg\.TagSection' object is deprecated, use 'find' instead\.", \
  67     DeprecationWarning)
  68
  69 ###############################################################################
  70
  71 def get_type(f, session):
  72     """
  73     Get the file type of C{f}
  74
  75     @type f: dict
  76     @param f: file entry from Changes object
  77
  78     @type session: SQLA Session
  79     @param session: SQL Alchemy session object
  80
  81     @rtype: string
  82     @return: filetype
  83
  84     """
  85     # Determine the type
  86     if f.has_key("dbtype"):
  87         file_type = f["dbtype"]
  88     elif re_source_ext.match(f["type"]):
  89         file_type = "dsc"
  90     else:
  91         file_type = f["type"]
  92         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))
  93
  94     # Validate the override type
  95     type_id = get_override_type(file_type, session)
  96     if type_id is None:
  97         utils.fubar("invalid type (%s) for new.  Say wha?" % (file_type))
  98
  99     return file_type
 100
 101 ################################################################################
 102
 103 # Determine what parts in a .changes are NEW
 104
 105 def determine_new(filename, changes, files, warn=1, session = None, dsc = None, new = {}):
 106     """
 107     Determine what parts in a C{changes} file are NEW.
 108
 109     @type filename: str
 110     @param filename: changes filename
 111
 112     @type changes: Upload.Pkg.changes dict
 113     @param changes: Changes dictionary
 114
 115     @type files: Upload.Pkg.files dict
 116     @param files: Files dictionary
 117
 118     @type warn: bool
 119     @param warn: Warn if overrides are added for (old)stable
 120
 121     @type dsc: Upload.Pkg.dsc dict
 122     @param dsc: (optional); Dsc dictionary
 123
 124     @type new: dict
 125     @param new: new packages as returned by a previous call to this function, but override information may have changed
 126
 127     @rtype: dict
 128     @return: dictionary of NEW components.
 129
 130     """
 131     # TODO: This should all use the database instead of parsing the changes
 132     # file again
 133     byhand = {}
 134
 135     dbchg = get_dbchange(filename, session)
 136     if dbchg is None:
 137         print "Warning: cannot find changes file in database; won't check byhand"
 138
 139     # Try to get the Package-Set field from an included .dsc file (if possible).
 140     if dsc:
 141         for package, entry in build_package_set(dsc, session).items():
 142             if not new.has_key(package):
 143                 new[package] = entry
 144
 145     # Build up a list of potentially new things
 146     for name, f in files.items():
 147         # Keep a record of byhand elements
 148         if f["section"] == "byhand":
 149             byhand[name] = 1
 150             continue
 151
 152         pkg = f["package"]
 153         priority = f["priority"]
 154         section = f["section"]
 155         file_type = get_type(f, session)
 156         component = f["component"]
 157
 158         if file_type == "dsc":
 159             priority = "source"
 160
 161         if not new.has_key(pkg):
 162             new[pkg] = {}
 163             new[pkg]["priority"] = priority
 164             new[pkg]["section"] = section
 165             new[pkg]["type"] = file_type
 166             new[pkg]["component"] = component
 167             new[pkg]["files"] = []
 168         else:
 169             old_type = new[pkg]["type"]
 170             if old_type != file_type:
 171                 # source gets trumped by deb or udeb
 172                 if old_type == "dsc":
 173                     new[pkg]["priority"] = priority
 174                     new[pkg]["section"] = section
 175                     new[pkg]["type"] = file_type
 176                     new[pkg]["component"] = component
 177
 178         new[pkg]["files"].append(name)
 179
 180         if f.has_key("othercomponents"):
 181             new[pkg]["othercomponents"] = f["othercomponents"]
 182
 183     # Fix up the list of target suites
 184     cnf = Config()
 185     for suite in changes["suite"].keys():
 186         oldsuite = get_suite(suite, session)
 187         if not oldsuite:
 188             print "WARNING: Invalid suite %s found" % suite
 189             continue
 190
 191         if oldsuite.overridesuite:
 192             newsuite = get_suite(oldsuite.overridesuite, session)
 193
 194             if newsuite:
 195                 print "INFORMATION: Using overrides from suite %s instead of suite %s" % (
 196                     oldsuite.overridesuite, suite)
 197                 del changes["suite"][suite]
 198                 changes["suite"][oldsuite.overridesuite] = 1
 199             else:
 200                 print "WARNING: Told to use overridesuite %s for %s but it doesn't exist.  Bugger" % (
 201                     oldsuite.overridesuite, suite)
 202
 203     # Check for unprocessed byhand files
 204     if dbchg is not None:
 205         for b in byhand.keys():
 206             # Find the file entry in the database
 207             found = False
 208             for f in dbchg.files:
 209                 if f.filename == b:
 210                     found = True
 211                     # If it's processed, we can ignore it
 212                     if f.processed:
 213                         del byhand[b]
 214                     break
 215
 216             if not found:
 217                 print "Warning: Couldn't find BYHAND item %s in the database; assuming unprocessed"
 218
 219     # Check for new stuff
 220     for suite in changes["suite"].keys():
 221         for pkg in new.keys():
 222             ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
 223             if len(ql) > 0:
 224                 for file_entry in new[pkg]["files"]:
 225                     if files[file_entry].has_key("new"):
 226                         del files[file_entry]["new"]
 227                 del new[pkg]
 228
 229     if warn:
 230         for s in ['stable', 'oldstable']:
 231             if changes["suite"].has_key(s):
 232                 print "WARNING: overrides will be added for %s!" % s
 233         for pkg in new.keys():
 234             if new[pkg].has_key("othercomponents"):
 235                 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 236
 237     return new, byhand
 238
 239 ################################################################################
 240
 241 def check_valid(new, session = None):
 242     """
 243     Check if section and priority for NEW packages exist in database.
 244     Additionally does sanity checks:
 245       - debian-installer packages have to be udeb (or source)
 246       - non debian-installer packages can not be udeb
 247       - source priority can only be assigned to dsc file types
 248
 249     @type new: dict
 250     @param new: Dict of new packages with their section, priority and type.
 251
 252     """
 253     for pkg in new.keys():
 254         section_name = new[pkg]["section"]
 255         priority_name = new[pkg]["priority"]
 256         file_type = new[pkg]["type"]
 257
 258         section = get_section(section_name, session)
 259         if section is None:
 260             new[pkg]["section id"] = -1
 261         else:
 262             new[pkg]["section id"] = section.section_id
 263
 264         priority = get_priority(priority_name, session)
 265         if priority is None:
 266             new[pkg]["priority id"] = -1
 267         else:
 268             new[pkg]["priority id"] = priority.priority_id
 269
 270         # Sanity checks
 271         di = section_name.find("debian-installer") != -1
 272
 273         # If d-i, we must be udeb and vice-versa
 274         if     (di and file_type not in ("udeb", "dsc")) or \
 275            (not di and file_type == "udeb"):
 276             new[pkg]["section id"] = -1
 277
 278         # If dsc we need to be source and vice-versa
 279         if (priority == "source" and file_type != "dsc") or \
 280            (priority != "source" and file_type == "dsc"):
 281             new[pkg]["priority id"] = -1
 282
 283 ###############################################################################
 284
 285 # Used by Upload.check_timestamps
 286 class TarTime(object):
 287     def __init__(self, future_cutoff, past_cutoff):
 288         self.reset()
 289         self.future_cutoff = future_cutoff
 290         self.past_cutoff = past_cutoff
 291
 292     def reset(self):
 293         self.future_files = {}
 294         self.ancient_files = {}
 295
 296     def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
 297         if MTime > self.future_cutoff:
 298             self.future_files[Name] = MTime
 299         if MTime < self.past_cutoff:
 300             self.ancient_files[Name] = MTime
 301
 302 ###############################################################################
 303
 304 def prod_maintainer(notes, upload):
 305     cnf = Config()
 306
 307     # Here we prepare an editor and get them ready to prod...
 308     (fd, temp_filename) = utils.temp_filename()
 309     temp_file = os.fdopen(fd, 'w')
 310     for note in notes:
 311         temp_file.write(note.comment)
 312     temp_file.close()
 313     editor = os.environ.get("EDITOR","vi")
 314     answer = 'E'
 315     while answer == 'E':
 316         os.system("%s %s" % (editor, temp_filename))
 317         temp_fh = utils.open_file(temp_filename)
 318         prod_message = "".join(temp_fh.readlines())
 319         temp_fh.close()
 320         print "Prod message:"
 321         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 322         prompt = "[P]rod, Edit, Abandon, Quit ?"
 323         answer = "XXX"
 324         while prompt.find(answer) == -1:
 325             answer = utils.our_raw_input(prompt)
 326             m = re_default_answer.search(prompt)
 327             if answer == "":
 328                 answer = m.group(1)
 329             answer = answer[:1].upper()
 330     os.unlink(temp_filename)
 331     if answer == 'A':
 332         return
 333     elif answer == 'Q':
 334         end()
 335         sys.exit(0)
 336     # Otherwise, do the proding...
 337     user_email_address = utils.whoami() + " <%s>" % (
 338         cnf["Dinstall::MyAdminAddress"])
 339
 340     Subst = upload.Subst
 341
 342     Subst["__FROM_ADDRESS__"] = user_email_address
 343     Subst["__PROD_MESSAGE__"] = prod_message
 344     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
 345
 346     prod_mail_message = utils.TemplateSubst(
 347         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 348
 349     # Send the prod mail
 350     utils.send_mail(prod_mail_message)
 351
 352     print "Sent prodding message"
 353
 354 ################################################################################
 355
 356 def edit_note(note, upload, session, trainee=False):
 357     # Write the current data to a temporary file
 358     (fd, temp_filename) = utils.temp_filename()
 359     editor = os.environ.get("EDITOR","vi")
 360     answer = 'E'
 361     while answer == 'E':
 362         os.system("%s %s" % (editor, temp_filename))
 363         temp_file = utils.open_file(temp_filename)
 364         newnote = temp_file.read().rstrip()
 365         temp_file.close()
 366         print "New Note:"
 367         print utils.prefix_multi_line_string(newnote,"  ")
 368         prompt = "[D]one, Edit, Abandon, Quit ?"
 369         answer = "XXX"
 370         while prompt.find(answer) == -1:
 371             answer = utils.our_raw_input(prompt)
 372             m = re_default_answer.search(prompt)
 373             if answer == "":
 374                 answer = m.group(1)
 375             answer = answer[:1].upper()
 376     os.unlink(temp_filename)
 377     if answer == 'A':
 378         return
 379     elif answer == 'Q':
 380         end()
 381         sys.exit(0)
 382
 383     comment = NewComment()
 384     comment.package = upload.pkg.changes["source"]
 385     comment.version = upload.pkg.changes["version"]
 386     comment.comment = newnote
 387     comment.author  = utils.whoami()
 388     comment.trainee = trainee
 389     session.add(comment)
 390     session.commit()
 391
 392 ###############################################################################
 393
 394 # suite names DMs can upload to
 395 dm_suites = ['unstable', 'experimental']
 396
 397 def get_newest_source(source, session):
 398     'returns the newest DBSource object in dm_suites'
 399     ## the most recent version of the package uploaded to unstable or
 400     ## experimental includes the field "DM-Upload-Allowed: yes" in the source
 401     ## section of its control file
 402     q = session.query(DBSource).filter_by(source = source). \
 403         filter(DBSource.suites.any(Suite.suite_name.in_(dm_suites))). \
 404         order_by(desc('source.version'))
 405     return q.first()
 406
 407 def get_suite_version_by_source(source, session):
 408     'returns a list of tuples (suite_name, version) for source package'
 409     q = session.query(Suite.suite_name, DBSource.version). \
 410         join(Suite.sources).filter_by(source = source)
 411     return q.all()
 412
 413 def get_source_by_package_and_suite(package, suite_name, session):
 414     '''
 415     returns a DBSource query filtered by DBBinary.package and this package's
 416     suite_name
 417     '''
 418     return session.query(DBSource). \
 419         join(DBSource.binaries).filter_by(package = package). \
 420         join(DBBinary.suites).filter_by(suite_name = suite_name)
 421
 422 def get_suite_version_by_package(package, arch_string, session):
 423     '''
 424     returns a list of tuples (suite_name, version) for binary package and
 425     arch_string
 426     '''
 427     return session.query(Suite.suite_name, DBBinary.version). \
 428         join(Suite.binaries).filter_by(package = package). \
 429         join(DBBinary.architecture). \
 430         filter(Architecture.arch_string.in_([arch_string, 'all'])).all()
 431
 432 class Upload(object):
 433     """
 434     Everything that has to do with an upload processed.
 435
 436     """
 437     def __init__(self):
 438         self.logger = None
 439         self.pkg = Changes()
 440         self.reset()
 441
 442     ###########################################################################
 443
 444     def reset (self):
 445         """ Reset a number of internal variables."""
 446
 447         # Initialize the substitution template map
 448         cnf = Config()
 449         self.Subst = {}
 450         self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
 451         self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
 452         self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
 453         self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 454
 455         self.rejects = []
 456         self.warnings = []
 457         self.notes = []
 458
 459         self.later_check_files = []
 460
 461         self.pkg.reset()
 462
 463     def package_info(self):
 464         """
 465         Format various messages from this Upload to send to the maintainer.
 466         """
 467
 468         msgs = (
 469             ('Reject Reasons', self.rejects),
 470             ('Warnings', self.warnings),
 471             ('Notes', self.notes),
 472         )
 473
 474         msg = ''
 475         for title, messages in msgs:
 476             if messages:
 477                 msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))
 478         msg += '\n\n'
 479
 480         return msg
 481
 482     ###########################################################################
 483     def update_subst(self):
 484         """ Set up the per-package template substitution mappings """
 485
 486         cnf = Config()
 487
 488         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
 489         if not self.pkg.changes.has_key("architecture") or not \
 490            isinstance(self.pkg.changes["architecture"], dict):
 491             self.pkg.changes["architecture"] = { "Unknown" : "" }
 492
 493         # and maintainer2047 may not exist.
 494         if not self.pkg.changes.has_key("maintainer2047"):
 495             self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
 496
 497         self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
 498         self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
 499         self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
 500
 501         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 502         if self.pkg.changes["architecture"].has_key("source") and \
 503            self.pkg.changes["changedby822"] != "" and \
 504            (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
 505
 506             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
 507             self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
 508             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
 509         else:
 510             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
 511             self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
 512             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
 513
 514         # Process policy doesn't set the fingerprint field and I don't want to make it
 515         # do it for now as I don't want to have to deal with the case where we accepted
 516         # the package into PU-NEW, but the fingerprint has gone away from the keyring in
 517         # the meantime so the package will be remarked as rejectable.  Urgh.
 518         # TODO: Fix this properly
 519         if self.pkg.changes.has_key('fingerprint'):
 520             session = DBConn().session()
 521             fpr = get_fingerprint(self.pkg.changes['fingerprint'], session)
 522             if fpr and self.check_if_upload_is_sponsored("%s@debian.org" % fpr.uid.uid, fpr.uid.name):
 523                 if self.pkg.changes.has_key("sponsoremail"):
 524                     self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
 525             session.close()
 526
 527         if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
 528             self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
 529
 530         # Apply any global override of the Maintainer field
 531         if cnf.get("Dinstall::OverrideMaintainer"):
 532             self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
 533             self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
 534
 535         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
 536         self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
 537         self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
 538         self.Subst["__SUITE__"] = ", ".join(self.pkg.changes["distribution"])
 539
 540     ###########################################################################
 541     def load_changes(self, filename):
 542         """
 543         Load a changes file and setup a dictionary around it. Also checks for mandantory
 544         fields  within.
 545
 546         @type filename: string
 547         @param filename: Changes filename, full path.
 548
 549         @rtype: boolean
 550         @return: whether the changes file was valid or not.  We may want to
 551                  reject even if this is True (see what gets put in self.rejects).
 552                  This is simply to prevent us even trying things later which will
 553                  fail because we couldn't properly parse the file.
 554         """
 555         Cnf = Config()
 556         self.pkg.changes_file = filename
 557
 558         # Parse the .changes field into a dictionary
 559         try:
 560             self.pkg.changes.update(parse_changes(filename))
 561         except CantOpenError:
 562             self.rejects.append("%s: can't read file." % (filename))
 563             return False
 564         except ParseChangesError, line:
 565             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 566             return False
 567         except ChangesUnicodeError:
 568             self.rejects.append("%s: changes file not proper utf-8" % (filename))
 569             return False
 570
 571         # Parse the Files field from the .changes into another dictionary
 572         try:
 573             self.pkg.files.update(utils.build_file_list(self.pkg.changes))
 574         except ParseChangesError, line:
 575             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
 576             return False
 577         except UnknownFormatError, format:
 578             self.rejects.append("%s: unknown format '%s'." % (filename, format))
 579             return False
 580
 581         # Check for mandatory fields
 582         for i in ("distribution", "source", "binary", "architecture",
 583                   "version", "maintainer", "files", "changes", "description"):
 584             if not self.pkg.changes.has_key(i):
 585                 # Avoid undefined errors later
 586                 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
 587                 return False
 588
 589         # Strip a source version in brackets from the source field
 590         if re_strip_srcver.search(self.pkg.changes["source"]):
 591             self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
 592
 593         # Ensure the source field is a valid package name.
 594         if not re_valid_pkg_name.match(self.pkg.changes["source"]):
 595             self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
 596
 597         # Split multi-value fields into a lower-level dictionary
 598         for i in ("architecture", "distribution", "binary", "closes"):
 599             o = self.pkg.changes.get(i, "")
 600             if o != "":
 601                 del self.pkg.changes[i]
 602
 603             self.pkg.changes[i] = {}
 604
 605             for j in o.split():
 606                 self.pkg.changes[i][j] = 1
 607
 608         # Fix the Maintainer: field to be RFC822/2047 compatible
 609         try:
 610             (self.pkg.changes["maintainer822"],
 611              self.pkg.changes["maintainer2047"],
 612              self.pkg.changes["maintainername"],
 613              self.pkg.changes["maintaineremail"]) = \
 614                    fix_maintainer (self.pkg.changes["maintainer"])
 615         except ParseMaintError, msg:
 616             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
 617                    % (filename, self.pkg.changes["maintainer"], msg))
 618
 619         # ...likewise for the Changed-By: field if it exists.
 620         try:
 621             (self.pkg.changes["changedby822"],
 622              self.pkg.changes["changedby2047"],
 623              self.pkg.changes["changedbyname"],
 624              self.pkg.changes["changedbyemail"]) = \
 625                    fix_maintainer (self.pkg.changes.get("changed-by", ""))
 626         except ParseMaintError, msg:
 627             self.pkg.changes["changedby822"] = ""
 628             self.pkg.changes["changedby2047"] = ""
 629             self.pkg.changes["changedbyname"] = ""
 630             self.pkg.changes["changedbyemail"] = ""
 631
 632             self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
 633                    % (filename, self.pkg.changes["changed-by"], msg))
 634
 635         # Ensure all the values in Closes: are numbers
 636         if self.pkg.changes.has_key("closes"):
 637             for i in self.pkg.changes["closes"].keys():
 638                 if re_isanum.match (i) == None:
 639                     self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
 640
 641         # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
 642         self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
 643         self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
 644
 645         # Check the .changes is non-empty
 646         if not self.pkg.files:
 647             self.rejects.append("%s: nothing to do (Files field is empty)." % (os.path.basename(self.pkg.changes_file)))
 648             return False
 649
 650         # Changes was syntactically valid even if we'll reject
 651         return True
 652
 653     ###########################################################################
 654
 655     def check_distributions(self):
 656         "Check and map the Distribution field"
 657
 658         Cnf = Config()
 659
 660         # Handle suite mappings
 661         for m in Cnf.ValueList("SuiteMappings"):
 662             args = m.split()
 663             mtype = args[0]
 664             if mtype == "map" or mtype == "silent-map":
 665                 (source, dest) = args[1:3]
 666                 if self.pkg.changes["distribution"].has_key(source):
 667                     del self.pkg.changes["distribution"][source]
 668                     self.pkg.changes["distribution"][dest] = 1
 669                     if mtype != "silent-map":
 670                         self.notes.append("Mapping %s to %s." % (source, dest))
 671                 if self.pkg.changes.has_key("distribution-version"):
 672                     if self.pkg.changes["distribution-version"].has_key(source):
 673                         self.pkg.changes["distribution-version"][source]=dest
 674             elif mtype == "map-unreleased":
 675                 (source, dest) = args[1:3]
 676                 if self.pkg.changes["distribution"].has_key(source):
 677                     for arch in self.pkg.changes["architecture"].keys():
 678                         if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
 679                             self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
 680                             del self.pkg.changes["distribution"][source]
 681                             self.pkg.changes["distribution"][dest] = 1
 682                             break
 683             elif mtype == "ignore":
 684                 suite = args[1]
 685                 if self.pkg.changes["distribution"].has_key(suite):
 686                     del self.pkg.changes["distribution"][suite]
 687                     self.warnings.append("Ignoring %s as a target suite." % (suite))
 688             elif mtype == "reject":
 689                 suite = args[1]
 690                 if self.pkg.changes["distribution"].has_key(suite):
 691                     self.rejects.append("Uploads to %s are not accepted." % (suite))
 692             elif mtype == "propup-version":
 693                 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
 694                 #
 695                 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
 696                 if self.pkg.changes["distribution"].has_key(args[1]):
 697                     self.pkg.changes.setdefault("distribution-version", {})
 698                     for suite in args[2:]:
 699                         self.pkg.changes["distribution-version"][suite] = suite
 700
 701         # Ensure there is (still) a target distribution
 702         if len(self.pkg.changes["distribution"].keys()) < 1:
 703             self.rejects.append("No valid distribution remaining.")
 704
 705         # Ensure target distributions exist
 706         for suite in self.pkg.changes["distribution"].keys():
 707             if not Cnf.has_key("Suite::%s" % (suite)):
 708                 self.rejects.append("Unknown distribution `%s'." % (suite))
 709
 710     ###########################################################################
 711
 712     def binary_file_checks(self, f, session):
 713         cnf = Config()
 714         entry = self.pkg.files[f]
 715
 716         # Extract package control information
 717         deb_file = utils.open_file(f)
 718         try:
 719             control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
 720         except:
 721             self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
 722             deb_file.close()
 723             # Can't continue, none of the checks on control would work.
 724             return
 725
 726         # Check for mandantory "Description:"
 727         deb_file.seek(0)
 728         try:
 729             apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
 730         except:
 731             self.rejects.append("%s: Missing Description in binary package" % (f))
 732             return
 733
 734         deb_file.close()
 735
 736         # Check for mandatory fields
 737         for field in [ "Package", "Architecture", "Version" ]:
 738             if control.Find(field) == None:
 739                 # Can't continue
 740                 self.rejects.append("%s: No %s field in control." % (f, field))
 741                 return
 742
 743         # Ensure the package name matches the one give in the .changes
 744         if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
 745             self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
 746
 747         # Validate the package field
 748         package = control.Find("Package")
 749         if not re_valid_pkg_name.match(package):
 750             self.rejects.append("%s: invalid package name '%s'." % (f, package))
 751
 752         # Validate the version field
 753         version = control.Find("Version")
 754         if not re_valid_version.match(version):
 755             self.rejects.append("%s: invalid version number '%s'." % (f, version))
 756
 757         # Ensure the architecture of the .deb is one we know about.
 758         default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
 759         architecture = control.Find("Architecture")
 760         upload_suite = self.pkg.changes["distribution"].keys()[0]
 761
 762         if      architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session = session)] \
 763             and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session = session)]:
 764             self.rejects.append("Unknown architecture '%s'." % (architecture))
 765
 766         # Ensure the architecture of the .deb is one of the ones
 767         # listed in the .changes.
 768         if not self.pkg.changes["architecture"].has_key(architecture):
 769             self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
 770
 771         # Sanity-check the Depends field
 772         depends = control.Find("Depends")
 773         if depends == '':
 774             self.rejects.append("%s: Depends field is empty." % (f))
 775
 776         # Sanity-check the Provides field
 777         provides = control.Find("Provides")
 778         if provides:
 779             provide = re_spacestrip.sub('', provides)
 780             if provide == '':
 781                 self.rejects.append("%s: Provides field is empty." % (f))
 782             prov_list = provide.split(",")
 783             for prov in prov_list:
 784                 if not re_valid_pkg_name.match(prov):
 785                     self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
 786
 787         # If there is a Built-Using field, we need to check we can find the
 788         # exact source version
 789         built_using = control.Find("Built-Using")
 790         if built_using:
 791             try:
 792                 entry["built-using"] = []
 793                 for dep in apt_pkg.parse_depends(built_using):
 794                     bu_s, bu_v, bu_e = dep[0]
 795                     # Check that it's an exact match dependency and we have
 796                     # some form of version
 797                     if bu_e != "=" or len(bu_v) < 1:
 798                         self.rejects.append("%s: Built-Using contains non strict dependency (%s %s %s)" % (f, bu_s, bu_e, bu_v))
 799                     else:
 800                         # Find the source id for this version
 801                         bu_so = get_sources_from_name(bu_s, version=bu_v, session = session)
 802                         if len(bu_so) != 1:
 803                             self.rejects.append("%s: Built-Using (%s = %s): Cannot find source package" % (f, bu_s, bu_v))
 804                         else:
 805                             entry["built-using"].append( (bu_so[0].source, bu_so[0].version, ) )
 806
 807             except ValueError, e:
 808                 self.rejects.append("%s: Cannot parse Built-Using field: %s" % (f, str(e)))
 809
 810
 811         # Check the section & priority match those given in the .changes (non-fatal)
 812         if     control.Find("Section") and entry["section"] != "" \
 813            and entry["section"] != control.Find("Section"):
 814             self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
 815                                 (f, control.Find("Section", ""), entry["section"]))
 816         if control.Find("Priority") and entry["priority"] != "" \
 817            and entry["priority"] != control.Find("Priority"):
 818             self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
 819                                 (f, control.Find("Priority", ""), entry["priority"]))
 820
 821         entry["package"] = package
 822         entry["architecture"] = architecture
 823         entry["version"] = version
 824         entry["maintainer"] = control.Find("Maintainer", "")
 825
 826         if f.endswith(".udeb"):
 827             self.pkg.files[f]["dbtype"] = "udeb"
 828         elif f.endswith(".deb"):
 829             self.pkg.files[f]["dbtype"] = "deb"
 830         else:
 831             self.rejects.append("%s is neither a .deb or a .udeb." % (f))
 832
 833         entry["source"] = control.Find("Source", entry["package"])
 834
 835         # Get the source version
 836         source = entry["source"]
 837         source_version = ""
 838
 839         if source.find("(") != -1:
 840             m = re_extract_src_version.match(source)
 841             source = m.group(1)
 842             source_version = m.group(2)
 843
 844         if not source_version:
 845             source_version = self.pkg.files[f]["version"]
 846
 847         entry["source package"] = source
 848         entry["source version"] = source_version
 849
 850         # Ensure the filename matches the contents of the .deb
 851         m = re_isadeb.match(f)
 852
 853         #  package name
 854         file_package = m.group(1)
 855         if entry["package"] != file_package:
 856             self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
 857                                 (f, file_package, entry["dbtype"], entry["package"]))
 858         epochless_version = re_no_epoch.sub('', control.Find("Version"))
 859
 860         #  version
 861         file_version = m.group(2)
 862         if epochless_version != file_version:
 863             self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
 864                                 (f, file_version, entry["dbtype"], epochless_version))
 865
 866         #  architecture
 867         file_architecture = m.group(3)
 868         if entry["architecture"] != file_architecture:
 869             self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
 870                                 (f, file_architecture, entry["dbtype"], entry["architecture"]))
 871
 872         # Check for existent source
 873         source_version = entry["source version"]
 874         source_package = entry["source package"]
 875         if self.pkg.changes["architecture"].has_key("source"):
 876             if source_version != self.pkg.changes["version"]:
 877                 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
 878                                     (source_version, f, self.pkg.changes["version"]))
 879         else:
 880             # Check in the SQL database
 881             if not source_exists(source_package, source_version, suites = \
 882                 self.pkg.changes["distribution"].keys(), session = session):
 883                 # Check in one of the other directories
 884                 source_epochless_version = re_no_epoch.sub('', source_version)
 885                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
 886                 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
 887                     entry["byhand"] = 1
 888                 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
 889                     entry["new"] = 1
 890                 else:
 891                     dsc_file_exists = False
 892                     for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
 893                         if cnf.has_key("Dir::Queue::%s" % (myq)):
 894                             if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
 895                                 dsc_file_exists = True
 896                                 break
 897
 898                     if not dsc_file_exists:
 899                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
 900
 901         # Check the version and for file overwrites
 902         self.check_binary_against_db(f, session)
 903
 904     def source_file_checks(self, f, session):
 905         entry = self.pkg.files[f]
 906
 907         m = re_issource.match(f)
 908         if not m:
 909             return
 910
 911         entry["package"] = m.group(1)
 912         entry["version"] = m.group(2)
 913         entry["type"] = m.group(3)
 914
 915         # Ensure the source package name matches the Source filed in the .changes
 916         if self.pkg.changes["source"] != entry["package"]:
 917             self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
 918
 919         # Ensure the source version matches the version in the .changes file
 920         if re_is_orig_source.match(f):
 921             changes_version = self.pkg.changes["chopversion2"]
 922         else:
 923             changes_version = self.pkg.changes["chopversion"]
 924
 925         if changes_version != entry["version"]:
 926             self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
 927
 928         # Ensure the .changes lists source in the Architecture field
 929         if not self.pkg.changes["architecture"].has_key("source"):
 930             self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
 931
 932         # Check the signature of a .dsc file
 933         if entry["type"] == "dsc":
 934             # check_signature returns either:
 935             #  (None, [list, of, rejects]) or (signature, [])
 936             (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
 937             for j in rejects:
 938                 self.rejects.append(j)
 939
 940         entry["architecture"] = "source"
 941
 942     def per_suite_file_checks(self, f, suite, session):
 943         cnf = Config()
 944         entry = self.pkg.files[f]
 945
 946         # Skip byhand
 947         if entry.has_key("byhand"):
 948             return
 949
 950         # Check we have fields we need to do these checks
 951         oktogo = True
 952         for m in ['component', 'package', 'priority', 'size', 'md5sum']:
 953             if not entry.has_key(m):
 954                 self.rejects.append("file '%s' does not have field %s set" % (f, m))
 955                 oktogo = False
 956
 957         if not oktogo:
 958             return
 959
 960         # Handle component mappings
 961         for m in cnf.ValueList("ComponentMappings"):
 962             (source, dest) = m.split()
 963             if entry["component"] == source:
 964                 entry["original component"] = source
 965                 entry["component"] = dest
 966
 967         # Ensure the component is valid for the target suite
 968         if cnf.has_key("Suite:%s::Components" % (suite)) and \
 969            entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
 970             self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
 971             return
 972
 973         # Validate the component
 974         if not get_component(entry["component"], session):
 975             self.rejects.append("file '%s' has unknown component '%s'." % (f, entry["component"]))
 976             return
 977
 978         # See if the package is NEW
 979         if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
 980             entry["new"] = 1
 981
 982         # Validate the priority
 983         if entry["priority"].find('/') != -1:
 984             self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
 985
 986         # Determine the location
 987         location = cnf["Dir::Pool"]
 988         l = get_location(location, entry["component"], session=session)
 989         if l is None:
 990             self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s)" % entry["component"])
 991             entry["location id"] = -1
 992         else:
 993             entry["location id"] = l.location_id
 994
 995         # Check the md5sum & size against existing files (if any)
 996         entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
 997
 998         found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
 999                                          entry["size"], entry["md5sum"], entry["location id"])
1000
1001         if found is None:
1002             self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
1003         elif found is False and poolfile is not None:
1004             self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
1005         else:
1006             if poolfile is None:
1007                 entry["files id"] = None
1008             else:
1009                 entry["files id"] = poolfile.file_id
1010
1011         # Check for packages that have moved from one component to another
1012         entry['suite'] = suite
1013         arch_list = [entry["architecture"], 'all']
1014         component = get_component_by_package_suite(self.pkg.files[f]['package'], \
1015             [suite], arch_list = arch_list, session = session)
1016         if component is not None:
1017             entry["othercomponents"] = component
1018
1019     def check_files(self, action=True):
1020         file_keys = self.pkg.files.keys()
1021         holding = Holding()
1022         cnf = Config()
1023
1024         if action:
1025             cwd = os.getcwd()
1026             os.chdir(self.pkg.directory)
1027             for f in file_keys:
1028                 ret = holding.copy_to_holding(f)
1029                 if ret is not None:
1030                     self.warnings.append('Could not copy %s to holding; will attempt to find in DB later' % f)
1031
1032             os.chdir(cwd)
1033
1034         # check we already know the changes file
1035         # [NB: this check must be done post-suite mapping]
1036         base_filename = os.path.basename(self.pkg.changes_file)
1037
1038         session = DBConn().session()
1039
1040         try:
1041             dbc = session.query(DBChange).filter_by(changesname=base_filename).one()
1042             # if in the pool or in a queue other than unchecked, reject
1043             if (dbc.in_queue is None) \
1044                    or (dbc.in_queue is not None
1045                        and dbc.in_queue.queue_name not in ["unchecked", "newstage"]):
1046                 self.rejects.append("%s file already known to dak" % base_filename)
1047         except NoResultFound, e:
1048             # not known, good
1049             pass
1050
1051         has_binaries = False
1052         has_source = False
1053
1054         for f, entry in self.pkg.files.items():
1055             # Ensure the file does not already exist in one of the accepted directories
1056             for d in [ "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
1057                 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
1058                 if os.path.exists(os.path.join(cnf["Dir::Queue::%s" % (d) ], f)):
1059                     self.rejects.append("%s file already exists in the %s directory." % (f, d))
1060
1061             if not re_taint_free.match(f):
1062                 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
1063
1064             # Check the file is readable
1065             if os.access(f, os.R_OK) == 0:
1066                 # When running in -n, copy_to_holding() won't have
1067                 # generated the reject_message, so we need to.
1068                 if action:
1069                     if os.path.exists(f):
1070                         self.rejects.append("Can't read `%s'. [permission denied]" % (f))
1071                     else:
1072                         # Don't directly reject, mark to check later to deal with orig's
1073                         # we can find in the pool
1074                         self.later_check_files.append(f)
1075                 entry["type"] = "unreadable"
1076                 continue
1077
1078             # If it's byhand skip remaining checks
1079             if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
1080                 entry["byhand"] = 1
1081                 entry["type"] = "byhand"
1082
1083             # Checks for a binary package...
1084             elif re_isadeb.match(f):
1085                 has_binaries = True
1086                 entry["type"] = "deb"
1087
1088                 # This routine appends to self.rejects/warnings as appropriate
1089                 self.binary_file_checks(f, session)
1090
1091             # Checks for a source package...
1092             elif re_issource.match(f):
1093                 has_source = True
1094
1095                 # This routine appends to self.rejects/warnings as appropriate
1096                 self.source_file_checks(f, session)
1097
1098             # Not a binary or source package?  Assume byhand...
1099             else:
1100                 entry["byhand"] = 1
1101                 entry["type"] = "byhand"
1102
1103             # Per-suite file checks
1104             entry["oldfiles"] = {}
1105             for suite in self.pkg.changes["distribution"].keys():
1106                 self.per_suite_file_checks(f, suite, session)
1107
1108         session.close()
1109
1110         # If the .changes file says it has source, it must have source.
1111         if self.pkg.changes["architecture"].has_key("source"):
1112             if not has_source:
1113                 self.rejects.append("no source found and Architecture line in changes mention source.")
1114
1115             if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
1116                 self.rejects.append("source only uploads are not supported.")
1117
1118     ###########################################################################
1119
1120     def __dsc_filename(self):
1121         """
1122         Returns: (Status, Dsc_Filename)
1123         where
1124           Status: Boolean; True when there was no error, False otherwise
1125           Dsc_Filename: String; name of the dsc file if Status is True, reason for the error otherwise
1126         """
1127         dsc_filename = None
1128
1129         # find the dsc
1130         for name, entry in self.pkg.files.items():
1131             if entry.has_key("type") and entry["type"] == "dsc":
1132                 if dsc_filename:
1133                     return False, "cannot process a .changes file with multiple .dsc's."
1134                 else:
1135                     dsc_filename = name
1136
1137         if not dsc_filename:
1138             return False, "source uploads must contain a dsc file"
1139
1140         return True, dsc_filename
1141
1142     def load_dsc(self, action=True, signing_rules=1):
1143         """
1144         Find and load the dsc from self.pkg.files into self.dsc
1145
1146         Returns: (Status, Reason)
1147         where
1148           Status: Boolean; True when there was no error, False otherwise
1149           Reason: String; When Status is False this describes the error
1150         """
1151
1152         # find the dsc
1153         (status, dsc_filename) = self.__dsc_filename()
1154         if not status:
1155             # If status is false, dsc_filename has the reason
1156             return False, dsc_filename
1157
1158         try:
1159             self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=signing_rules, dsc_file=1))
1160         except CantOpenError:
1161             if not action:
1162                 return False, "%s: can't read file." % (dsc_filename)
1163         except ParseChangesError, line:
1164             return False, "%s: parse error, can't grok: %s." % (dsc_filename, line)
1165         except InvalidDscError, line:
1166             return False, "%s: syntax error on line %s." % (dsc_filename, line)
1167         except ChangesUnicodeError:
1168             return False, "%s: dsc file not proper utf-8." % (dsc_filename)
1169
1170         return True, None
1171
1172     ###########################################################################
1173
1174     def check_dsc(self, action=True, session=None):
1175         """Returns bool indicating whether or not the source changes are valid"""
1176         # Ensure there is source to check
1177         if not self.pkg.changes["architecture"].has_key("source"):
1178             return True
1179
1180         (status, reason) = self.load_dsc(action=action)
1181         if not status:
1182             self.rejects.append(reason)
1183             return False
1184         (status, dsc_filename) = self.__dsc_filename()
1185         if not status:
1186             # If status is false, dsc_filename has the reason
1187             self.rejects.append(dsc_filename)
1188             return False
1189
1190         # Build up the file list of files mentioned by the .dsc
1191         try:
1192             self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
1193         except NoFilesFieldError:
1194             self.rejects.append("%s: no Files: field." % (dsc_filename))
1195             return False
1196         except UnknownFormatError, format:
1197             self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
1198             return False
1199         except ParseChangesError, line:
1200             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
1201             return False
1202
1203         # Enforce mandatory fields
1204         for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
1205             if not self.pkg.dsc.has_key(i):
1206                 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
1207                 return False
1208
1209         # Validate the source and version fields
1210         if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
1211             self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
1212         if not re_valid_version.match(self.pkg.dsc["version"]):
1213             self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
1214
1215         # Only a limited list of source formats are allowed in each suite
1216         for dist in self.pkg.changes["distribution"].keys():
1217             allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
1218             if self.pkg.dsc["format"] not in allowed:
1219                 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
1220
1221         # Validate the Maintainer field
1222         try:
1223             # We ignore the return value
1224             fix_maintainer(self.pkg.dsc["maintainer"])
1225         except ParseMaintError, msg:
1226             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
1227                                  % (dsc_filename, self.pkg.dsc["maintainer"], msg))
1228
1229         # Validate the build-depends field(s)
1230         for field_name in [ "build-depends", "build-depends-indep" ]:
1231             field = self.pkg.dsc.get(field_name)
1232             if field:
1233                 # Have apt try to parse them...
1234                 try:
1235                     apt_pkg.ParseSrcDepends(field)
1236                 except:
1237                     self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
1238
1239         # Ensure the version number in the .dsc matches the version number in the .changes
1240         epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1241         changes_version = self.pkg.files[dsc_filename]["version"]
1242
1243         if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1244             self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1245
1246         # Ensure the Files field contain only what's expected
1247         self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
1248
1249         # Ensure source is newer than existing source in target suites
1250         session = DBConn().session()
1251         self.check_source_against_db(dsc_filename, session)
1252         self.check_dsc_against_db(dsc_filename, session)
1253
1254         dbchg = get_dbchange(self.pkg.changes_file, session)
1255
1256         # Finally, check if we're missing any files
1257         for f in self.later_check_files:
1258             print 'XXX: %s' % f
1259             # Check if we've already processed this file if we have a dbchg object
1260             ok = False
1261             if dbchg:
1262                 for pf in dbchg.files:
1263                     if pf.filename == f and pf.processed:
1264                         self.notes.append('%s was already processed so we can go ahead' % f)
1265                         ok = True
1266                         del self.pkg.files[f]
1267             if not ok:
1268                 self.rejects.append("Could not find file %s references in changes" % f)
1269
1270         session.close()
1271
1272         return True
1273
1274     ###########################################################################
1275
1276     def get_changelog_versions(self, source_dir):
1277         """Extracts a the source package and (optionally) grabs the
1278         version history out of debian/changelog for the BTS."""
1279
1280         cnf = Config()
1281
1282         # Find the .dsc (again)
1283         dsc_filename = None
1284         for f in self.pkg.files.keys():
1285             if self.pkg.files[f]["type"] == "dsc":
1286                 dsc_filename = f
1287
1288         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1289         if not dsc_filename:
1290             return
1291
1292         # Create a symlink mirror of the source files in our temporary directory
1293         for f in self.pkg.files.keys():
1294             m = re_issource.match(f)
1295             if m:
1296                 src = os.path.join(source_dir, f)
1297                 # If a file is missing for whatever reason, give up.
1298                 if not os.path.exists(src):
1299                     return
1300                 ftype = m.group(3)
1301                 if re_is_orig_source.match(f) and self.pkg.orig_files.has_key(f) and \
1302                    self.pkg.orig_files[f].has_key("path"):
1303                     continue
1304                 dest = os.path.join(os.getcwd(), f)
1305                 os.symlink(src, dest)
1306
1307         # If the orig files are not a part of the upload, create symlinks to the
1308         # existing copies.
1309         for orig_file in self.pkg.orig_files.keys():
1310             if not self.pkg.orig_files[orig_file].has_key("path"):
1311                 continue
1312             dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
1313             os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
1314
1315         # Extract the source
1316         try:
1317             unpacked = UnpackedSource(dsc_filename)
1318         except:
1319             self.rejects.append("'dpkg-source -x' failed for %s." % dsc_filename)
1320             return
1321
1322         if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1323             return
1324
1325         # Get the upstream version
1326         upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1327         if re_strip_revision.search(upstr_version):
1328             upstr_version = re_strip_revision.sub('', upstr_version)
1329
1330         # Ensure the changelog file exists
1331         changelog_file = unpacked.get_changelog_file()
1332         if changelog_file is None:
1333             self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1334             return
1335
1336         # Parse the changelog
1337         self.pkg.dsc["bts changelog"] = ""
1338         for line in changelog_file.readlines():
1339             m = re_changelog_versions.match(line)
1340             if m:
1341                 self.pkg.dsc["bts changelog"] += line
1342         changelog_file.close()
1343         unpacked.cleanup()
1344
1345         # Check we found at least one revision in the changelog
1346         if not self.pkg.dsc["bts changelog"]:
1347             self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1348
1349     def check_source(self):
1350         # Bail out if:
1351         #    a) there's no source
1352         if not self.pkg.changes["architecture"].has_key("source"):
1353             return
1354
1355         tmpdir = utils.temp_dirname()
1356
1357         # Move into the temporary directory
1358         cwd = os.getcwd()
1359         os.chdir(tmpdir)
1360
1361         # Get the changelog version history
1362         self.get_changelog_versions(cwd)
1363
1364         # Move back and cleanup the temporary tree
1365         os.chdir(cwd)
1366
1367         try:
1368             shutil.rmtree(tmpdir)
1369         except OSError, e:
1370             if e.errno != errno.EACCES:
1371                 print "foobar"
1372                 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1373
1374             self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1375             # We probably have u-r or u-w directories so chmod everything
1376             # and try again.
1377             cmd = "chmod -R u+rwx %s" % (tmpdir)
1378             result = os.system(cmd)
1379             if result != 0:
1380                 utils.fubar("'%s' failed with result %s." % (cmd, result))
1381             shutil.rmtree(tmpdir)
1382         except Exception, e:
1383             print "foobar2 (%s)" % e
1384             utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1385
1386     ###########################################################################
1387     def ensure_hashes(self):
1388         # Make sure we recognise the format of the Files: field in the .changes
1389         format = self.pkg.changes.get("format", "0.0").split(".", 1)
1390         if len(format) == 2:
1391             format = int(format[0]), int(format[1])
1392         else:
1393             format = int(float(format[0])), 0
1394
1395         # We need to deal with the original changes blob, as the fields we need
1396         # might not be in the changes dict serialised into the .dak anymore.
1397         orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1398
1399         # Copy the checksums over to the current changes dict.  This will keep
1400         # the existing modifications to it intact.
1401         for field in orig_changes:
1402             if field.startswith('checksums-'):
1403                 self.pkg.changes[field] = orig_changes[field]
1404
1405         # Check for unsupported hashes
1406         for j in utils.check_hash_fields(".changes", self.pkg.changes):
1407             self.rejects.append(j)
1408
1409         for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1410             self.rejects.append(j)
1411
1412         # We have to calculate the hash if we have an earlier changes version than
1413         # the hash appears in rather than require it exist in the changes file
1414         for hashname, hashfunc, version in utils.known_hashes:
1415             # TODO: Move _ensure_changes_hash into this class
1416             for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1417                 self.rejects.append(j)
1418             if "source" in self.pkg.changes["architecture"]:
1419                 # TODO: Move _ensure_dsc_hash into this class
1420                 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1421                     self.rejects.append(j)
1422
1423     def check_hashes(self):
1424         for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1425             self.rejects.append(m)
1426
1427         for m in utils.check_size(".changes", self.pkg.files):
1428             self.rejects.append(m)
1429
1430         for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1431             self.rejects.append(m)
1432
1433         for m in utils.check_size(".dsc", self.pkg.dsc_files):
1434             self.rejects.append(m)
1435
1436         self.ensure_hashes()
1437
1438     ###########################################################################
1439
1440     def ensure_orig(self, target_dir='.', session=None):
1441         """
1442         Ensures that all orig files mentioned in the changes file are present
1443         in target_dir. If they do not exist, they are symlinked into place.
1444
1445         An list containing the symlinks that were created are returned (so they
1446         can be removed).
1447         """
1448
1449         symlinked = []
1450         cnf = Config()
1451
1452         for filename, entry in self.pkg.dsc_files.iteritems():
1453             if not re_is_orig_source.match(filename):
1454                 # File is not an orig; ignore
1455                 continue
1456
1457             if os.path.exists(filename):
1458                 # File exists, no need to continue
1459                 continue
1460
1461             def symlink_if_valid(path):
1462                 f = utils.open_file(path)
1463                 md5sum = apt_pkg.md5sum(f)
1464                 f.close()
1465
1466                 fingerprint = (os.stat(path)[stat.ST_SIZE], md5sum)
1467                 expected = (int(entry['size']), entry['md5sum'])
1468
1469                 if fingerprint != expected:
1470                     return False
1471
1472                 dest = os.path.join(target_dir, filename)
1473
1474                 os.symlink(path, dest)
1475                 symlinked.append(dest)
1476
1477                 return True
1478
1479             session_ = session
1480             if session is None:
1481                 session_ = DBConn().session()
1482
1483             found = False
1484
1485             # Look in the pool
1486             for poolfile in get_poolfile_like_name('%s' % filename, session_):
1487                 poolfile_path = os.path.join(
1488                     poolfile.location.path, poolfile.filename
1489                 )
1490
1491                 if symlink_if_valid(poolfile_path):
1492                     found = True
1493                     break
1494
1495             if session is None:
1496                 session_.close()
1497
1498             if found:
1499                 continue
1500
1501             # Look in some other queues for the file
1502             queues = ('New', 'Byhand', 'ProposedUpdates',
1503                 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
1504
1505             for queue in queues:
1506                 if not cnf.get('Dir::Queue::%s' % queue):
1507                     continue
1508
1509                 queuefile_path = os.path.join(
1510                     cnf['Dir::Queue::%s' % queue], filename
1511                 )
1512
1513                 if not os.path.exists(queuefile_path):
1514                     # Does not exist in this queue
1515                     continue
1516
1517                 if symlink_if_valid(queuefile_path):
1518                     break
1519
1520         return symlinked
1521
1522     ###########################################################################
1523
1524     def check_lintian(self):
1525         """
1526         Extends self.rejects by checking the output of lintian against tags
1527         specified in Dinstall::LintianTags.
1528         """
1529
1530         cnf = Config()
1531
1532         # Don't reject binary uploads
1533         if not self.pkg.changes['architecture'].has_key('source'):
1534             return
1535
1536         # Only check some distributions
1537         for dist in ('unstable', 'experimental'):
1538             if dist in self.pkg.changes['distribution']:
1539                 break
1540         else:
1541             return
1542
1543         # If we do not have a tagfile, don't do anything
1544         tagfile = cnf.get("Dinstall::LintianTags")
1545         if not tagfile:
1546             return
1547
1548         # Parse the yaml file
1549         sourcefile = file(tagfile, 'r')
1550         sourcecontent = sourcefile.read()
1551         sourcefile.close()
1552
1553         try:
1554             lintiantags = yaml.load(sourcecontent)['lintian']
1555         except yaml.YAMLError, msg:
1556             utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
1557             return
1558
1559         # Try and find all orig mentioned in the .dsc
1560         symlinked = self.ensure_orig()
1561
1562         # Setup the input file for lintian
1563         fd, temp_filename = utils.temp_filename()
1564         temptagfile = os.fdopen(fd, 'w')
1565         for tags in lintiantags.values():
1566             temptagfile.writelines(['%s\n' % x for x in tags])
1567         temptagfile.close()
1568
1569         try:
1570             cmd = "lintian --show-overrides --tags-from-file %s %s" % \
1571                 (temp_filename, self.pkg.changes_file)
1572
1573             result, output = commands.getstatusoutput(cmd)
1574         finally:
1575             # Remove our tempfile and any symlinks we created
1576             os.unlink(temp_filename)
1577
1578             for symlink in symlinked:
1579                 os.unlink(symlink)
1580
1581         if result == 2:
1582             utils.warn("lintian failed for %s [return code: %s]." % \
1583                 (self.pkg.changes_file, result))
1584             utils.warn(utils.prefix_multi_line_string(output, \
1585                 " [possible output:] "))
1586
1587         def log(*txt):
1588             if self.logger:
1589                 self.logger.log(
1590                     [self.pkg.changes_file, "check_lintian"] + list(txt)
1591                 )
1592
1593         # Generate messages
1594         parsed_tags = parse_lintian_output(output)
1595         self.rejects.extend(
1596             generate_reject_messages(parsed_tags, lintiantags, log=log)
1597         )
1598
1599     ###########################################################################
1600     def check_urgency(self):
1601         cnf = Config()
1602         if self.pkg.changes["architecture"].has_key("source"):
1603             if not self.pkg.changes.has_key("urgency"):
1604                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1605             self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1606             if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1607                 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1608                                      (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1609                 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1610
1611     ###########################################################################
1612
1613     # Sanity check the time stamps of files inside debs.
1614     # [Files in the near future cause ugly warnings and extreme time
1615     #  travel can cause errors on extraction]
1616
1617     def check_timestamps(self):
1618         Cnf = Config()
1619
1620         future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1621         past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1622         tar = TarTime(future_cutoff, past_cutoff)
1623
1624         for filename, entry in self.pkg.files.items():
1625             if entry["type"] == "deb":
1626                 tar.reset()
1627                 try:
1628                     deb_file = utils.open_file(filename)
1629                     apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1630                     deb_file.seek(0)
1631                     try:
1632                         apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1633                     except SystemError, e:
1634                         # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1635                         if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1636                             raise
1637                         deb_file.seek(0)
1638                         apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1639
1640                     deb_file.close()
1641
1642                     future_files = tar.future_files.keys()
1643                     if future_files:
1644                         num_future_files = len(future_files)
1645                         future_file = future_files[0]
1646                         future_date = tar.future_files[future_file]
1647                         self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1648                                % (filename, num_future_files, future_file, time.ctime(future_date)))
1649
1650                     ancient_files = tar.ancient_files.keys()
1651                     if ancient_files:
1652                         num_ancient_files = len(ancient_files)
1653                         ancient_file = ancient_files[0]
1654                         ancient_date = tar.ancient_files[ancient_file]
1655                         self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1656                                % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1657                 except:
1658                     self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1659
1660     def check_if_upload_is_sponsored(self, uid_email, uid_name):
1661         if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1662             sponsored = False
1663         elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1664             sponsored = False
1665             if uid_name == "":
1666                 sponsored = True
1667         else:
1668             sponsored = True
1669             if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
1670                 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1671                 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1672                     self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1673                         self.pkg.changes["sponsoremail"] = uid_email
1674
1675         return sponsored
1676
1677
1678     ###########################################################################
1679     # check_signed_by_key checks
1680     ###########################################################################
1681
1682     def check_signed_by_key(self):
1683         """Ensure the .changes is signed by an authorized uploader."""
1684         session = DBConn().session()
1685
1686         # First of all we check that the person has proper upload permissions
1687         # and that this upload isn't blocked
1688         fpr = get_fingerprint(self.pkg.changes['fingerprint'], session=session)
1689
1690         if fpr is None:
1691             self.rejects.append("Cannot find fingerprint %s" % self.pkg.changes["fingerprint"])
1692             return
1693
1694         # TODO: Check that import-keyring adds UIDs properly
1695         if not fpr.uid:
1696             self.rejects.append("Cannot find uid for fingerprint %s.  Please contact ftpmaster@debian.org" % fpr.fingerprint)
1697             return
1698
1699         # Check that the fingerprint which uploaded has permission to do so
1700         self.check_upload_permissions(fpr, session)
1701
1702         # Check that this package is not in a transition
1703         self.check_transition(session)
1704
1705         session.close()
1706
1707
1708     def check_upload_permissions(self, fpr, session):
1709         # Check any one-off upload blocks
1710         self.check_upload_blocks(fpr, session)
1711
1712         # Start with DM as a special case
1713         # DM is a special case unfortunately, so we check it first
1714         # (keys with no source access get more access than DMs in one
1715         #  way; DMs can only upload for their packages whether source
1716         #  or binary, whereas keys with no access might be able to
1717         #  upload some binaries)
1718         if fpr.source_acl.access_level == 'dm':
1719             self.check_dm_upload(fpr, session)
1720         else:
1721             # Check source-based permissions for other types
1722             if self.pkg.changes["architecture"].has_key("source") and \
1723                 fpr.source_acl.access_level is None:
1724                 rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
1725                 rej += '\nPlease contact ftpmaster if you think this is incorrect'
1726                 self.rejects.append(rej)
1727                 return
1728             # If not a DM, we allow full upload rights
1729             uid_email = "%s@debian.org" % (fpr.uid.uid)
1730             self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
1731
1732
1733         # Check binary upload permissions
1734         # By this point we know that DMs can't have got here unless they
1735         # are allowed to deal with the package concerned so just apply
1736         # normal checks
1737         if fpr.binary_acl.access_level == 'full':
1738             return
1739
1740         # Otherwise we're in the map case
1741         tmparches = self.pkg.changes["architecture"].copy()
1742         tmparches.pop('source', None)
1743
1744         for bam in fpr.binary_acl_map:
1745             tmparches.pop(bam.architecture.arch_string, None)
1746
1747         if len(tmparches.keys()) > 0:
1748             if fpr.binary_reject:
1749                 rej = ".changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
1750                 rej += "\narchitectures involved are: ", ",".join(tmparches.keys())
1751                 self.rejects.append(rej)
1752             else:
1753                 # TODO: This is where we'll implement reject vs throw away binaries later
1754                 rej = "Uhm.  I'm meant to throw away the binaries now but that's not implemented yet"
1755                 rej += "\nPlease complain to ftpmaster@debian.org as this shouldn't have been turned on"
1756                 rej += "\nFingerprint: %s", (fpr.fingerprint)
1757                 self.rejects.append(rej)
1758
1759
1760     def check_upload_blocks(self, fpr, session):
1761         """Check whether any upload blocks apply to this source, source
1762            version, uid / fpr combination"""
1763
1764         def block_rej_template(fb):
1765             rej = 'Manual upload block in place for package %s' % fb.source
1766             if fb.version is not None:
1767                 rej += ', version %s' % fb.version
1768             return rej
1769
1770         for fb in session.query(UploadBlock).filter_by(source = self.pkg.changes['source']).all():
1771             # version is None if the block applies to all versions
1772             if fb.version is None or fb.version == self.pkg.changes['version']:
1773                 # Check both fpr and uid - either is enough to cause a reject
1774                 if fb.fpr is not None:
1775                     if fb.fpr.fingerprint == fpr.fingerprint:
1776                         self.rejects.append(block_rej_template(fb) + ' for fingerprint %s\nReason: %s' % (fpr.fingerprint, fb.reason))
1777                 if fb.uid is not None:
1778                     if fb.uid == fpr.uid:
1779                         self.rejects.append(block_rej_template(fb) + ' for uid %s\nReason: %s' % (fb.uid.uid, fb.reason))
1780
1781
1782     def check_dm_upload(self, fpr, session):
1783         # Quoth the GR (http://www.debian.org/vote/2007/vote_003):
1784         ## none of the uploaded packages are NEW
1785         rej = False
1786         for f in self.pkg.files.keys():
1787             if self.pkg.files[f].has_key("byhand"):
1788                 self.rejects.append("%s may not upload BYHAND file %s" % (fpr.uid.uid, f))
1789                 rej = True
1790             if self.pkg.files[f].has_key("new"):
1791                 self.rejects.append("%s may not upload NEW file %s" % (fpr.uid.uid, f))
1792                 rej = True
1793
1794         if rej:
1795             return
1796
1797         r = get_newest_source(self.pkg.changes["source"], session)
1798
1799         if r is None:
1800             rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"]
1801             self.rejects.append(rej)
1802             return
1803
1804         if not r.dm_upload_allowed:
1805             rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version)
1806             self.rejects.append(rej)
1807             return
1808
1809         ## the Maintainer: field of the uploaded .changes file corresponds with
1810         ## the owner of the key used (ie, non-developer maintainers may not sponsor
1811         ## uploads)
1812         if self.check_if_upload_is_sponsored(fpr.uid.uid, fpr.uid.name):
1813             self.rejects.append("%s (%s) is not authorised to sponsor uploads" % (fpr.uid.uid, fpr.fingerprint))
1814
1815         ## the most recent version of the package uploaded to unstable or
1816         ## experimental lists the uploader in the Maintainer: or Uploaders: fields (ie,
1817         ## non-developer maintainers cannot NMU or hijack packages)
1818
1819         # srcuploaders includes the maintainer
1820         accept = False
1821         for sup in r.srcuploaders:
1822             (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1823             # Eww - I hope we never have two people with the same name in Debian
1824             if email == fpr.uid.uid or name == fpr.uid.name:
1825                 accept = True
1826                 break
1827
1828         if not accept:
1829             self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (fpr.uid.uid, self.pkg.changes["source"]))
1830             return
1831
1832         ## none of the packages are being taken over from other source packages
1833         for b in self.pkg.changes["binary"].keys():
1834             for suite in self.pkg.changes["distribution"].keys():
1835                 for s in get_source_by_package_and_suite(b, suite, session):
1836                     if s.source != self.pkg.changes["source"]:
1837                         self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite))
1838
1839
1840
1841     def check_transition(self, session):
1842         cnf = Config()
1843
1844         sourcepkg = self.pkg.changes["source"]
1845
1846         # No sourceful upload -> no need to do anything else, direct return
1847         # We also work with unstable uploads, not experimental or those going to some
1848         # proposed-updates queue
1849         if "source" not in self.pkg.changes["architecture"] or \
1850            "unstable" not in self.pkg.changes["distribution"]:
1851             return
1852
1853         # Also only check if there is a file defined (and existant) with
1854         # checks.
1855         transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1856         if transpath == "" or not os.path.exists(transpath):
1857             return
1858
1859         # Parse the yaml file
1860         sourcefile = file(transpath, 'r')
1861         sourcecontent = sourcefile.read()
1862         try:
1863             transitions = yaml.load(sourcecontent)
1864         except yaml.YAMLError, msg:
1865             # This shouldn't happen, there is a wrapper to edit the file which
1866             # checks it, but we prefer to be safe than ending up rejecting
1867             # everything.
1868             utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1869             return
1870
1871         # Now look through all defined transitions
1872         for trans in transitions:
1873             t = transitions[trans]
1874             source = t["source"]
1875             expected = t["new"]
1876
1877             # Will be None if nothing is in testing.
1878             current = get_source_in_suite(source, "testing", session)
1879             if current is not None:
1880                 compare = apt_pkg.VersionCompare(current.version, expected)
1881
1882             if current is None or compare < 0:
1883                 # This is still valid, the current version in testing is older than
1884                 # the new version we wait for, or there is none in testing yet
1885
1886                 # Check if the source we look at is affected by this.
1887                 if sourcepkg in t['packages']:
1888                     # The source is affected, lets reject it.
1889
1890                     rejectmsg = "%s: part of the %s transition.\n\n" % (
1891                         sourcepkg, trans)
1892
1893                     if current is not None:
1894                         currentlymsg = "at version %s" % (current.version)
1895                     else:
1896                         currentlymsg = "not present in testing"
1897
1898                     rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1899
1900                     rejectmsg += "\n".join(textwrap.wrap("""Your package
1901 is part of a testing transition designed to get %s migrated (it is
1902 currently %s, we need version %s).  This transition is managed by the
1903 Release Team, and %s is the Release-Team member responsible for it.
1904 Please mail debian-release@lists.debian.org or contact %s directly if you
1905 need further assistance.  You might want to upload to experimental until this
1906 transition is done."""
1907                             % (source, currentlymsg, expected,t["rm"], t["rm"])))
1908
1909                     self.rejects.append(rejectmsg)
1910                     return
1911
1912     ###########################################################################
1913     # End check_signed_by_key checks
1914     ###########################################################################
1915
1916     def build_summaries(self):
1917         """ Build a summary of changes the upload introduces. """
1918
1919         (byhand, new, summary, override_summary) = self.pkg.file_summary()
1920
1921         short_summary = summary
1922
1923         # This is for direport's benefit...
1924         f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1925
1926         if byhand or new:
1927             summary += "Changes: " + f
1928
1929         summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1930
1931         summary += self.announce(short_summary, 0)
1932
1933         return (summary, short_summary)
1934
1935     ###########################################################################
1936
1937     def close_bugs(self, summary, action):
1938         """
1939         Send mail to close bugs as instructed by the closes field in the changes file.
1940         Also add a line to summary if any work was done.
1941
1942         @type summary: string
1943         @param summary: summary text, as given by L{build_summaries}
1944
1945         @type action: bool
1946         @param action: Set to false no real action will be done.
1947
1948         @rtype: string
1949         @return: summary. If action was taken, extended by the list of closed bugs.
1950
1951         """
1952
1953         template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1954
1955         bugs = self.pkg.changes["closes"].keys()
1956
1957         if not bugs:
1958             return summary
1959
1960         bugs.sort()
1961         summary += "Closing bugs: "
1962         for bug in bugs:
1963             summary += "%s " % (bug)
1964             if action:
1965                 self.update_subst()
1966                 self.Subst["__BUG_NUMBER__"] = bug
1967                 if self.pkg.changes["distribution"].has_key("stable"):
1968                     self.Subst["__STABLE_WARNING__"] = """
1969 Note that this package is not part of the released stable Debian
1970 distribution.  It may have dependencies on other unreleased software,
1971 or other instabilities.  Please take care if you wish to install it.
1972 The update will eventually make its way into the next released Debian
1973 distribution."""
1974                 else:
1975                     self.Subst["__STABLE_WARNING__"] = ""
1976                 mail_message = utils.TemplateSubst(self.Subst, template)
1977                 utils.send_mail(mail_message)
1978
1979                 # Clear up after ourselves
1980                 del self.Subst["__BUG_NUMBER__"]
1981                 del self.Subst["__STABLE_WARNING__"]
1982
1983         if action and self.logger:
1984             self.logger.log(["closing bugs"] + bugs)
1985
1986         summary += "\n"
1987
1988         return summary
1989
1990     ###########################################################################
1991
1992     def announce(self, short_summary, action):
1993         """
1994         Send an announce mail about a new upload.
1995
1996         @type short_summary: string
1997         @param short_summary: Short summary text to include in the mail
1998
1999         @type action: bool
2000         @param action: Set to false no real action will be done.
2001
2002         @rtype: string
2003         @return: Textstring about action taken.
2004
2005         """
2006
2007         cnf = Config()
2008         announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
2009
2010         # Only do announcements for source uploads with a recent dpkg-dev installed
2011         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
2012            self.pkg.changes["architecture"].has_key("source"):
2013             return ""
2014
2015         lists_done = {}
2016         summary = ""
2017
2018         self.Subst["__SHORT_SUMMARY__"] = short_summary
2019
2020         for dist in self.pkg.changes["distribution"].keys():
2021             suite = get_suite(dist)
2022             if suite is None: continue
2023             announce_list = suite.announce
2024             if announce_list == "" or lists_done.has_key(announce_list):
2025                 continue
2026
2027             lists_done[announce_list] = 1
2028             summary += "Announcing to %s\n" % (announce_list)
2029
2030             if action:
2031                 self.update_subst()
2032                 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
2033                 if cnf.get("Dinstall::TrackingServer") and \
2034                    self.pkg.changes["architecture"].has_key("source"):
2035                     trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
2036                     self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
2037
2038                 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
2039                 utils.send_mail(mail_message)
2040
2041                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
2042
2043         if cnf.FindB("Dinstall::CloseBugs"):
2044             summary = self.close_bugs(summary, action)
2045
2046         del self.Subst["__SHORT_SUMMARY__"]
2047
2048         return summary
2049
2050     ###########################################################################
2051     @session_wrapper
2052     def accept (self, summary, short_summary, session=None):
2053         """
2054         Accept an upload.
2055
2056         This moves all files referenced from the .changes into the pool,
2057         sends the accepted mail, announces to lists, closes bugs and
2058         also checks for override disparities. If enabled it will write out
2059         the version history for the BTS Version Tracking and will finally call
2060         L{queue_build}.
2061
2062         @type summary: string
2063         @param summary: Summary text
2064
2065         @type short_summary: string
2066         @param short_summary: Short summary
2067         """
2068
2069         cnf = Config()
2070         stats = SummaryStats()
2071
2072         print "Installing."
2073         self.logger.log(["installing changes", self.pkg.changes_file])
2074
2075         binaries = []
2076         poolfiles = []
2077
2078         # Add the .dsc file to the DB first
2079         for newfile, entry in self.pkg.files.items():
2080             if entry["type"] == "dsc":
2081                 source, dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
2082                 for j in pfs:
2083                     poolfiles.append(j)
2084
2085         # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
2086         for newfile, entry in self.pkg.files.items():
2087             if entry["type"] == "deb":
2088                 b, pf = add_deb_to_db(self, newfile, session)
2089                 binaries.append(b)
2090                 poolfiles.append(pf)
2091
2092         # If this is a sourceful diff only upload that is moving
2093         # cross-component we need to copy the .orig files into the new
2094         # component too for the same reasons as above.
2095         # XXX: mhy: I think this should be in add_dsc_to_db
2096         if self.pkg.changes["architecture"].has_key("source"):
2097             for orig_file in self.pkg.orig_files.keys():
2098                 if not self.pkg.orig_files[orig_file].has_key("id"):
2099                     continue # Skip if it's not in the pool
2100                 orig_file_id = self.pkg.orig_files[orig_file]["id"]
2101                 if self.pkg.orig_files[orig_file]["location"] == dsc_location_id:
2102                     continue # Skip if the location didn't change
2103
2104                 # Do the move
2105                 oldf = get_poolfile_by_id(orig_file_id, session)
2106                 old_filename = os.path.join(oldf.location.path, oldf.filename)
2107                 old_dat = {'size': oldf.filesize,   'md5sum': oldf.md5sum,
2108                            'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
2109
2110                 new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
2111
2112                 # TODO: Care about size/md5sum collisions etc
2113                 (found, newf) = check_poolfile(new_filename, old_dat['size'], old_dat['md5sum'], dsc_location_id, session)
2114
2115                 # TODO: Uhm, what happens if newf isn't None - something has gone badly and we should cope
2116                 if newf is None:
2117                     utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
2118                     newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
2119
2120                     session.flush()
2121
2122                     # Don't reference the old file from this changes
2123                     for p in poolfiles:
2124                         if p.file_id == oldf.file_id:
2125                             poolfiles.remove(p)
2126
2127                     poolfiles.append(newf)
2128
2129                     # Fix up the DSC references
2130                     toremove = []
2131
2132                     for df in source.srcfiles:
2133                         if df.poolfile.file_id == oldf.file_id:
2134                             # Add a new DSC entry and mark the old one for deletion
2135                             # Don't do it in the loop so we don't change the thing we're iterating over
2136                             newdscf = DSCFile()
2137                             newdscf.source_id = source.source_id
2138                             newdscf.poolfile_id = newf.file_id
2139                             session.add(newdscf)
2140
2141                             toremove.append(df)
2142
2143                     for df in toremove:
2144                         session.delete(df)
2145
2146                     # Flush our changes
2147                     session.flush()
2148
2149                     # Make sure that our source object is up-to-date
2150                     session.expire(source)
2151
2152         # Add changelog information to the database
2153         self.store_changelog()
2154
2155         # Install the files into the pool
2156         for newfile, entry in self.pkg.files.items():
2157             destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
2158             utils.move(newfile, destination)
2159             self.logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
2160             stats.accept_bytes += float(entry["size"])
2161
2162         # Copy the .changes file across for suite which need it.
2163         copy_changes = dict([(x.copychanges, '')
2164                              for x in session.query(Suite).filter(Suite.suite_name.in_(self.pkg.changes["distribution"].keys())).all()
2165                              if x.copychanges is not None])
2166
2167         for dest in copy_changes.keys():
2168             utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
2169
2170         # We're done - commit the database changes
2171         session.commit()
2172         # Our SQL session will automatically start a new transaction after
2173         # the last commit
2174
2175         # Now ensure that the metadata has been added
2176         # This has to be done after we copy the files into the pool
2177         # For source if we have it:
2178         if self.pkg.changes["architecture"].has_key("source"):
2179             import_metadata_into_db(source, session)
2180
2181         # Now for any of our binaries
2182         for b in binaries:
2183             import_metadata_into_db(b, session)
2184
2185         session.commit()
2186
2187         # Move the .changes into the 'done' directory
2188         utils.move(self.pkg.changes_file,
2189                    os.path.join(cnf["Dir::Queue::Done"], os.path.basename(self.pkg.changes_file)))
2190
2191         if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"):
2192             UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])
2193
2194         self.update_subst()
2195         self.Subst["__SUMMARY__"] = summary
2196         mail_message = utils.TemplateSubst(self.Subst,
2197                                            os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted'))
2198         utils.send_mail(mail_message)
2199         self.announce(short_summary, 1)
2200
2201         ## Helper stuff for DebBugs Version Tracking
2202         if cnf.Find("Dir::Queue::BTSVersionTrack"):
2203             if self.pkg.changes["architecture"].has_key("source"):
2204                 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
2205                 version_history = os.fdopen(fd, 'w')
2206                 version_history.write(self.pkg.dsc["bts changelog"])
2207                 version_history.close()
2208                 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
2209                                       self.pkg.changes_file[:-8]+".versions")
2210                 os.rename(temp_filename, filename)
2211                 os.chmod(filename, 0644)
2212
2213             # Write out the binary -> source mapping.
2214             (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
2215             debinfo = os.fdopen(fd, 'w')
2216             for name, entry in sorted(self.pkg.files.items()):
2217                 if entry["type"] == "deb":
2218                     line = " ".join([entry["package"], entry["version"],
2219                                      entry["architecture"], entry["source package"],
2220                                      entry["source version"]])
2221                     debinfo.write(line+"\n")
2222             debinfo.close()
2223             filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
2224                                   self.pkg.changes_file[:-8]+".debinfo")
2225             os.rename(temp_filename, filename)
2226             os.chmod(filename, 0644)
2227
2228         session.commit()
2229
2230         # Set up our copy queues (e.g. buildd queues)
2231         for suite_name in self.pkg.changes["distribution"].keys():
2232             suite = get_suite(suite_name, session)
2233             for q in suite.copy_queues:
2234                 for f in poolfiles:
2235                     q.add_file_from_pool(f)
2236
2237         session.commit()
2238
2239         # Finally...
2240         stats.accept_count += 1
2241
2242     def check_override(self):
2243         """
2244         Checks override entries for validity. Mails "Override disparity" warnings,
2245         if that feature is enabled.
2246
2247         Abandons the check if
2248           - override disparity checks are disabled
2249           - mail sending is disabled
2250         """
2251
2252         cnf = Config()
2253
2254         # Abandon the check if override disparity checks have been disabled
2255         if not cnf.FindB("Dinstall::OverrideDisparityCheck"):
2256             return
2257
2258         summary = self.pkg.check_override()
2259
2260         if summary == "":
2261             return
2262
2263         overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
2264
2265         self.update_subst()
2266         self.Subst["__SUMMARY__"] = summary
2267         mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
2268         utils.send_mail(mail_message)
2269         del self.Subst["__SUMMARY__"]
2270
2271     ###########################################################################
2272
2273     def remove(self, from_dir=None):
2274         """
2275         Used (for instance) in p-u to remove the package from unchecked
2276
2277         Also removes the package from holding area.
2278         """
2279         if from_dir is None:
2280             from_dir = self.pkg.directory
2281         h = Holding()
2282
2283         for f in self.pkg.files.keys():
2284             os.unlink(os.path.join(from_dir, f))
2285             if os.path.exists(os.path.join(h.holding_dir, f)):
2286                 os.unlink(os.path.join(h.holding_dir, f))
2287
2288         os.unlink(os.path.join(from_dir, self.pkg.changes_file))
2289         if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
2290             os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
2291
2292     ###########################################################################
2293
2294     def move_to_queue (self, queue):
2295         """
2296         Move files to a destination queue using the permissions in the table
2297         """
2298         h = Holding()
2299         utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
2300                    queue.path, perms=int(queue.change_perms, 8))
2301         for f in self.pkg.files.keys():
2302             utils.move(os.path.join(h.holding_dir, f), queue.path, perms=int(queue.perms, 8))
2303
2304     ###########################################################################
2305
2306     def force_reject(self, reject_files):
2307         """
2308         Forcefully move files from the current directory to the
2309         reject directory.  If any file already exists in the reject
2310         directory it will be moved to the morgue to make way for
2311         the new file.
2312
2313         @type reject_files: dict
2314         @param reject_files: file dictionary
2315
2316         """
2317
2318         cnf = Config()
2319
2320         for file_entry in reject_files:
2321             # Skip any files which don't exist or which we don't have permission to copy.
2322             if os.access(file_entry, os.R_OK) == 0:
2323                 continue
2324
2325             dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
2326
2327             try:
2328                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
2329             except OSError, e:
2330                 # File exists?  Let's find a new name by adding a number
2331                 if e.errno == errno.EEXIST:
2332                     try:
2333                         dest_file = utils.find_next_free(dest_file, 255)
2334                     except NoFreeFilenameError:
2335                         # Something's either gone badly Pete Tong, or
2336                         # someone is trying to exploit us.
2337                         utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Queue::Reject"]))
2338                         return
2339
2340                     # Make sure we really got it
2341                     try:
2342                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2343                     except OSError, e:
2344                         # Likewise
2345                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
2346                         return
2347                 else:
2348                     raise
2349             # If we got here, we own the destination file, so we can
2350             # safely overwrite it.
2351             utils.move(file_entry, dest_file, 1, perms=0660)
2352             os.close(dest_fd)
2353
2354     ###########################################################################
2355     def do_reject (self, manual=0, reject_message="", notes=""):
2356         """
2357         Reject an upload. If called without a reject message or C{manual} is
2358         true, spawn an editor so the user can write one.
2359
2360         @type manual: bool
2361         @param manual: manual or automated rejection
2362
2363         @type reject_message: string
2364         @param reject_message: A reject message
2365
2366         @return: 0
2367
2368         """
2369         # If we weren't given a manual rejection message, spawn an
2370         # editor so the user can add one in...
2371         if manual and not reject_message:
2372             (fd, temp_filename) = utils.temp_filename()
2373             temp_file = os.fdopen(fd, 'w')
2374             if len(notes) > 0:
2375                 for note in notes:
2376                     temp_file.write("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
2377                                     % (note.author, note.version, note.notedate, note.comment))
2378             temp_file.close()
2379             editor = os.environ.get("EDITOR","vi")
2380             answer = 'E'
2381             while answer == 'E':
2382                 os.system("%s %s" % (editor, temp_filename))
2383                 temp_fh = utils.open_file(temp_filename)
2384                 reject_message = "".join(temp_fh.readlines())
2385                 temp_fh.close()
2386                 print "Reject message:"
2387                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1)
2388                 prompt = "[R]eject, Edit, Abandon, Quit ?"
2389                 answer = "XXX"
2390                 while prompt.find(answer) == -1:
2391                     answer = utils.our_raw_input(prompt)
2392                     m = re_default_answer.search(prompt)
2393                     if answer == "":
2394                         answer = m.group(1)
2395                     answer = answer[:1].upper()
2396             os.unlink(temp_filename)
2397             if answer == 'A':
2398                 return 1
2399             elif answer == 'Q':
2400                 sys.exit(0)
2401
2402         print "Rejecting.\n"
2403
2404         cnf = Config()
2405
2406         reason_filename = self.pkg.changes_file[:-8] + ".reason"
2407         reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2408
2409         # Move all the files into the reject directory
2410         reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
2411         self.force_reject(reject_files)
2412
2413         # If we fail here someone is probably trying to exploit the race
2414         # so let's just raise an exception ...
2415         if os.path.exists(reason_filename):
2416             os.unlink(reason_filename)
2417         reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2418
2419         rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
2420
2421         self.update_subst()
2422         if not manual:
2423             self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2424             self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
2425             self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)"
2426             os.write(reason_fd, reject_message)
2427             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2428         else:
2429             # Build up the rejection email
2430             user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
2431             self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
2432             self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
2433             self.Subst["__REJECT_MESSAGE__"] = ""
2434             self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2435             reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2436             # Write the rejection email out as the <foo>.reason file
2437             os.write(reason_fd, reject_mail_message)
2438
2439         del self.Subst["__REJECTOR_ADDRESS__"]
2440         del self.Subst["__MANUAL_REJECT_MESSAGE__"]
2441         del self.Subst["__CC__"]
2442
2443         os.close(reason_fd)
2444
2445         # Send the rejection mail
2446         utils.send_mail(reject_mail_message)
2447
2448         if self.logger:
2449             self.logger.log(["rejected", self.pkg.changes_file])
2450
2451         return 0
2452
2453     ################################################################################
2454     def in_override_p(self, package, component, suite, binary_type, filename, session):
2455         """
2456         Check if a package already has override entries in the DB
2457
2458         @type package: string
2459         @param package: package name
2460
2461         @type component: string
2462         @param component: database id of the component
2463
2464         @type suite: int
2465         @param suite: database id of the suite
2466
2467         @type binary_type: string
2468         @param binary_type: type of the package
2469
2470         @type filename: string
2471         @param filename: filename we check
2472
2473         @return: the database result. But noone cares anyway.
2474
2475         """
2476
2477         cnf = Config()
2478
2479         if binary_type == "": # must be source
2480             file_type = "dsc"
2481         else:
2482             file_type = binary_type
2483
2484         # Override suite name; used for example with proposed-updates
2485         oldsuite = get_suite(suite, session)
2486         if (not oldsuite is None) and oldsuite.overridesuite:
2487             suite = oldsuite.overridesuite
2488
2489         result = get_override(package, suite, component, file_type, session)
2490
2491         # If checking for a source package fall back on the binary override type
2492         if file_type == "dsc" and len(result) < 1:
2493             result = get_override(package, suite, component, ['deb', 'udeb'], session)
2494
2495         # Remember the section and priority so we can check them later if appropriate
2496         if len(result) > 0:
2497             result = result[0]
2498             self.pkg.files[filename]["override section"] = result.section.section
2499             self.pkg.files[filename]["override priority"] = result.priority.priority
2500             return result
2501
2502         return None
2503
2504     ################################################################################
2505     def get_anyversion(self, sv_list, suite):
2506         """
2507         @type sv_list: list
2508         @param sv_list: list of (suite, version) tuples to check
2509
2510         @type suite: string
2511         @param suite: suite name
2512
2513         Description: TODO
2514         """
2515         Cnf = Config()
2516         anyversion = None
2517         anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
2518         for (s, v) in sv_list:
2519             if s in [ x.lower() for x in anysuite ]:
2520                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
2521                     anyversion = v
2522
2523         return anyversion
2524
2525     ################################################################################
2526
2527     def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
2528         """
2529         @type sv_list: list
2530         @param sv_list: list of (suite, version) tuples to check
2531
2532         @type filename: string
2533         @param filename: XXX
2534
2535         @type new_version: string
2536         @param new_version: XXX
2537
2538         Ensure versions are newer than existing packages in target
2539         suites and that cross-suite version checking rules as
2540         set out in the conf file are satisfied.
2541         """
2542
2543         cnf = Config()
2544
2545         # Check versions for each target suite
2546         for target_suite in self.pkg.changes["distribution"].keys():
2547             must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
2548             must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
2549
2550             # Enforce "must be newer than target suite" even if conffile omits it
2551             if target_suite not in must_be_newer_than:
2552                 must_be_newer_than.append(target_suite)
2553
2554             for (suite, existent_version) in sv_list:
2555                 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
2556
2557                 if suite in must_be_newer_than and sourceful and vercmp < 1:
2558                     self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2559
2560                 if suite in must_be_older_than and vercmp > -1:
2561                     cansave = 0
2562
2563                     if self.pkg.changes.get('distribution-version', {}).has_key(suite):
2564                         # we really use the other suite, ignoring the conflicting one ...
2565                         addsuite = self.pkg.changes["distribution-version"][suite]
2566
2567                         add_version = self.get_anyversion(sv_list, addsuite)
2568                         target_version = self.get_anyversion(sv_list, target_suite)
2569
2570                         if not add_version:
2571                             # not add_version can only happen if we map to a suite
2572                             # that doesn't enhance the suite we're propup'ing from.
2573                             # so "propup-ver x a b c; map a d" is a problem only if
2574                             # d doesn't enhance a.
2575                             #
2576                             # i think we could always propagate in this case, rather
2577                             # than complaining. either way, this isn't a REJECT issue
2578                             #
2579                             # And - we really should complain to the dorks who configured dak
2580                             self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
2581                             self.pkg.changes.setdefault("propdistribution", {})
2582                             self.pkg.changes["propdistribution"][addsuite] = 1
2583                             cansave = 1
2584                         elif not target_version:
2585                             # not targets_version is true when the package is NEW
2586                             # we could just stick with the "...old version..." REJECT
2587                             # for this, I think.
2588                             self.rejects.append("Won't propogate NEW packages.")
2589                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2590                             # propogation would be redundant. no need to reject though.
2591                             self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2592                             cansave = 1
2593                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2594                              apt_pkg.VersionCompare(add_version, target_version) >= 0:
2595                             # propogate!!
2596                             self.warnings.append("Propogating upload to %s" % (addsuite))
2597                             self.pkg.changes.setdefault("propdistribution", {})
2598                             self.pkg.changes["propdistribution"][addsuite] = 1
2599                             cansave = 1
2600
2601                     if not cansave:
2602                         self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2603
2604     ################################################################################
2605     def check_binary_against_db(self, filename, session):
2606         # Ensure version is sane
2607         self.cross_suite_version_check( \
2608             get_suite_version_by_package(self.pkg.files[filename]["package"], \
2609                 self.pkg.files[filename]["architecture"], session),
2610             filename, self.pkg.files[filename]["version"], sourceful=False)
2611
2612         # Check for any existing copies of the file
2613         q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"])
2614         q = q.filter_by(version=self.pkg.files[filename]["version"])
2615         q = q.join(Architecture).filter_by(arch_string=self.pkg.files[filename]["architecture"])
2616
2617         if q.count() > 0:
2618             self.rejects.append("%s: can not overwrite existing copy already in the archive." % filename)
2619
2620     ################################################################################
2621
2622     def check_source_against_db(self, filename, session):
2623         source = self.pkg.dsc.get("source")
2624         version = self.pkg.dsc.get("version")
2625
2626         # Ensure version is sane
2627         self.cross_suite_version_check( \
2628             get_suite_version_by_source(source, session), filename, version,
2629             sourceful=True)
2630
2631     ################################################################################
2632     def check_dsc_against_db(self, filename, session):
2633         """
2634
2635         @warning: NB: this function can remove entries from the 'files' index [if
2636          the orig tarball is a duplicate of the one in the archive]; if
2637          you're iterating over 'files' and call this function as part of
2638          the loop, be sure to add a check to the top of the loop to
2639          ensure you haven't just tried to dereference the deleted entry.
2640
2641         """
2642
2643         Cnf = Config()
2644         self.pkg.orig_files = {} # XXX: do we need to clear it?
2645         orig_files = self.pkg.orig_files
2646
2647         # Try and find all files mentioned in the .dsc.  This has
2648         # to work harder to cope with the multiple possible
2649         # locations of an .orig.tar.gz.
2650         # The ordering on the select is needed to pick the newest orig
2651         # when it exists in multiple places.
2652         for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2653             found = None
2654             if self.pkg.files.has_key(dsc_name):
2655                 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2656                 actual_size = int(self.pkg.files[dsc_name]["size"])
2657                 found = "%s in incoming" % (dsc_name)
2658
2659                 # Check the file does not already exist in the archive
2660                 ql = get_poolfile_like_name(dsc_name, session)
2661
2662                 # Strip out anything that isn't '%s' or '/%s$'
2663                 for i in ql:
2664                     if not i.filename.endswith(dsc_name):
2665                         ql.remove(i)
2666
2667                 # "[dak] has not broken them.  [dak] has fixed a
2668                 # brokenness.  Your crappy hack exploited a bug in
2669                 # the old dinstall.
2670                 #
2671                 # "(Come on!  I thought it was always obvious that
2672                 # one just doesn't release different files with
2673                 # the same name and version.)"
2674                 #                        -- ajk@ on d-devel@l.d.o
2675
2676                 if len(ql) > 0:
2677                     # Ignore exact matches for .orig.tar.gz
2678                     match = 0
2679                     if re_is_orig_source.match(dsc_name):
2680                         for i in ql:
2681                             if self.pkg.files.has_key(dsc_name) and \
2682                                int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2683                                self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2684                                 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2685                                 # TODO: Don't delete the entry, just mark it as not needed
2686                                 # This would fix the stupidity of changing something we often iterate over
2687                                 # whilst we're doing it
2688                                 del self.pkg.files[dsc_name]
2689                                 dsc_entry["files id"] = i.file_id
2690                                 if not orig_files.has_key(dsc_name):
2691                                     orig_files[dsc_name] = {}
2692                                 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
2693                                 match = 1
2694
2695                                 # Don't bitch that we couldn't find this file later
2696                                 try:
2697                                     self.later_check_files.remove(dsc_name)
2698                                 except ValueError:
2699                                     pass
2700
2701
2702                     if not match:
2703                         self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2704
2705             elif re_is_orig_source.match(dsc_name):
2706                 # Check in the pool
2707                 ql = get_poolfile_like_name(dsc_name, session)
2708
2709                 # Strip out anything that isn't '%s' or '/%s$'
2710                 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2711                 for i in ql:
2712                     if not i.filename.endswith(dsc_name):
2713                         ql.remove(i)
2714
2715                 if len(ql) > 0:
2716                     # Unfortunately, we may get more than one match here if,
2717                     # for example, the package was in potato but had an -sa
2718                     # upload in woody.  So we need to choose the right one.
2719
2720                     # default to something sane in case we don't match any or have only one
2721                     x = ql[0]
2722
2723                     if len(ql) > 1:
2724                         for i in ql:
2725                             old_file = os.path.join(i.location.path, i.filename)
2726                             old_file_fh = utils.open_file(old_file)
2727                             actual_md5 = apt_pkg.md5sum(old_file_fh)
2728                             old_file_fh.close()
2729                             actual_size = os.stat(old_file)[stat.ST_SIZE]
2730                             if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2731                                 x = i
2732
2733                     old_file = os.path.join(i.location.path, i.filename)
2734                     old_file_fh = utils.open_file(old_file)
2735                     actual_md5 = apt_pkg.md5sum(old_file_fh)
2736                     old_file_fh.close()
2737                     actual_size = os.stat(old_file)[stat.ST_SIZE]
2738                     found = old_file
2739                     suite_type = x.location.archive_type
2740                     # need this for updating dsc_files in install()
2741                     dsc_entry["files id"] = x.file_id
2742                     # See install() in process-accepted...
2743                     if not orig_files.has_key(dsc_name):
2744                         orig_files[dsc_name] = {}
2745                     orig_files[dsc_name]["id"] = x.file_id
2746                     orig_files[dsc_name]["path"] = old_file
2747                     orig_files[dsc_name]["location"] = x.location.location_id
2748                 else:
2749                     # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2750                     # Not there? Check the queue directories...
2751                     for directory in [ "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2752                         if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2753                             continue
2754                         in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2755                         if os.path.exists(in_otherdir):
2756                             in_otherdir_fh = utils.open_file(in_otherdir)
2757                             actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2758                             in_otherdir_fh.close()
2759                             actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2760                             found = in_otherdir
2761                             if not orig_files.has_key(dsc_name):
2762                                 orig_files[dsc_name] = {}
2763                             orig_files[dsc_name]["path"] = in_otherdir
2764
2765                     if not found:
2766                         self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (filename, dsc_name))
2767                         continue
2768             else:
2769                 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (filename, dsc_name))
2770                 continue
2771             if actual_md5 != dsc_entry["md5sum"]:
2772                 self.rejects.append("md5sum for %s doesn't match %s." % (found, filename))
2773             if actual_size != int(dsc_entry["size"]):
2774                 self.rejects.append("size for %s doesn't match %s." % (found, filename))
2775
2776     ################################################################################
2777     # This is used by process-new and process-holding to recheck a changes file
2778     # at the time we're running.  It mainly wraps various other internal functions
2779     # and is similar to accepted_checks - these should probably be tidied up
2780     # and combined
2781     def recheck(self, session):
2782         cnf = Config()
2783         for f in self.pkg.files.keys():
2784             # The .orig.tar.gz can disappear out from under us is it's a
2785             # duplicate of one in the archive.
2786             if not self.pkg.files.has_key(f):
2787                 continue
2788
2789             entry = self.pkg.files[f]
2790
2791             # Check that the source still exists
2792             if entry["type"] == "deb":
2793                 source_version = entry["source version"]
2794                 source_package = entry["source package"]
2795                 if not self.pkg.changes["architecture"].has_key("source") \
2796                    and not source_exists(source_package, source_version, \
2797                     suites = self.pkg.changes["distribution"].keys(), session = session):
2798                     source_epochless_version = re_no_epoch.sub('', source_version)
2799                     dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
2800                     found = False
2801                     for q in ["Embargoed", "Unembargoed", "Newstage"]:
2802                         if cnf.has_key("Dir::Queue::%s" % (q)):
2803                             if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
2804                                 found = True
2805                     if not found:
2806                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
2807
2808             # Version and file overwrite checks
2809             if entry["type"] == "deb":
2810                 self.check_binary_against_db(f, session)
2811             elif entry["type"] == "dsc":
2812                 self.check_source_against_db(f, session)
2813                 self.check_dsc_against_db(f, session)
2814
2815     ################################################################################
2816     def accepted_checks(self, overwrite_checks, session):
2817         # Recheck anything that relies on the database; since that's not
2818         # frozen between accept and our run time when called from p-a.
2819
2820         # overwrite_checks is set to False when installing to stable/oldstable
2821
2822         propogate={}
2823         nopropogate={}
2824
2825         # Find the .dsc (again)
2826         dsc_filename = None
2827         for f in self.pkg.files.keys():
2828             if self.pkg.files[f]["type"] == "dsc":
2829                 dsc_filename = f
2830
2831         for checkfile in self.pkg.files.keys():
2832             # The .orig.tar.gz can disappear out from under us is it's a
2833             # duplicate of one in the archive.
2834             if not self.pkg.files.has_key(checkfile):
2835                 continue
2836
2837             entry = self.pkg.files[checkfile]
2838
2839             # Check that the source still exists
2840             if entry["type"] == "deb":
2841                 source_version = entry["source version"]
2842                 source_package = entry["source package"]
2843                 if not self.pkg.changes["architecture"].has_key("source") \
2844                    and not source_exists(source_package, source_version, \
2845                     suites = self.pkg.changes["distribution"].keys(), \
2846                     session = session):
2847                     self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2848
2849             # Version and file overwrite checks
2850             if overwrite_checks:
2851                 if entry["type"] == "deb":
2852                     self.check_binary_against_db(checkfile, session)
2853                 elif entry["type"] == "dsc":
2854                     self.check_source_against_db(checkfile, session)
2855                     self.check_dsc_against_db(dsc_filename, session)
2856
2857             # propogate in the case it is in the override tables:
2858             for suite in self.pkg.changes.get("propdistribution", {}).keys():
2859                 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2860                     propogate[suite] = 1
2861                 else:
2862                     nopropogate[suite] = 1
2863
2864         for suite in propogate.keys():
2865             if suite in nopropogate:
2866                 continue
2867             self.pkg.changes["distribution"][suite] = 1
2868
2869         for checkfile in self.pkg.files.keys():
2870             # Check the package is still in the override tables
2871             for suite in self.pkg.changes["distribution"].keys():
2872                 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2873                     self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2874
2875     ################################################################################
2876     # If any file of an upload has a recent mtime then chances are good
2877     # the file is still being uploaded.
2878
2879     def upload_too_new(self):
2880         cnf = Config()
2881         too_new = False
2882         # Move back to the original directory to get accurate time stamps
2883         cwd = os.getcwd()
2884         os.chdir(self.pkg.directory)
2885         file_list = self.pkg.files.keys()
2886         file_list.extend(self.pkg.dsc_files.keys())
2887         file_list.append(self.pkg.changes_file)
2888         for f in file_list:
2889             try:
2890                 last_modified = time.time()-os.path.getmtime(f)
2891                 if last_modified < int(cnf["Dinstall::SkipTime"]):
2892                     too_new = True
2893                     break
2894             except:
2895                 pass
2896
2897         os.chdir(cwd)
2898         return too_new
2899
2900     def store_changelog(self):
2901
2902         # Skip binary-only upload if it is not a bin-NMU
2903         if not self.pkg.changes['architecture'].has_key('source'):
2904             from daklib.regexes import re_bin_only_nmu
2905             if not re_bin_only_nmu.search(self.pkg.changes['version']):
2906                 return
2907
2908         session = DBConn().session()
2909
2910         # Check if upload already has a changelog entry
2911         query = """SELECT changelog_id FROM changes WHERE source = :source
2912                    AND version = :version AND architecture = :architecture AND changelog_id != 0"""
2913         if session.execute(query, {'source': self.pkg.changes['source'], \
2914                                    'version': self.pkg.changes['version'], \
2915                                    'architecture': " ".join(self.pkg.changes['architecture'].keys())}).rowcount:
2916             session.commit()
2917             return
2918
2919         # Add current changelog text into changelogs_text table, return created ID
2920         query = "INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id"
2921         ID = session.execute(query, {'changelog': self.pkg.changes['changes']}).fetchone()[0]
2922
2923         # Link ID to the upload available in changes table
2924         query = """UPDATE changes SET changelog_id = :id WHERE source = :source
2925                    AND version = :version AND architecture = :architecture"""
2926         session.execute(query, {'id': ID, 'source': self.pkg.changes['source'], \
2927                                 'version': self.pkg.changes['version'], \
2928                                 'architecture': " ".join(self.pkg.changes['architecture'].keys())})
2929
2930         session.commit()