5 Queue utility functions for dak
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ###############################################################################
41 from sqlalchemy.sql.expression import desc
42 from sqlalchemy.orm.exc import NoResultFound
46 from dak_exceptions import *
49 from config import Config
50 from holding import Holding
51 from urgencylog import UrgencyLog
53 from summarystats import SummaryStats
54 from utils import parse_changes, check_dsc_files
55 from textutils import fix_maintainer
56 from binary import Binary
57 from lintian import parse_lintian_output, generate_reject_messages
59 ###############################################################################
61 def get_type(f, session):
63 Get the file type of C{f}
66 @param f: file entry from Changes object
68 @type session: SQLA Session
69 @param session: SQL Alchemy session object
76 if f.has_key("dbtype"):
77 file_type = f["dbtype"]
78 elif re_source_ext.match(f["type"]):
82 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
84 # Validate the override type
85 type_id = get_override_type(file_type, session)
87 utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
91 ################################################################################
93 # Determine what parts in a .changes are NEW
95 def determine_new(changes, files, warn=1, session = None):
97 Determine what parts in a C{changes} file are NEW.
99 @type changes: Upload.Pkg.changes dict
100 @param changes: Changes dictionary
102 @type files: Upload.Pkg.files dict
103 @param files: Files dictionary
106 @param warn: Warn if overrides are added for (old)stable
109 @return: dictionary of NEW components.
114 # Build up a list of potentially new things
115 for name, f in files.items():
116 # Skip byhand elements
117 # if f["type"] == "byhand":
120 priority = f["priority"]
121 section = f["section"]
122 file_type = get_type(f, session)
123 component = f["component"]
125 if file_type == "dsc":
128 if not new.has_key(pkg):
130 new[pkg]["priority"] = priority
131 new[pkg]["section"] = section
132 new[pkg]["type"] = file_type
133 new[pkg]["component"] = component
134 new[pkg]["files"] = []
136 old_type = new[pkg]["type"]
137 if old_type != file_type:
138 # source gets trumped by deb or udeb
139 if old_type == "dsc":
140 new[pkg]["priority"] = priority
141 new[pkg]["section"] = section
142 new[pkg]["type"] = file_type
143 new[pkg]["component"] = component
145 new[pkg]["files"].append(name)
147 if f.has_key("othercomponents"):
148 new[pkg]["othercomponents"] = f["othercomponents"]
150 # Fix up the list of target suites
152 for suite in changes["suite"].keys():
153 override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
155 (olderr, newerr) = (get_suite(suite, session) == None,
156 get_suite(override, session) == None)
158 (oinv, newinv) = ("", "")
159 if olderr: oinv = "invalid "
160 if newerr: ninv = "invalid "
161 print "warning: overriding %ssuite %s to %ssuite %s" % (
162 oinv, suite, ninv, override)
163 del changes["suite"][suite]
164 changes["suite"][override] = 1
166 for suite in changes["suite"].keys():
167 for pkg in new.keys():
168 ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
170 for file_entry in new[pkg]["files"]:
171 if files[file_entry].has_key("new"):
172 del files[file_entry]["new"]
176 for s in ['stable', 'oldstable']:
177 if changes["suite"].has_key(s):
178 print "WARNING: overrides will be added for %s!" % s
179 for pkg in new.keys():
180 if new[pkg].has_key("othercomponents"):
181 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
185 ################################################################################
187 def check_valid(new, session = None):
189 Check if section and priority for NEW packages exist in database.
190 Additionally does sanity checks:
191 - debian-installer packages have to be udeb (or source)
192 - non debian-installer packages can not be udeb
193 - source priority can only be assigned to dsc file types
196 @param new: Dict of new packages with their section, priority and type.
199 for pkg in new.keys():
200 section_name = new[pkg]["section"]
201 priority_name = new[pkg]["priority"]
202 file_type = new[pkg]["type"]
204 section = get_section(section_name, session)
206 new[pkg]["section id"] = -1
208 new[pkg]["section id"] = section.section_id
210 priority = get_priority(priority_name, session)
212 new[pkg]["priority id"] = -1
214 new[pkg]["priority id"] = priority.priority_id
217 di = section_name.find("debian-installer") != -1
219 # If d-i, we must be udeb and vice-versa
220 if (di and file_type not in ("udeb", "dsc")) or \
221 (not di and file_type == "udeb"):
222 new[pkg]["section id"] = -1
224 # If dsc we need to be source and vice-versa
225 if (priority == "source" and file_type != "dsc") or \
226 (priority != "source" and file_type == "dsc"):
227 new[pkg]["priority id"] = -1
229 ###############################################################################
231 # Used by Upload.check_timestamps
232 class TarTime(object):
233 def __init__(self, future_cutoff, past_cutoff):
235 self.future_cutoff = future_cutoff
236 self.past_cutoff = past_cutoff
239 self.future_files = {}
240 self.ancient_files = {}
242 def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
243 if MTime > self.future_cutoff:
244 self.future_files[Name] = MTime
245 if MTime < self.past_cutoff:
246 self.ancient_files[Name] = MTime
248 ###############################################################################
250 class Upload(object):
252 Everything that has to do with an upload processed.
260 ###########################################################################
263 """ Reset a number of internal variables."""
265 # Initialize the substitution template map
268 self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
269 self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
270 self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
271 self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
277 self.later_check_files = []
281 def package_info(self):
283 Format various messages from this Upload to send to the maintainer.
287 ('Reject Reasons', self.rejects),
288 ('Warnings', self.warnings),
289 ('Notes', self.notes),
293 for title, messages in msgs:
295 msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))
300 ###########################################################################
301 def update_subst(self):
302 """ Set up the per-package template substitution mappings """
306 # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
307 if not self.pkg.changes.has_key("architecture") or not \
308 isinstance(self.pkg.changes["architecture"], dict):
309 self.pkg.changes["architecture"] = { "Unknown" : "" }
311 # and maintainer2047 may not exist.
312 if not self.pkg.changes.has_key("maintainer2047"):
313 self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
315 self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
316 self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
317 self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
319 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
320 if self.pkg.changes["architecture"].has_key("source") and \
321 self.pkg.changes["changedby822"] != "" and \
322 (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
324 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
325 self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
326 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
328 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
329 self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
330 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
332 # Process policy doesn't set the fingerprint field and I don't want to make it
333 # do it for now as I don't want to have to deal with the case where we accepted
334 # the package into PU-NEW, but the fingerprint has gone away from the keyring in
335 # the meantime so the package will be remarked as rejectable. Urgh.
336 # TODO: Fix this properly
337 if self.pkg.changes.has_key('fingerprint'):
338 session = DBConn().session()
339 fpr = get_fingerprint(self.pkg.changes['fingerprint'], session)
340 if self.check_if_upload_is_sponsored("%s@debian.org" % fpr.uid.uid, fpr.uid.name):
341 self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
344 if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
345 self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
347 # Apply any global override of the Maintainer field
348 if cnf.get("Dinstall::OverrideMaintainer"):
349 self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
350 self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
352 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
353 self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
354 self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
356 ###########################################################################
357 def load_changes(self, filename):
359 Load a changes file and setup a dictionary around it. Also checks for mandantory
362 @type filename: string
363 @param filename: Changes filename, full path.
366 @return: whether the changes file was valid or not. We may want to
367 reject even if this is True (see what gets put in self.rejects).
368 This is simply to prevent us even trying things later which will
369 fail because we couldn't properly parse the file.
372 self.pkg.changes_file = filename
374 # Parse the .changes field into a dictionary
376 self.pkg.changes.update(parse_changes(filename))
377 except CantOpenError:
378 self.rejects.append("%s: can't read file." % (filename))
380 except ParseChangesError, line:
381 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
383 except ChangesUnicodeError:
384 self.rejects.append("%s: changes file not proper utf-8" % (filename))
387 # Parse the Files field from the .changes into another dictionary
389 self.pkg.files.update(utils.build_file_list(self.pkg.changes))
390 except ParseChangesError, line:
391 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
393 except UnknownFormatError, format:
394 self.rejects.append("%s: unknown format '%s'." % (filename, format))
397 # Check for mandatory fields
398 for i in ("distribution", "source", "binary", "architecture",
399 "version", "maintainer", "files", "changes", "description"):
400 if not self.pkg.changes.has_key(i):
401 # Avoid undefined errors later
402 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
405 # Strip a source version in brackets from the source field
406 if re_strip_srcver.search(self.pkg.changes["source"]):
407 self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
409 # Ensure the source field is a valid package name.
410 if not re_valid_pkg_name.match(self.pkg.changes["source"]):
411 self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
413 # Split multi-value fields into a lower-level dictionary
414 for i in ("architecture", "distribution", "binary", "closes"):
415 o = self.pkg.changes.get(i, "")
417 del self.pkg.changes[i]
419 self.pkg.changes[i] = {}
422 self.pkg.changes[i][j] = 1
424 # Fix the Maintainer: field to be RFC822/2047 compatible
426 (self.pkg.changes["maintainer822"],
427 self.pkg.changes["maintainer2047"],
428 self.pkg.changes["maintainername"],
429 self.pkg.changes["maintaineremail"]) = \
430 fix_maintainer (self.pkg.changes["maintainer"])
431 except ParseMaintError, msg:
432 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
433 % (filename, self.pkg.changes["maintainer"], msg))
435 # ...likewise for the Changed-By: field if it exists.
437 (self.pkg.changes["changedby822"],
438 self.pkg.changes["changedby2047"],
439 self.pkg.changes["changedbyname"],
440 self.pkg.changes["changedbyemail"]) = \
441 fix_maintainer (self.pkg.changes.get("changed-by", ""))
442 except ParseMaintError, msg:
443 self.pkg.changes["changedby822"] = ""
444 self.pkg.changes["changedby2047"] = ""
445 self.pkg.changes["changedbyname"] = ""
446 self.pkg.changes["changedbyemail"] = ""
448 self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
449 % (filename, self.pkg.changes["changed-by"], msg))
451 # Ensure all the values in Closes: are numbers
452 if self.pkg.changes.has_key("closes"):
453 for i in self.pkg.changes["closes"].keys():
454 if re_isanum.match (i) == None:
455 self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
457 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
458 self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
459 self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
461 # Check the .changes is non-empty
462 if not self.pkg.files:
463 self.rejects.append("%s: nothing to do (Files field is empty)." % (os.path.basename(self.pkg.changes_file)))
466 # Changes was syntactically valid even if we'll reject
469 ###########################################################################
471 def check_distributions(self):
472 "Check and map the Distribution field"
476 # Handle suite mappings
477 for m in Cnf.ValueList("SuiteMappings"):
480 if mtype == "map" or mtype == "silent-map":
481 (source, dest) = args[1:3]
482 if self.pkg.changes["distribution"].has_key(source):
483 del self.pkg.changes["distribution"][source]
484 self.pkg.changes["distribution"][dest] = 1
485 if mtype != "silent-map":
486 self.notes.append("Mapping %s to %s." % (source, dest))
487 if self.pkg.changes.has_key("distribution-version"):
488 if self.pkg.changes["distribution-version"].has_key(source):
489 self.pkg.changes["distribution-version"][source]=dest
490 elif mtype == "map-unreleased":
491 (source, dest) = args[1:3]
492 if self.pkg.changes["distribution"].has_key(source):
493 for arch in self.pkg.changes["architecture"].keys():
494 if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
495 self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
496 del self.pkg.changes["distribution"][source]
497 self.pkg.changes["distribution"][dest] = 1
499 elif mtype == "ignore":
501 if self.pkg.changes["distribution"].has_key(suite):
502 del self.pkg.changes["distribution"][suite]
503 self.warnings.append("Ignoring %s as a target suite." % (suite))
504 elif mtype == "reject":
506 if self.pkg.changes["distribution"].has_key(suite):
507 self.rejects.append("Uploads to %s are not accepted." % (suite))
508 elif mtype == "propup-version":
509 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
511 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
512 if self.pkg.changes["distribution"].has_key(args[1]):
513 self.pkg.changes.setdefault("distribution-version", {})
514 for suite in args[2:]:
515 self.pkg.changes["distribution-version"][suite] = suite
517 # Ensure there is (still) a target distribution
518 if len(self.pkg.changes["distribution"].keys()) < 1:
519 self.rejects.append("No valid distribution remaining.")
521 # Ensure target distributions exist
522 for suite in self.pkg.changes["distribution"].keys():
523 if not Cnf.has_key("Suite::%s" % (suite)):
524 self.rejects.append("Unknown distribution `%s'." % (suite))
526 ###########################################################################
528 def binary_file_checks(self, f, session):
530 entry = self.pkg.files[f]
532 # Extract package control information
533 deb_file = utils.open_file(f)
535 control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
537 self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
539 # Can't continue, none of the checks on control would work.
542 # Check for mandantory "Description:"
545 apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
547 self.rejects.append("%s: Missing Description in binary package" % (f))
552 # Check for mandatory fields
553 for field in [ "Package", "Architecture", "Version" ]:
554 if control.Find(field) == None:
556 self.rejects.append("%s: No %s field in control." % (f, field))
559 # Ensure the package name matches the one give in the .changes
560 if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
561 self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
563 # Validate the package field
564 package = control.Find("Package")
565 if not re_valid_pkg_name.match(package):
566 self.rejects.append("%s: invalid package name '%s'." % (f, package))
568 # Validate the version field
569 version = control.Find("Version")
570 if not re_valid_version.match(version):
571 self.rejects.append("%s: invalid version number '%s'." % (f, version))
573 # Ensure the architecture of the .deb is one we know about.
574 default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
575 architecture = control.Find("Architecture")
576 upload_suite = self.pkg.changes["distribution"].keys()[0]
578 if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session = session)] \
579 and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session = session)]:
580 self.rejects.append("Unknown architecture '%s'." % (architecture))
582 # Ensure the architecture of the .deb is one of the ones
583 # listed in the .changes.
584 if not self.pkg.changes["architecture"].has_key(architecture):
585 self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
587 # Sanity-check the Depends field
588 depends = control.Find("Depends")
590 self.rejects.append("%s: Depends field is empty." % (f))
592 # Sanity-check the Provides field
593 provides = control.Find("Provides")
595 provide = re_spacestrip.sub('', provides)
597 self.rejects.append("%s: Provides field is empty." % (f))
598 prov_list = provide.split(",")
599 for prov in prov_list:
600 if not re_valid_pkg_name.match(prov):
601 self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
603 # Check the section & priority match those given in the .changes (non-fatal)
604 if control.Find("Section") and entry["section"] != "" \
605 and entry["section"] != control.Find("Section"):
606 self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
607 (f, control.Find("Section", ""), entry["section"]))
608 if control.Find("Priority") and entry["priority"] != "" \
609 and entry["priority"] != control.Find("Priority"):
610 self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
611 (f, control.Find("Priority", ""), entry["priority"]))
613 entry["package"] = package
614 entry["architecture"] = architecture
615 entry["version"] = version
616 entry["maintainer"] = control.Find("Maintainer", "")
618 if f.endswith(".udeb"):
619 self.pkg.files[f]["dbtype"] = "udeb"
620 elif f.endswith(".deb"):
621 self.pkg.files[f]["dbtype"] = "deb"
623 self.rejects.append("%s is neither a .deb or a .udeb." % (f))
625 entry["source"] = control.Find("Source", entry["package"])
627 # Get the source version
628 source = entry["source"]
631 if source.find("(") != -1:
632 m = re_extract_src_version.match(source)
634 source_version = m.group(2)
636 if not source_version:
637 source_version = self.pkg.files[f]["version"]
639 entry["source package"] = source
640 entry["source version"] = source_version
642 # Ensure the filename matches the contents of the .deb
643 m = re_isadeb.match(f)
646 file_package = m.group(1)
647 if entry["package"] != file_package:
648 self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
649 (f, file_package, entry["dbtype"], entry["package"]))
650 epochless_version = re_no_epoch.sub('', control.Find("Version"))
653 file_version = m.group(2)
654 if epochless_version != file_version:
655 self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
656 (f, file_version, entry["dbtype"], epochless_version))
659 file_architecture = m.group(3)
660 if entry["architecture"] != file_architecture:
661 self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
662 (f, file_architecture, entry["dbtype"], entry["architecture"]))
664 # Check for existent source
665 source_version = entry["source version"]
666 source_package = entry["source package"]
667 if self.pkg.changes["architecture"].has_key("source"):
668 if source_version != self.pkg.changes["version"]:
669 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
670 (source_version, f, self.pkg.changes["version"]))
672 # Check in the SQL database
673 if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
674 # Check in one of the other directories
675 source_epochless_version = re_no_epoch.sub('', source_version)
676 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
677 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
679 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
682 dsc_file_exists = False
683 for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates", "Lenny-Volatile-Proposed-Updates"]:
684 if cnf.has_key("Dir::Queue::%s" % (myq)):
685 if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
686 dsc_file_exists = True
689 if not dsc_file_exists:
690 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
692 # Check the version and for file overwrites
693 self.check_binary_against_db(f, session)
695 # Temporarily disable contents generation until we change the table storage layout
698 #if len(b.rejects) > 0:
699 # for j in b.rejects:
700 # self.rejects.append(j)
702 def source_file_checks(self, f, session):
703 entry = self.pkg.files[f]
705 m = re_issource.match(f)
709 entry["package"] = m.group(1)
710 entry["version"] = m.group(2)
711 entry["type"] = m.group(3)
713 # Ensure the source package name matches the Source filed in the .changes
714 if self.pkg.changes["source"] != entry["package"]:
715 self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
717 # Ensure the source version matches the version in the .changes file
718 if re_is_orig_source.match(f):
719 changes_version = self.pkg.changes["chopversion2"]
721 changes_version = self.pkg.changes["chopversion"]
723 if changes_version != entry["version"]:
724 self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
726 # Ensure the .changes lists source in the Architecture field
727 if not self.pkg.changes["architecture"].has_key("source"):
728 self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
730 # Check the signature of a .dsc file
731 if entry["type"] == "dsc":
732 # check_signature returns either:
733 # (None, [list, of, rejects]) or (signature, [])
734 (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
736 self.rejects.append(j)
738 entry["architecture"] = "source"
740 def per_suite_file_checks(self, f, suite, session):
742 entry = self.pkg.files[f]
745 if entry.has_key("byhand"):
748 # Check we have fields we need to do these checks
750 for m in ['component', 'package', 'priority', 'size', 'md5sum']:
751 if not entry.has_key(m):
752 self.rejects.append("file '%s' does not have field %s set" % (f, m))
758 # Handle component mappings
759 for m in cnf.ValueList("ComponentMappings"):
760 (source, dest) = m.split()
761 if entry["component"] == source:
762 entry["original component"] = source
763 entry["component"] = dest
765 # Ensure the component is valid for the target suite
766 if cnf.has_key("Suite:%s::Components" % (suite)) and \
767 entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
768 self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
771 # Validate the component
772 if not get_component(entry["component"], session):
773 self.rejects.append("file '%s' has unknown component '%s'." % (f, entry["component"]))
776 # See if the package is NEW
777 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
780 # Validate the priority
781 if entry["priority"].find('/') != -1:
782 self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
784 # Determine the location
785 location = cnf["Dir::Pool"]
786 l = get_location(location, entry["component"], session=session)
788 self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s)" % entry["component"])
789 entry["location id"] = -1
791 entry["location id"] = l.location_id
793 # Check the md5sum & size against existing files (if any)
794 entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
796 found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
797 entry["size"], entry["md5sum"], entry["location id"])
800 self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
801 elif found is False and poolfile is not None:
802 self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
805 entry["files id"] = None
807 entry["files id"] = poolfile.file_id
809 # Check for packages that have moved from one component to another
810 entry['suite'] = suite
811 res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
813 entry["othercomponents"] = res.fetchone()[0]
815 def check_files(self, action=True):
816 file_keys = self.pkg.files.keys()
822 os.chdir(self.pkg.directory)
824 ret = holding.copy_to_holding(f)
826 self.warnings.append('Could not copy %s to holding; will attempt to find in DB later' % f)
830 # check we already know the changes file
831 # [NB: this check must be done post-suite mapping]
832 base_filename = os.path.basename(self.pkg.changes_file)
834 session = DBConn().session()
837 dbc = session.query(DBChange).filter_by(changesname=base_filename).one()
838 # if in the pool or in a queue other than unchecked, reject
839 if (dbc.in_queue is None) \
840 or (dbc.in_queue is not None
841 and dbc.in_queue.queue_name not in ["unchecked", "newstage"]):
842 self.rejects.append("%s file already known to dak" % base_filename)
843 except NoResultFound, e:
850 for f, entry in self.pkg.files.items():
851 # Ensure the file does not already exist in one of the accepted directories
852 for d in [ "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
853 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
854 if os.path.exists(os.path.join(cnf["Dir::Queue::%s" % (d) ], f)):
855 self.rejects.append("%s file already exists in the %s directory." % (f, d))
857 if not re_taint_free.match(f):
858 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
860 # Check the file is readable
861 if os.access(f, os.R_OK) == 0:
862 # When running in -n, copy_to_holding() won't have
863 # generated the reject_message, so we need to.
865 if os.path.exists(f):
866 self.rejects.append("Can't read `%s'. [permission denied]" % (f))
868 # Don't directly reject, mark to check later to deal with orig's
869 # we can find in the pool
870 self.later_check_files.append(f)
871 entry["type"] = "unreadable"
874 # If it's byhand skip remaining checks
875 if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
877 entry["type"] = "byhand"
879 # Checks for a binary package...
880 elif re_isadeb.match(f):
882 entry["type"] = "deb"
884 # This routine appends to self.rejects/warnings as appropriate
885 self.binary_file_checks(f, session)
887 # Checks for a source package...
888 elif re_issource.match(f):
891 # This routine appends to self.rejects/warnings as appropriate
892 self.source_file_checks(f, session)
894 # Not a binary or source package? Assume byhand...
897 entry["type"] = "byhand"
899 # Per-suite file checks
900 entry["oldfiles"] = {}
901 for suite in self.pkg.changes["distribution"].keys():
902 self.per_suite_file_checks(f, suite, session)
906 # If the .changes file says it has source, it must have source.
907 if self.pkg.changes["architecture"].has_key("source"):
909 self.rejects.append("no source found and Architecture line in changes mention source.")
911 if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
912 self.rejects.append("source only uploads are not supported.")
914 ###########################################################################
915 def check_dsc(self, action=True, session=None):
916 """Returns bool indicating whether or not the source changes are valid"""
917 # Ensure there is source to check
918 if not self.pkg.changes["architecture"].has_key("source"):
923 for f, entry in self.pkg.files.items():
924 if entry["type"] == "dsc":
926 self.rejects.append("can not process a .changes file with multiple .dsc's.")
931 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
933 self.rejects.append("source uploads must contain a dsc file")
936 # Parse the .dsc file
938 self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1, dsc_file=1))
939 except CantOpenError:
940 # if not -n copy_to_holding() will have done this for us...
942 self.rejects.append("%s: can't read file." % (dsc_filename))
943 except ParseChangesError, line:
944 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
945 except InvalidDscError, line:
946 self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
947 except ChangesUnicodeError:
948 self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
950 # Build up the file list of files mentioned by the .dsc
952 self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
953 except NoFilesFieldError:
954 self.rejects.append("%s: no Files: field." % (dsc_filename))
956 except UnknownFormatError, format:
957 self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
959 except ParseChangesError, line:
960 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
963 # Enforce mandatory fields
964 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
965 if not self.pkg.dsc.has_key(i):
966 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
969 # Validate the source and version fields
970 if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
971 self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
972 if not re_valid_version.match(self.pkg.dsc["version"]):
973 self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
975 # Only a limited list of source formats are allowed in each suite
976 for dist in self.pkg.changes["distribution"].keys():
977 allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
978 if self.pkg.dsc["format"] not in allowed:
979 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
981 # Validate the Maintainer field
983 # We ignore the return value
984 fix_maintainer(self.pkg.dsc["maintainer"])
985 except ParseMaintError, msg:
986 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
987 % (dsc_filename, self.pkg.dsc["maintainer"], msg))
989 # Validate the build-depends field(s)
990 for field_name in [ "build-depends", "build-depends-indep" ]:
991 field = self.pkg.dsc.get(field_name)
993 # Have apt try to parse them...
995 apt_pkg.ParseSrcDepends(field)
997 self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
999 # Ensure the version number in the .dsc matches the version number in the .changes
1000 epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1001 changes_version = self.pkg.files[dsc_filename]["version"]
1003 if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1004 self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1006 # Ensure the Files field contain only what's expected
1007 self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
1009 # Ensure source is newer than existing source in target suites
1010 session = DBConn().session()
1011 self.check_source_against_db(dsc_filename, session)
1012 self.check_dsc_against_db(dsc_filename, session)
1015 # Finally, check if we're missing any files
1016 for f in self.later_check_files:
1017 self.rejects.append("Could not find file %s references in changes" % f)
1021 ###########################################################################
1023 def get_changelog_versions(self, source_dir):
1024 """Extracts a the source package and (optionally) grabs the
1025 version history out of debian/changelog for the BTS."""
1029 # Find the .dsc (again)
1031 for f in self.pkg.files.keys():
1032 if self.pkg.files[f]["type"] == "dsc":
1035 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1036 if not dsc_filename:
1039 # Create a symlink mirror of the source files in our temporary directory
1040 for f in self.pkg.files.keys():
1041 m = re_issource.match(f)
1043 src = os.path.join(source_dir, f)
1044 # If a file is missing for whatever reason, give up.
1045 if not os.path.exists(src):
1048 if re_is_orig_source.match(f) and self.pkg.orig_files.has_key(f) and \
1049 self.pkg.orig_files[f].has_key("path"):
1051 dest = os.path.join(os.getcwd(), f)
1052 os.symlink(src, dest)
1054 # If the orig files are not a part of the upload, create symlinks to the
1056 for orig_file in self.pkg.orig_files.keys():
1057 if not self.pkg.orig_files[orig_file].has_key("path"):
1059 dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
1060 os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
1062 # Extract the source
1063 cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1064 (result, output) = commands.getstatusoutput(cmd)
1066 self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1067 self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "))
1070 if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1073 # Get the upstream version
1074 upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1075 if re_strip_revision.search(upstr_version):
1076 upstr_version = re_strip_revision.sub('', upstr_version)
1078 # Ensure the changelog file exists
1079 changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1080 if not os.path.exists(changelog_filename):
1081 self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1084 # Parse the changelog
1085 self.pkg.dsc["bts changelog"] = ""
1086 changelog_file = utils.open_file(changelog_filename)
1087 for line in changelog_file.readlines():
1088 m = re_changelog_versions.match(line)
1090 self.pkg.dsc["bts changelog"] += line
1091 changelog_file.close()
1093 # Check we found at least one revision in the changelog
1094 if not self.pkg.dsc["bts changelog"]:
1095 self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1097 def check_source(self):
1099 # a) there's no source
1100 if not self.pkg.changes["architecture"].has_key("source"):
1103 tmpdir = utils.temp_dirname()
1105 # Move into the temporary directory
1109 # Get the changelog version history
1110 self.get_changelog_versions(cwd)
1112 # Move back and cleanup the temporary tree
1116 shutil.rmtree(tmpdir)
1118 if e.errno != errno.EACCES:
1120 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1122 self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1123 # We probably have u-r or u-w directories so chmod everything
1125 cmd = "chmod -R u+rwx %s" % (tmpdir)
1126 result = os.system(cmd)
1128 utils.fubar("'%s' failed with result %s." % (cmd, result))
1129 shutil.rmtree(tmpdir)
1130 except Exception, e:
1131 print "foobar2 (%s)" % e
1132 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1134 ###########################################################################
1135 def ensure_hashes(self):
1136 # Make sure we recognise the format of the Files: field in the .changes
1137 format = self.pkg.changes.get("format", "0.0").split(".", 1)
1138 if len(format) == 2:
1139 format = int(format[0]), int(format[1])
1141 format = int(float(format[0])), 0
1143 # We need to deal with the original changes blob, as the fields we need
1144 # might not be in the changes dict serialised into the .dak anymore.
1145 orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1147 # Copy the checksums over to the current changes dict. This will keep
1148 # the existing modifications to it intact.
1149 for field in orig_changes:
1150 if field.startswith('checksums-'):
1151 self.pkg.changes[field] = orig_changes[field]
1153 # Check for unsupported hashes
1154 for j in utils.check_hash_fields(".changes", self.pkg.changes):
1155 self.rejects.append(j)
1157 for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1158 self.rejects.append(j)
1160 # We have to calculate the hash if we have an earlier changes version than
1161 # the hash appears in rather than require it exist in the changes file
1162 for hashname, hashfunc, version in utils.known_hashes:
1163 # TODO: Move _ensure_changes_hash into this class
1164 for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1165 self.rejects.append(j)
1166 if "source" in self.pkg.changes["architecture"]:
1167 # TODO: Move _ensure_dsc_hash into this class
1168 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1169 self.rejects.append(j)
1171 def check_hashes(self):
1172 for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1173 self.rejects.append(m)
1175 for m in utils.check_size(".changes", self.pkg.files):
1176 self.rejects.append(m)
1178 for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1179 self.rejects.append(m)
1181 for m in utils.check_size(".dsc", self.pkg.dsc_files):
1182 self.rejects.append(m)
1184 self.ensure_hashes()
1186 ###########################################################################
1188 def ensure_orig(self, target_dir='.', session=None):
1190 Ensures that all orig files mentioned in the changes file are present
1191 in target_dir. If they do not exist, they are symlinked into place.
1193 An list containing the symlinks that were created are returned (so they
1200 for filename, entry in self.pkg.dsc_files.iteritems():
1201 if not re_is_orig_source.match(filename):
1202 # File is not an orig; ignore
1205 if os.path.exists(filename):
1206 # File exists, no need to continue
1209 def symlink_if_valid(path):
1210 f = utils.open_file(path)
1211 md5sum = apt_pkg.md5sum(f)
1214 fingerprint = (os.stat(path)[stat.ST_SIZE], md5sum)
1215 expected = (int(entry['size']), entry['md5sum'])
1217 if fingerprint != expected:
1220 dest = os.path.join(target_dir, filename)
1222 os.symlink(path, dest)
1223 symlinked.append(dest)
1229 session_ = DBConn().session()
1234 for poolfile in get_poolfile_like_name('%s' % filename, session_):
1235 poolfile_path = os.path.join(
1236 poolfile.location.path, poolfile.filename
1239 if symlink_if_valid(poolfile_path):
1249 # Look in some other queues for the file
1250 queues = ('New', 'Byhand', 'ProposedUpdates',
1251 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
1253 for queue in queues:
1254 if not cnf.get('Dir::Queue::%s' % queue):
1257 queuefile_path = os.path.join(
1258 cnf['Dir::Queue::%s' % queue], filename
1261 if not os.path.exists(queuefile_path):
1262 # Does not exist in this queue
1265 if symlink_if_valid(queuefile_path):
1270 ###########################################################################
1272 def check_lintian(self):
1274 Extends self.rejects by checking the output of lintian against tags
1275 specified in Dinstall::LintianTags.
1280 # Don't reject binary uploads
1281 if not self.pkg.changes['architecture'].has_key('source'):
1284 # Only check some distributions
1285 for dist in ('unstable', 'experimental'):
1286 if dist in self.pkg.changes['distribution']:
1291 # If we do not have a tagfile, don't do anything
1292 tagfile = cnf.get("Dinstall::LintianTags")
1296 # Parse the yaml file
1297 sourcefile = file(tagfile, 'r')
1298 sourcecontent = sourcefile.read()
1302 lintiantags = yaml.load(sourcecontent)['lintian']
1303 except yaml.YAMLError, msg:
1304 utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
1307 # Try and find all orig mentioned in the .dsc
1308 symlinked = self.ensure_orig()
1310 # Setup the input file for lintian
1311 fd, temp_filename = utils.temp_filename()
1312 temptagfile = os.fdopen(fd, 'w')
1313 for tags in lintiantags.values():
1314 temptagfile.writelines(['%s\n' % x for x in tags])
1318 cmd = "lintian --show-overrides --tags-from-file %s %s" % \
1319 (temp_filename, self.pkg.changes_file)
1321 result, output = commands.getstatusoutput(cmd)
1323 # Remove our tempfile and any symlinks we created
1324 os.unlink(temp_filename)
1326 for symlink in symlinked:
1330 utils.warn("lintian failed for %s [return code: %s]." % \
1331 (self.pkg.changes_file, result))
1332 utils.warn(utils.prefix_multi_line_string(output, \
1333 " [possible output:] "))
1338 [self.pkg.changes_file, "check_lintian"] + list(txt)
1342 parsed_tags = parse_lintian_output(output)
1343 self.rejects.extend(
1344 generate_reject_messages(parsed_tags, lintiantags, log=log)
1347 ###########################################################################
1348 def check_urgency(self):
1350 if self.pkg.changes["architecture"].has_key("source"):
1351 if not self.pkg.changes.has_key("urgency"):
1352 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1353 self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1354 if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1355 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1356 (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1357 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1359 ###########################################################################
1361 # Sanity check the time stamps of files inside debs.
1362 # [Files in the near future cause ugly warnings and extreme time
1363 # travel can cause errors on extraction]
1365 def check_timestamps(self):
1368 future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1369 past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1370 tar = TarTime(future_cutoff, past_cutoff)
1372 for filename, entry in self.pkg.files.items():
1373 if entry["type"] == "deb":
1376 deb_file = utils.open_file(filename)
1377 apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1380 apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1381 except SystemError, e:
1382 # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1383 if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1386 apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1390 future_files = tar.future_files.keys()
1392 num_future_files = len(future_files)
1393 future_file = future_files[0]
1394 future_date = tar.future_files[future_file]
1395 self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1396 % (filename, num_future_files, future_file, time.ctime(future_date)))
1398 ancient_files = tar.ancient_files.keys()
1400 num_ancient_files = len(ancient_files)
1401 ancient_file = ancient_files[0]
1402 ancient_date = tar.ancient_files[ancient_file]
1403 self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1404 % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1406 self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1408 def check_if_upload_is_sponsored(self, uid_email, uid_name):
1409 if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1411 elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1417 if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
1418 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1419 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1420 self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1421 self.pkg.changes["sponsoremail"] = uid_email
1426 ###########################################################################
1427 # check_signed_by_key checks
1428 ###########################################################################
1430 def check_signed_by_key(self):
1431 """Ensure the .changes is signed by an authorized uploader."""
1432 session = DBConn().session()
1434 # First of all we check that the person has proper upload permissions
1435 # and that this upload isn't blocked
1436 fpr = get_fingerprint(self.pkg.changes['fingerprint'], session=session)
1439 self.rejects.append("Cannot find fingerprint %s" % self.pkg.changes["fingerprint"])
1442 # TODO: Check that import-keyring adds UIDs properly
1444 self.rejects.append("Cannot find uid for fingerprint %s. Please contact ftpmaster@debian.org" % fpr.fingerprint)
1447 # Check that the fingerprint which uploaded has permission to do so
1448 self.check_upload_permissions(fpr, session)
1450 # Check that this package is not in a transition
1451 self.check_transition(session)
1456 def check_upload_permissions(self, fpr, session):
1457 # Check any one-off upload blocks
1458 self.check_upload_blocks(fpr, session)
1460 # Start with DM as a special case
1461 # DM is a special case unfortunately, so we check it first
1462 # (keys with no source access get more access than DMs in one
1463 # way; DMs can only upload for their packages whether source
1464 # or binary, whereas keys with no access might be able to
1465 # upload some binaries)
1466 if fpr.source_acl.access_level == 'dm':
1467 self.check_dm_upload(fpr, session)
1469 # Check source-based permissions for other types
1470 if self.pkg.changes["architecture"].has_key("source") and \
1471 fpr.source_acl.access_level is None:
1472 rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
1473 rej += '\nPlease contact ftpmaster if you think this is incorrect'
1474 self.rejects.append(rej)
1476 # If not a DM, we allow full upload rights
1477 uid_email = "%s@debian.org" % (fpr.uid.uid)
1478 self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
1481 # Check binary upload permissions
1482 # By this point we know that DMs can't have got here unless they
1483 # are allowed to deal with the package concerned so just apply
1485 if fpr.binary_acl.access_level == 'full':
1488 # Otherwise we're in the map case
1489 tmparches = self.pkg.changes["architecture"].copy()
1490 tmparches.pop('source', None)
1492 for bam in fpr.binary_acl_map:
1493 tmparches.pop(bam.architecture.arch_string, None)
1495 if len(tmparches.keys()) > 0:
1496 if fpr.binary_reject:
1497 rej = ".changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
1498 rej += "\narchitectures involved are: ", ",".join(tmparches.keys())
1499 self.rejects.append(rej)
1501 # TODO: This is where we'll implement reject vs throw away binaries later
1502 rej = "Uhm. I'm meant to throw away the binaries now but that's not implemented yet"
1503 rej += "\nPlease complain to ftpmaster@debian.org as this shouldn't have been turned on"
1504 rej += "\nFingerprint: %s", (fpr.fingerprint)
1505 self.rejects.append(rej)
1508 def check_upload_blocks(self, fpr, session):
1509 """Check whether any upload blocks apply to this source, source
1510 version, uid / fpr combination"""
1512 def block_rej_template(fb):
1513 rej = 'Manual upload block in place for package %s' % fb.source
1514 if fb.version is not None:
1515 rej += ', version %s' % fb.version
1518 for fb in session.query(UploadBlock).filter_by(source = self.pkg.changes['source']).all():
1519 # version is None if the block applies to all versions
1520 if fb.version is None or fb.version == self.pkg.changes['version']:
1521 # Check both fpr and uid - either is enough to cause a reject
1522 if fb.fpr is not None:
1523 if fb.fpr.fingerprint == fpr.fingerprint:
1524 self.rejects.append(block_rej_template(fb) + ' for fingerprint %s\nReason: %s' % (fpr.fingerprint, fb.reason))
1525 if fb.uid is not None:
1526 if fb.uid == fpr.uid:
1527 self.rejects.append(block_rej_template(fb) + ' for uid %s\nReason: %s' % (fb.uid.uid, fb.reason))
1530 def check_dm_upload(self, fpr, session):
1531 # Quoth the GR (http://www.debian.org/vote/2007/vote_003):
1532 ## none of the uploaded packages are NEW
1534 for f in self.pkg.files.keys():
1535 if self.pkg.files[f].has_key("byhand"):
1536 self.rejects.append("%s may not upload BYHAND file %s" % (fpr.uid.uid, f))
1538 if self.pkg.files[f].has_key("new"):
1539 self.rejects.append("%s may not upload NEW file %s" % (fpr.uid.uid, f))
1545 ## the most recent version of the package uploaded to unstable or
1546 ## experimental includes the field "DM-Upload-Allowed: yes" in the source
1547 ## section of its control file
1548 q = session.query(DBSource).filter_by(source=self.pkg.changes["source"])
1549 q = q.join(SrcAssociation)
1550 q = q.join(Suite).filter(Suite.suite_name.in_(['unstable', 'experimental']))
1551 q = q.order_by(desc('source.version')).limit(1)
1556 rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"]
1557 self.rejects.append(rej)
1561 if not r.dm_upload_allowed:
1562 rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version)
1563 self.rejects.append(rej)
1566 ## the Maintainer: field of the uploaded .changes file corresponds with
1567 ## the owner of the key used (ie, non-developer maintainers may not sponsor
1569 if self.check_if_upload_is_sponsored(fpr.uid.uid, fpr.uid.name):
1570 self.rejects.append("%s (%s) is not authorised to sponsor uploads" % (fpr.uid.uid, fpr.fingerprint))
1572 ## the most recent version of the package uploaded to unstable or
1573 ## experimental lists the uploader in the Maintainer: or Uploaders: fields (ie,
1574 ## non-developer maintainers cannot NMU or hijack packages)
1576 # srcuploaders includes the maintainer
1578 for sup in r.srcuploaders:
1579 (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1580 # Eww - I hope we never have two people with the same name in Debian
1581 if email == fpr.uid.uid or name == fpr.uid.name:
1586 self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (fpr.uid.uid, self.pkg.changes["source"]))
1589 ## none of the packages are being taken over from other source packages
1590 for b in self.pkg.changes["binary"].keys():
1591 for suite in self.pkg.changes["distribution"].keys():
1592 q = session.query(DBSource)
1593 q = q.join(DBBinary).filter_by(package=b)
1594 q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1597 if s.source != self.pkg.changes["source"]:
1598 self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite))
1602 def check_transition(self, session):
1605 sourcepkg = self.pkg.changes["source"]
1607 # No sourceful upload -> no need to do anything else, direct return
1608 # We also work with unstable uploads, not experimental or those going to some
1609 # proposed-updates queue
1610 if "source" not in self.pkg.changes["architecture"] or \
1611 "unstable" not in self.pkg.changes["distribution"]:
1614 # Also only check if there is a file defined (and existant) with
1616 transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1617 if transpath == "" or not os.path.exists(transpath):
1620 # Parse the yaml file
1621 sourcefile = file(transpath, 'r')
1622 sourcecontent = sourcefile.read()
1624 transitions = yaml.load(sourcecontent)
1625 except yaml.YAMLError, msg:
1626 # This shouldn't happen, there is a wrapper to edit the file which
1627 # checks it, but we prefer to be safe than ending up rejecting
1629 utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1632 # Now look through all defined transitions
1633 for trans in transitions:
1634 t = transitions[trans]
1635 source = t["source"]
1638 # Will be None if nothing is in testing.
1639 current = get_source_in_suite(source, "testing", session)
1640 if current is not None:
1641 compare = apt_pkg.VersionCompare(current.version, expected)
1643 if current is None or compare < 0:
1644 # This is still valid, the current version in testing is older than
1645 # the new version we wait for, or there is none in testing yet
1647 # Check if the source we look at is affected by this.
1648 if sourcepkg in t['packages']:
1649 # The source is affected, lets reject it.
1651 rejectmsg = "%s: part of the %s transition.\n\n" % (
1654 if current is not None:
1655 currentlymsg = "at version %s" % (current.version)
1657 currentlymsg = "not present in testing"
1659 rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1661 rejectmsg += "\n".join(textwrap.wrap("""Your package
1662 is part of a testing transition designed to get %s migrated (it is
1663 currently %s, we need version %s). This transition is managed by the
1664 Release Team, and %s is the Release-Team member responsible for it.
1665 Please mail debian-release@lists.debian.org or contact %s directly if you
1666 need further assistance. You might want to upload to experimental until this
1667 transition is done."""
1668 % (source, currentlymsg, expected,t["rm"], t["rm"])))
1670 self.rejects.append(rejectmsg)
1673 ###########################################################################
1674 # End check_signed_by_key checks
1675 ###########################################################################
1677 def build_summaries(self):
1678 """ Build a summary of changes the upload introduces. """
1680 (byhand, new, summary, override_summary) = self.pkg.file_summary()
1682 short_summary = summary
1684 # This is for direport's benefit...
1685 f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1688 summary += "Changes: " + f
1690 summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1692 summary += self.announce(short_summary, 0)
1694 return (summary, short_summary)
1696 ###########################################################################
1698 def close_bugs(self, summary, action):
1700 Send mail to close bugs as instructed by the closes field in the changes file.
1701 Also add a line to summary if any work was done.
1703 @type summary: string
1704 @param summary: summary text, as given by L{build_summaries}
1707 @param action: Set to false no real action will be done.
1710 @return: summary. If action was taken, extended by the list of closed bugs.
1714 template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1716 bugs = self.pkg.changes["closes"].keys()
1722 summary += "Closing bugs: "
1724 summary += "%s " % (bug)
1727 self.Subst["__BUG_NUMBER__"] = bug
1728 if self.pkg.changes["distribution"].has_key("stable"):
1729 self.Subst["__STABLE_WARNING__"] = """
1730 Note that this package is not part of the released stable Debian
1731 distribution. It may have dependencies on other unreleased software,
1732 or other instabilities. Please take care if you wish to install it.
1733 The update will eventually make its way into the next released Debian
1736 self.Subst["__STABLE_WARNING__"] = ""
1737 mail_message = utils.TemplateSubst(self.Subst, template)
1738 utils.send_mail(mail_message)
1740 # Clear up after ourselves
1741 del self.Subst["__BUG_NUMBER__"]
1742 del self.Subst["__STABLE_WARNING__"]
1744 if action and self.logger:
1745 self.logger.log(["closing bugs"] + bugs)
1751 ###########################################################################
1753 def announce(self, short_summary, action):
1755 Send an announce mail about a new upload.
1757 @type short_summary: string
1758 @param short_summary: Short summary text to include in the mail
1761 @param action: Set to false no real action will be done.
1764 @return: Textstring about action taken.
1769 announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1771 # Only do announcements for source uploads with a recent dpkg-dev installed
1772 if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1773 self.pkg.changes["architecture"].has_key("source"):
1779 self.Subst["__SHORT_SUMMARY__"] = short_summary
1781 for dist in self.pkg.changes["distribution"].keys():
1782 announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1783 if announce_list == "" or lists_done.has_key(announce_list):
1786 lists_done[announce_list] = 1
1787 summary += "Announcing to %s\n" % (announce_list)
1791 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1792 if cnf.get("Dinstall::TrackingServer") and \
1793 self.pkg.changes["architecture"].has_key("source"):
1794 trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1795 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1797 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1798 utils.send_mail(mail_message)
1800 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1802 if cnf.FindB("Dinstall::CloseBugs"):
1803 summary = self.close_bugs(summary, action)
1805 del self.Subst["__SHORT_SUMMARY__"]
1809 ###########################################################################
1811 def accept (self, summary, short_summary, session=None):
1815 This moves all files referenced from the .changes into the pool,
1816 sends the accepted mail, announces to lists, closes bugs and
1817 also checks for override disparities. If enabled it will write out
1818 the version history for the BTS Version Tracking and will finally call
1821 @type summary: string
1822 @param summary: Summary text
1824 @type short_summary: string
1825 @param short_summary: Short summary
1829 stats = SummaryStats()
1832 self.logger.log(["installing changes", self.pkg.changes_file])
1836 # Add the .dsc file to the DB first
1837 for newfile, entry in self.pkg.files.items():
1838 if entry["type"] == "dsc":
1839 source, dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
1843 # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
1844 for newfile, entry in self.pkg.files.items():
1845 if entry["type"] == "deb":
1846 poolfiles.append(add_deb_to_db(self, newfile, session))
1848 # If this is a sourceful diff only upload that is moving
1849 # cross-component we need to copy the .orig files into the new
1850 # component too for the same reasons as above.
1851 # XXX: mhy: I think this should be in add_dsc_to_db
1852 if self.pkg.changes["architecture"].has_key("source"):
1853 for orig_file in self.pkg.orig_files.keys():
1854 if not self.pkg.orig_files[orig_file].has_key("id"):
1855 continue # Skip if it's not in the pool
1856 orig_file_id = self.pkg.orig_files[orig_file]["id"]
1857 if self.pkg.orig_files[orig_file]["location"] == dsc_location_id:
1858 continue # Skip if the location didn't change
1861 oldf = get_poolfile_by_id(orig_file_id, session)
1862 old_filename = os.path.join(oldf.location.path, oldf.filename)
1863 old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
1864 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
1866 new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
1868 # TODO: Care about size/md5sum collisions etc
1869 (found, newf) = check_poolfile(new_filename, old_dat['size'], old_dat['md5sum'], dsc_location_id, session)
1871 # TODO: Uhm, what happens if newf isn't None - something has gone badly and we should cope
1873 utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
1874 newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
1878 # Don't reference the old file from this changes
1880 if p.file_id == oldf.file_id:
1883 poolfiles.append(newf)
1885 # Fix up the DSC references
1888 for df in source.srcfiles:
1889 if df.poolfile.file_id == oldf.file_id:
1890 # Add a new DSC entry and mark the old one for deletion
1891 # Don't do it in the loop so we don't change the thing we're iterating over
1893 newdscf.source_id = source.source_id
1894 newdscf.poolfile_id = newf.file_id
1895 session.add(newdscf)
1905 # Make sure that our source object is up-to-date
1906 session.expire(source)
1908 # Add changelog information to the database
1909 self.store_changelog()
1911 # Install the files into the pool
1912 for newfile, entry in self.pkg.files.items():
1913 destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
1914 utils.move(newfile, destination)
1915 self.logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
1916 stats.accept_bytes += float(entry["size"])
1918 # Copy the .changes file across for suite which need it.
1920 for suite_name in self.pkg.changes["distribution"].keys():
1921 if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
1922 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
1924 for dest in copy_changes.keys():
1925 utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
1927 # We're done - commit the database changes
1929 # Our SQL session will automatically start a new transaction after
1932 # Move the .changes into the 'done' directory
1933 utils.move(self.pkg.changes_file,
1934 os.path.join(cnf["Dir::Queue::Done"], os.path.basename(self.pkg.changes_file)))
1936 if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"):
1937 UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])
1940 self.Subst["__SUITE__"] = ""
1941 self.Subst["__SUMMARY__"] = summary
1942 mail_message = utils.TemplateSubst(self.Subst,
1943 os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted'))
1944 utils.send_mail(mail_message)
1945 self.announce(short_summary, 1)
1947 ## Helper stuff for DebBugs Version Tracking
1948 if cnf.Find("Dir::Queue::BTSVersionTrack"):
1949 if self.pkg.changes["architecture"].has_key("source"):
1950 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1951 version_history = os.fdopen(fd, 'w')
1952 version_history.write(self.pkg.dsc["bts changelog"])
1953 version_history.close()
1954 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1955 self.pkg.changes_file[:-8]+".versions")
1956 os.rename(temp_filename, filename)
1957 os.chmod(filename, 0644)
1959 # Write out the binary -> source mapping.
1960 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1961 debinfo = os.fdopen(fd, 'w')
1962 for name, entry in sorted(self.pkg.files.items()):
1963 if entry["type"] == "deb":
1964 line = " ".join([entry["package"], entry["version"],
1965 entry["architecture"], entry["source package"],
1966 entry["source version"]])
1967 debinfo.write(line+"\n")
1969 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1970 self.pkg.changes_file[:-8]+".debinfo")
1971 os.rename(temp_filename, filename)
1972 os.chmod(filename, 0644)
1976 # Set up our copy queues (e.g. buildd queues)
1977 for suite_name in self.pkg.changes["distribution"].keys():
1978 suite = get_suite(suite_name, session)
1979 for q in suite.copy_queues:
1981 q.add_file_from_pool(f)
1986 stats.accept_count += 1
1988 def check_override(self):
1990 Checks override entries for validity. Mails "Override disparity" warnings,
1991 if that feature is enabled.
1993 Abandons the check if
1994 - override disparity checks are disabled
1995 - mail sending is disabled
2000 # Abandon the check if override disparity checks have been disabled
2001 if not cnf.FindB("Dinstall::OverrideDisparityCheck"):
2004 summary = self.pkg.check_override()
2009 overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
2012 self.Subst["__SUMMARY__"] = summary
2013 mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
2014 utils.send_mail(mail_message)
2015 del self.Subst["__SUMMARY__"]
2017 ###########################################################################
2019 def remove(self, from_dir=None):
2021 Used (for instance) in p-u to remove the package from unchecked
2023 Also removes the package from holding area.
2025 if from_dir is None:
2026 from_dir = self.pkg.directory
2029 for f in self.pkg.files.keys():
2030 os.unlink(os.path.join(from_dir, f))
2031 if os.path.exists(os.path.join(h.holding_dir, f)):
2032 os.unlink(os.path.join(h.holding_dir, f))
2034 os.unlink(os.path.join(from_dir, self.pkg.changes_file))
2035 if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
2036 os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
2038 ###########################################################################
2040 def move_to_queue (self, queue):
2042 Move files to a destination queue using the permissions in the table
2045 utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
2046 queue.path, perms=int(queue.change_perms, 8))
2047 for f in self.pkg.files.keys():
2048 utils.move(os.path.join(h.holding_dir, f), queue.path, perms=int(queue.perms, 8))
2050 ###########################################################################
2052 def force_reject(self, reject_files):
2054 Forcefully move files from the current directory to the
2055 reject directory. If any file already exists in the reject
2056 directory it will be moved to the morgue to make way for
2059 @type reject_files: dict
2060 @param reject_files: file dictionary
2066 for file_entry in reject_files:
2067 # Skip any files which don't exist or which we don't have permission to copy.
2068 if os.access(file_entry, os.R_OK) == 0:
2071 dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
2074 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
2076 # File exists? Let's find a new name by adding a number
2077 if e.errno == errno.EEXIST:
2079 dest_file = utils.find_next_free(dest_file, 255)
2080 except NoFreeFilenameError:
2081 # Something's either gone badly Pete Tong, or
2082 # someone is trying to exploit us.
2083 utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Queue::Reject"]))
2086 # Make sure we really got it
2088 dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2091 utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
2095 # If we got here, we own the destination file, so we can
2096 # safely overwrite it.
2097 utils.move(file_entry, dest_file, 1, perms=0660)
2100 ###########################################################################
2101 def do_reject (self, manual=0, reject_message="", notes=""):
2103 Reject an upload. If called without a reject message or C{manual} is
2104 true, spawn an editor so the user can write one.
2107 @param manual: manual or automated rejection
2109 @type reject_message: string
2110 @param reject_message: A reject message
2115 # If we weren't given a manual rejection message, spawn an
2116 # editor so the user can add one in...
2117 if manual and not reject_message:
2118 (fd, temp_filename) = utils.temp_filename()
2119 temp_file = os.fdopen(fd, 'w')
2122 temp_file.write("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
2123 % (note.author, note.version, note.notedate, note.comment))
2125 editor = os.environ.get("EDITOR","vi")
2127 while answer == 'E':
2128 os.system("%s %s" % (editor, temp_filename))
2129 temp_fh = utils.open_file(temp_filename)
2130 reject_message = "".join(temp_fh.readlines())
2132 print "Reject message:"
2133 print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1)
2134 prompt = "[R]eject, Edit, Abandon, Quit ?"
2136 while prompt.find(answer) == -1:
2137 answer = utils.our_raw_input(prompt)
2138 m = re_default_answer.search(prompt)
2141 answer = answer[:1].upper()
2142 os.unlink(temp_filename)
2148 print "Rejecting.\n"
2152 reason_filename = self.pkg.changes_file[:-8] + ".reason"
2153 reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2155 # Move all the files into the reject directory
2156 reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
2157 self.force_reject(reject_files)
2159 # If we fail here someone is probably trying to exploit the race
2160 # so let's just raise an exception ...
2161 if os.path.exists(reason_filename):
2162 os.unlink(reason_filename)
2163 reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2165 rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
2169 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2170 self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
2171 self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)"
2172 os.write(reason_fd, reject_message)
2173 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2175 # Build up the rejection email
2176 user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
2177 self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
2178 self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
2179 self.Subst["__REJECT_MESSAGE__"] = ""
2180 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2181 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2182 # Write the rejection email out as the <foo>.reason file
2183 os.write(reason_fd, reject_mail_message)
2185 del self.Subst["__REJECTOR_ADDRESS__"]
2186 del self.Subst["__MANUAL_REJECT_MESSAGE__"]
2187 del self.Subst["__CC__"]
2191 # Send the rejection mail
2192 utils.send_mail(reject_mail_message)
2195 self.logger.log(["rejected", self.pkg.changes_file])
2199 ################################################################################
2200 def in_override_p(self, package, component, suite, binary_type, filename, session):
2202 Check if a package already has override entries in the DB
2204 @type package: string
2205 @param package: package name
2207 @type component: string
2208 @param component: database id of the component
2211 @param suite: database id of the suite
2213 @type binary_type: string
2214 @param binary_type: type of the package
2216 @type filename: string
2217 @param filename: filename we check
2219 @return: the database result. But noone cares anyway.
2225 if binary_type == "": # must be source
2228 file_type = binary_type
2230 # Override suite name; used for example with proposed-updates
2231 if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
2232 suite = cnf["Suite::%s::OverrideSuite" % (suite)]
2234 result = get_override(package, suite, component, file_type, session)
2236 # If checking for a source package fall back on the binary override type
2237 if file_type == "dsc" and len(result) < 1:
2238 result = get_override(package, suite, component, ['deb', 'udeb'], session)
2240 # Remember the section and priority so we can check them later if appropriate
2243 self.pkg.files[filename]["override section"] = result.section.section
2244 self.pkg.files[filename]["override priority"] = result.priority.priority
2249 ################################################################################
2250 def get_anyversion(self, sv_list, suite):
2253 @param sv_list: list of (suite, version) tuples to check
2256 @param suite: suite name
2262 anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
2263 for (s, v) in sv_list:
2264 if s in [ x.lower() for x in anysuite ]:
2265 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
2270 ################################################################################
2272 def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
2275 @param sv_list: list of (suite, version) tuples to check
2277 @type filename: string
2278 @param filename: XXX
2280 @type new_version: string
2281 @param new_version: XXX
2283 Ensure versions are newer than existing packages in target
2284 suites and that cross-suite version checking rules as
2285 set out in the conf file are satisfied.
2290 # Check versions for each target suite
2291 for target_suite in self.pkg.changes["distribution"].keys():
2292 must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
2293 must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
2295 # Enforce "must be newer than target suite" even if conffile omits it
2296 if target_suite not in must_be_newer_than:
2297 must_be_newer_than.append(target_suite)
2299 for (suite, existent_version) in sv_list:
2300 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
2302 if suite in must_be_newer_than and sourceful and vercmp < 1:
2303 self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2305 if suite in must_be_older_than and vercmp > -1:
2308 if self.pkg.changes.get('distribution-version', {}).has_key(suite):
2309 # we really use the other suite, ignoring the conflicting one ...
2310 addsuite = self.pkg.changes["distribution-version"][suite]
2312 add_version = self.get_anyversion(sv_list, addsuite)
2313 target_version = self.get_anyversion(sv_list, target_suite)
2316 # not add_version can only happen if we map to a suite
2317 # that doesn't enhance the suite we're propup'ing from.
2318 # so "propup-ver x a b c; map a d" is a problem only if
2319 # d doesn't enhance a.
2321 # i think we could always propagate in this case, rather
2322 # than complaining. either way, this isn't a REJECT issue
2324 # And - we really should complain to the dorks who configured dak
2325 self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
2326 self.pkg.changes.setdefault("propdistribution", {})
2327 self.pkg.changes["propdistribution"][addsuite] = 1
2329 elif not target_version:
2330 # not targets_version is true when the package is NEW
2331 # we could just stick with the "...old version..." REJECT
2332 # for this, I think.
2333 self.rejects.append("Won't propogate NEW packages.")
2334 elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2335 # propogation would be redundant. no need to reject though.
2336 self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2338 elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2339 apt_pkg.VersionCompare(add_version, target_version) >= 0:
2341 self.warnings.append("Propogating upload to %s" % (addsuite))
2342 self.pkg.changes.setdefault("propdistribution", {})
2343 self.pkg.changes["propdistribution"][addsuite] = 1
2347 self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2349 ################################################################################
2350 def check_binary_against_db(self, filename, session):
2351 # Ensure version is sane
2352 q = session.query(BinAssociation)
2353 q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[filename]["package"])
2354 q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[filename]["architecture"], 'all']))
2356 self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2357 filename, self.pkg.files[filename]["version"], sourceful=False)
2359 # Check for any existing copies of the file
2360 q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"])
2361 q = q.filter_by(version=self.pkg.files[filename]["version"])
2362 q = q.join(Architecture).filter_by(arch_string=self.pkg.files[filename]["architecture"])
2365 self.rejects.append("%s: can not overwrite existing copy already in the archive." % filename)
2367 ################################################################################
2369 def check_source_against_db(self, filename, session):
2370 source = self.pkg.dsc.get("source")
2371 version = self.pkg.dsc.get("version")
2373 # Ensure version is sane
2374 q = session.query(SrcAssociation)
2375 q = q.join(DBSource).filter(DBSource.source==source)
2377 self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2378 filename, version, sourceful=True)
2380 ################################################################################
2381 def check_dsc_against_db(self, filename, session):
2384 @warning: NB: this function can remove entries from the 'files' index [if
2385 the orig tarball is a duplicate of the one in the archive]; if
2386 you're iterating over 'files' and call this function as part of
2387 the loop, be sure to add a check to the top of the loop to
2388 ensure you haven't just tried to dereference the deleted entry.
2393 self.pkg.orig_files = {} # XXX: do we need to clear it?
2394 orig_files = self.pkg.orig_files
2396 # Try and find all files mentioned in the .dsc. This has
2397 # to work harder to cope with the multiple possible
2398 # locations of an .orig.tar.gz.
2399 # The ordering on the select is needed to pick the newest orig
2400 # when it exists in multiple places.
2401 for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2403 if self.pkg.files.has_key(dsc_name):
2404 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2405 actual_size = int(self.pkg.files[dsc_name]["size"])
2406 found = "%s in incoming" % (dsc_name)
2408 # Check the file does not already exist in the archive
2409 ql = get_poolfile_like_name(dsc_name, session)
2411 # Strip out anything that isn't '%s' or '/%s$'
2413 if not i.filename.endswith(dsc_name):
2416 # "[dak] has not broken them. [dak] has fixed a
2417 # brokenness. Your crappy hack exploited a bug in
2420 # "(Come on! I thought it was always obvious that
2421 # one just doesn't release different files with
2422 # the same name and version.)"
2423 # -- ajk@ on d-devel@l.d.o
2426 # Ignore exact matches for .orig.tar.gz
2428 if re_is_orig_source.match(dsc_name):
2430 if self.pkg.files.has_key(dsc_name) and \
2431 int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2432 self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2433 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2434 # TODO: Don't delete the entry, just mark it as not needed
2435 # This would fix the stupidity of changing something we often iterate over
2436 # whilst we're doing it
2437 del self.pkg.files[dsc_name]
2438 dsc_entry["files id"] = i.file_id
2439 if not orig_files.has_key(dsc_name):
2440 orig_files[dsc_name] = {}
2441 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
2444 # Don't bitch that we couldn't find this file later
2446 self.later_check_files.remove(dsc_name)
2452 self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2454 elif re_is_orig_source.match(dsc_name):
2456 ql = get_poolfile_like_name(dsc_name, session)
2458 # Strip out anything that isn't '%s' or '/%s$'
2459 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2461 if not i.filename.endswith(dsc_name):
2465 # Unfortunately, we may get more than one match here if,
2466 # for example, the package was in potato but had an -sa
2467 # upload in woody. So we need to choose the right one.
2469 # default to something sane in case we don't match any or have only one
2474 old_file = os.path.join(i.location.path, i.filename)
2475 old_file_fh = utils.open_file(old_file)
2476 actual_md5 = apt_pkg.md5sum(old_file_fh)
2478 actual_size = os.stat(old_file)[stat.ST_SIZE]
2479 if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2482 old_file = os.path.join(i.location.path, i.filename)
2483 old_file_fh = utils.open_file(old_file)
2484 actual_md5 = apt_pkg.md5sum(old_file_fh)
2486 actual_size = os.stat(old_file)[stat.ST_SIZE]
2488 suite_type = x.location.archive_type
2489 # need this for updating dsc_files in install()
2490 dsc_entry["files id"] = x.file_id
2491 # See install() in process-accepted...
2492 if not orig_files.has_key(dsc_name):
2493 orig_files[dsc_name] = {}
2494 orig_files[dsc_name]["id"] = x.file_id
2495 orig_files[dsc_name]["path"] = old_file
2496 orig_files[dsc_name]["location"] = x.location.location_id
2498 # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2499 # Not there? Check the queue directories...
2500 for directory in [ "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2501 if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2503 in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2504 if os.path.exists(in_otherdir):
2505 in_otherdir_fh = utils.open_file(in_otherdir)
2506 actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2507 in_otherdir_fh.close()
2508 actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2510 if not orig_files.has_key(dsc_name):
2511 orig_files[dsc_name] = {}
2512 orig_files[dsc_name]["path"] = in_otherdir
2515 self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (filename, dsc_name))
2518 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (filename, dsc_name))
2520 if actual_md5 != dsc_entry["md5sum"]:
2521 self.rejects.append("md5sum for %s doesn't match %s." % (found, filename))
2522 if actual_size != int(dsc_entry["size"]):
2523 self.rejects.append("size for %s doesn't match %s." % (found, filename))
2525 ################################################################################
2526 # This is used by process-new and process-holding to recheck a changes file
2527 # at the time we're running. It mainly wraps various other internal functions
2528 # and is similar to accepted_checks - these should probably be tidied up
2530 def recheck(self, session):
2532 for f in self.pkg.files.keys():
2533 # The .orig.tar.gz can disappear out from under us is it's a
2534 # duplicate of one in the archive.
2535 if not self.pkg.files.has_key(f):
2538 entry = self.pkg.files[f]
2540 # Check that the source still exists
2541 if entry["type"] == "deb":
2542 source_version = entry["source version"]
2543 source_package = entry["source package"]
2544 if not self.pkg.changes["architecture"].has_key("source") \
2545 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
2546 source_epochless_version = re_no_epoch.sub('', source_version)
2547 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
2549 for q in ["Embargoed", "Unembargoed", "Newstage"]:
2550 if cnf.has_key("Dir::Queue::%s" % (q)):
2551 if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
2554 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
2556 # Version and file overwrite checks
2557 if entry["type"] == "deb":
2558 self.check_binary_against_db(f, session)
2559 elif entry["type"] == "dsc":
2560 self.check_source_against_db(f, session)
2561 self.check_dsc_against_db(f, session)
2563 ################################################################################
2564 def accepted_checks(self, overwrite_checks, session):
2565 # Recheck anything that relies on the database; since that's not
2566 # frozen between accept and our run time when called from p-a.
2568 # overwrite_checks is set to False when installing to stable/oldstable
2573 # Find the .dsc (again)
2575 for f in self.pkg.files.keys():
2576 if self.pkg.files[f]["type"] == "dsc":
2579 for checkfile in self.pkg.files.keys():
2580 # The .orig.tar.gz can disappear out from under us is it's a
2581 # duplicate of one in the archive.
2582 if not self.pkg.files.has_key(checkfile):
2585 entry = self.pkg.files[checkfile]
2587 # Check that the source still exists
2588 if entry["type"] == "deb":
2589 source_version = entry["source version"]
2590 source_package = entry["source package"]
2591 if not self.pkg.changes["architecture"].has_key("source") \
2592 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
2593 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2595 # Version and file overwrite checks
2596 if overwrite_checks:
2597 if entry["type"] == "deb":
2598 self.check_binary_against_db(checkfile, session)
2599 elif entry["type"] == "dsc":
2600 self.check_source_against_db(checkfile, session)
2601 self.check_dsc_against_db(dsc_filename, session)
2603 # propogate in the case it is in the override tables:
2604 for suite in self.pkg.changes.get("propdistribution", {}).keys():
2605 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2606 propogate[suite] = 1
2608 nopropogate[suite] = 1
2610 for suite in propogate.keys():
2611 if suite in nopropogate:
2613 self.pkg.changes["distribution"][suite] = 1
2615 for checkfile in self.pkg.files.keys():
2616 # Check the package is still in the override tables
2617 for suite in self.pkg.changes["distribution"].keys():
2618 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2619 self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2621 ################################################################################
2622 # This is not really a reject, but an unaccept, but since a) the code for
2623 # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2624 # extremely rare, for now we'll go with whining at our admin folks...
2626 def do_unaccept(self):
2630 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2631 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2632 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2633 self.Subst["__BCC__"] = "X-DAK: dak process-accepted"
2634 if cnf.has_key("Dinstall::Bcc"):
2635 self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2637 template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2639 reject_mail_message = utils.TemplateSubst(self.Subst, template)
2641 # Write the rejection email out as the <foo>.reason file
2642 reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2643 reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2645 # If we fail here someone is probably trying to exploit the race
2646 # so let's just raise an exception ...
2647 if os.path.exists(reject_filename):
2648 os.unlink(reject_filename)
2650 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2651 os.write(fd, reject_mail_message)
2654 utils.send_mail(reject_mail_message)
2656 del self.Subst["__REJECTOR_ADDRESS__"]
2657 del self.Subst["__REJECT_MESSAGE__"]
2658 del self.Subst["__CC__"]
2660 ################################################################################
2661 # If any file of an upload has a recent mtime then chances are good
2662 # the file is still being uploaded.
2664 def upload_too_new(self):
2667 # Move back to the original directory to get accurate time stamps
2669 os.chdir(self.pkg.directory)
2670 file_list = self.pkg.files.keys()
2671 file_list.extend(self.pkg.dsc_files.keys())
2672 file_list.append(self.pkg.changes_file)
2675 last_modified = time.time()-os.path.getmtime(f)
2676 if last_modified < int(cnf["Dinstall::SkipTime"]):
2685 def store_changelog(self):
2687 # Skip binary-only upload if it is not a bin-NMU
2688 if not self.pkg.changes['architecture'].has_key('source'):
2689 from daklib.regexes import re_bin_only_nmu
2690 if not re_bin_only_nmu.search(self.pkg.changes['version']):
2693 session = DBConn().session()
2695 # Check if upload already has a changelog entry
2696 query = """SELECT changelog_id FROM changes WHERE source = :source
2697 AND version = :version AND architecture = :architecture AND changelog_id != 0"""
2698 if session.execute(query, {'source': self.pkg.changes['source'], \
2699 'version': self.pkg.changes['version'], \
2700 'architecture': " ".join(self.pkg.changes['architecture'].keys())}).rowcount:
2704 # Add current changelog text into changelogs_text table, return created ID
2705 query = "INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id"
2706 ID = session.execute(query, {'changelog': self.pkg.changes['changes']}).fetchone()[0]
2708 # Link ID to the upload available in changes table
2709 query = """UPDATE changes SET changelog_id = :id WHERE source = :source
2710 AND version = :version AND architecture = :architecture"""
2711 session.execute(query, {'id': ID, 'source': self.pkg.changes['source'], \
2712 'version': self.pkg.changes['version'], \
2713 'architecture': " ".join(self.pkg.changes['architecture'].keys())})