5 Queue utility functions for dak
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ###############################################################################
41 from sqlalchemy.sql.expression import desc
42 from sqlalchemy.orm.exc import NoResultFound
46 from dak_exceptions import *
49 from config import Config
50 from holding import Holding
51 from urgencylog import UrgencyLog
53 from summarystats import SummaryStats
54 from utils import parse_changes, check_dsc_files
55 from textutils import fix_maintainer
56 from binary import Binary
57 from lintian import parse_lintian_output, generate_reject_messages
59 ###############################################################################
61 def get_type(f, session):
63 Get the file type of C{f}
66 @param f: file entry from Changes object
68 @type session: SQLA Session
69 @param session: SQL Alchemy session object
76 if f.has_key("dbtype"):
77 file_type = f["dbtype"]
78 elif re_source_ext.match(f["type"]):
81 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
83 # Validate the override type
84 type_id = get_override_type(file_type, session)
86 utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
90 ################################################################################
92 # Determine what parts in a .changes are NEW
94 def determine_new(changes, files, warn=1):
96 Determine what parts in a C{changes} file are NEW.
98 @type changes: Upload.Pkg.changes dict
99 @param changes: Changes dictionary
101 @type files: Upload.Pkg.files dict
102 @param files: Files dictionary
105 @param warn: Warn if overrides are added for (old)stable
108 @return: dictionary of NEW components.
113 session = DBConn().session()
115 # Build up a list of potentially new things
116 for name, f in files.items():
117 # Skip byhand elements
118 # if f["type"] == "byhand":
121 priority = f["priority"]
122 section = f["section"]
123 file_type = get_type(f, session)
124 component = f["component"]
126 if file_type == "dsc":
129 if not new.has_key(pkg):
131 new[pkg]["priority"] = priority
132 new[pkg]["section"] = section
133 new[pkg]["type"] = file_type
134 new[pkg]["component"] = component
135 new[pkg]["files"] = []
137 old_type = new[pkg]["type"]
138 if old_type != file_type:
139 # source gets trumped by deb or udeb
140 if old_type == "dsc":
141 new[pkg]["priority"] = priority
142 new[pkg]["section"] = section
143 new[pkg]["type"] = file_type
144 new[pkg]["component"] = component
146 new[pkg]["files"].append(name)
148 if f.has_key("othercomponents"):
149 new[pkg]["othercomponents"] = f["othercomponents"]
151 # Fix up the list of target suites
153 for suite in changes["suite"].keys():
154 override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
156 (olderr, newerr) = (get_suite(suite, session) == None,
157 get_suite(override, session) == None)
159 (oinv, newinv) = ("", "")
160 if olderr: oinv = "invalid "
161 if newerr: ninv = "invalid "
162 print "warning: overriding %ssuite %s to %ssuite %s" % (
163 oinv, suite, ninv, override)
164 del changes["suite"][suite]
165 changes["suite"][override] = 1
167 for suite in changes["suite"].keys():
168 for pkg in new.keys():
169 ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
171 for file_entry in new[pkg]["files"]:
172 if files[file_entry].has_key("new"):
173 del files[file_entry]["new"]
177 for s in ['stable', 'oldstable']:
178 if changes["suite"].has_key(s):
179 print "WARNING: overrides will be added for %s!" % s
180 for pkg in new.keys():
181 if new[pkg].has_key("othercomponents"):
182 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
188 ################################################################################
190 def check_valid(new):
192 Check if section and priority for NEW packages exist in database.
193 Additionally does sanity checks:
194 - debian-installer packages have to be udeb (or source)
195 - non debian-installer packages can not be udeb
196 - source priority can only be assigned to dsc file types
199 @param new: Dict of new packages with their section, priority and type.
202 for pkg in new.keys():
203 section_name = new[pkg]["section"]
204 priority_name = new[pkg]["priority"]
205 file_type = new[pkg]["type"]
207 section = get_section(section_name)
209 new[pkg]["section id"] = -1
211 new[pkg]["section id"] = section.section_id
213 priority = get_priority(priority_name)
215 new[pkg]["priority id"] = -1
217 new[pkg]["priority id"] = priority.priority_id
220 di = section_name.find("debian-installer") != -1
222 # If d-i, we must be udeb and vice-versa
223 if (di and file_type not in ("udeb", "dsc")) or \
224 (not di and file_type == "udeb"):
225 new[pkg]["section id"] = -1
227 # If dsc we need to be source and vice-versa
228 if (priority == "source" and file_type != "dsc") or \
229 (priority != "source" and file_type == "dsc"):
230 new[pkg]["priority id"] = -1
232 ###############################################################################
234 def check_status(files):
236 for f in files.keys():
237 if files[f].has_key("byhand"):
239 elif files[f].has_key("new"):
243 ###############################################################################
245 # Used by Upload.check_timestamps
246 class TarTime(object):
247 def __init__(self, future_cutoff, past_cutoff):
249 self.future_cutoff = future_cutoff
250 self.past_cutoff = past_cutoff
253 self.future_files = {}
254 self.ancient_files = {}
256 def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
257 if MTime > self.future_cutoff:
258 self.future_files[Name] = MTime
259 if MTime < self.past_cutoff:
260 self.ancient_files[Name] = MTime
262 ###############################################################################
264 class Upload(object):
266 Everything that has to do with an upload processed.
274 ###########################################################################
277 """ Reset a number of internal variables."""
279 # Initialize the substitution template map
282 self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
283 self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
284 self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
285 self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
293 def package_info(self):
295 Format various messages from this Upload to send to the maintainer.
299 ('Reject Reasons', self.rejects),
300 ('Warnings', self.warnings),
301 ('Notes', self.notes),
305 for title, messages in msgs:
307 msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))
312 ###########################################################################
313 def update_subst(self):
314 """ Set up the per-package template substitution mappings """
318 # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
319 if not self.pkg.changes.has_key("architecture") or not \
320 isinstance(self.pkg.changes["architecture"], dict):
321 self.pkg.changes["architecture"] = { "Unknown" : "" }
323 # and maintainer2047 may not exist.
324 if not self.pkg.changes.has_key("maintainer2047"):
325 self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
327 self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
328 self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
329 self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
331 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
332 if self.pkg.changes["architecture"].has_key("source") and \
333 self.pkg.changes["changedby822"] != "" and \
334 (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
336 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
337 self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
338 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
340 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
341 self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
342 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
344 if "sponsoremail" in self.pkg.changes:
345 self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
347 if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
348 self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
350 # Apply any global override of the Maintainer field
351 if cnf.get("Dinstall::OverrideMaintainer"):
352 self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
353 self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
355 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
356 self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
357 self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
359 ###########################################################################
360 def load_changes(self, filename):
363 @rvalue: whether the changes file was valid or not. We may want to
364 reject even if this is True (see what gets put in self.rejects).
365 This is simply to prevent us even trying things later which will
366 fail because we couldn't properly parse the file.
369 self.pkg.changes_file = filename
371 # Parse the .changes field into a dictionary
373 self.pkg.changes.update(parse_changes(filename))
374 except CantOpenError:
375 self.rejects.append("%s: can't read file." % (filename))
377 except ParseChangesError, line:
378 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
380 except ChangesUnicodeError:
381 self.rejects.append("%s: changes file not proper utf-8" % (filename))
384 # Parse the Files field from the .changes into another dictionary
386 self.pkg.files.update(utils.build_file_list(self.pkg.changes))
387 except ParseChangesError, line:
388 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
390 except UnknownFormatError, format:
391 self.rejects.append("%s: unknown format '%s'." % (filename, format))
394 # Check for mandatory fields
395 for i in ("distribution", "source", "binary", "architecture",
396 "version", "maintainer", "files", "changes", "description"):
397 if not self.pkg.changes.has_key(i):
398 # Avoid undefined errors later
399 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
402 # Strip a source version in brackets from the source field
403 if re_strip_srcver.search(self.pkg.changes["source"]):
404 self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
406 # Ensure the source field is a valid package name.
407 if not re_valid_pkg_name.match(self.pkg.changes["source"]):
408 self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
410 # Split multi-value fields into a lower-level dictionary
411 for i in ("architecture", "distribution", "binary", "closes"):
412 o = self.pkg.changes.get(i, "")
414 del self.pkg.changes[i]
416 self.pkg.changes[i] = {}
419 self.pkg.changes[i][j] = 1
421 # Fix the Maintainer: field to be RFC822/2047 compatible
423 (self.pkg.changes["maintainer822"],
424 self.pkg.changes["maintainer2047"],
425 self.pkg.changes["maintainername"],
426 self.pkg.changes["maintaineremail"]) = \
427 fix_maintainer (self.pkg.changes["maintainer"])
428 except ParseMaintError, msg:
429 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
430 % (filename, self.pkg.changes["maintainer"], msg))
432 # ...likewise for the Changed-By: field if it exists.
434 (self.pkg.changes["changedby822"],
435 self.pkg.changes["changedby2047"],
436 self.pkg.changes["changedbyname"],
437 self.pkg.changes["changedbyemail"]) = \
438 fix_maintainer (self.pkg.changes.get("changed-by", ""))
439 except ParseMaintError, msg:
440 self.pkg.changes["changedby822"] = ""
441 self.pkg.changes["changedby2047"] = ""
442 self.pkg.changes["changedbyname"] = ""
443 self.pkg.changes["changedbyemail"] = ""
445 self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
446 % (filename, self.pkg.changes["changed-by"], msg))
448 # Ensure all the values in Closes: are numbers
449 if self.pkg.changes.has_key("closes"):
450 for i in self.pkg.changes["closes"].keys():
451 if re_isanum.match (i) == None:
452 self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
454 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
455 self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
456 self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
458 # Check the .changes is non-empty
459 if not self.pkg.files:
460 self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
463 # Changes was syntactically valid even if we'll reject
466 ###########################################################################
468 def check_distributions(self):
469 "Check and map the Distribution field"
473 # Handle suite mappings
474 for m in Cnf.ValueList("SuiteMappings"):
477 if mtype == "map" or mtype == "silent-map":
478 (source, dest) = args[1:3]
479 if self.pkg.changes["distribution"].has_key(source):
480 del self.pkg.changes["distribution"][source]
481 self.pkg.changes["distribution"][dest] = 1
482 if mtype != "silent-map":
483 self.notes.append("Mapping %s to %s." % (source, dest))
484 if self.pkg.changes.has_key("distribution-version"):
485 if self.pkg.changes["distribution-version"].has_key(source):
486 self.pkg.changes["distribution-version"][source]=dest
487 elif mtype == "map-unreleased":
488 (source, dest) = args[1:3]
489 if self.pkg.changes["distribution"].has_key(source):
490 for arch in self.pkg.changes["architecture"].keys():
491 if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
492 self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
493 del self.pkg.changes["distribution"][source]
494 self.pkg.changes["distribution"][dest] = 1
496 elif mtype == "ignore":
498 if self.pkg.changes["distribution"].has_key(suite):
499 del self.pkg.changes["distribution"][suite]
500 self.warnings.append("Ignoring %s as a target suite." % (suite))
501 elif mtype == "reject":
503 if self.pkg.changes["distribution"].has_key(suite):
504 self.rejects.append("Uploads to %s are not accepted." % (suite))
505 elif mtype == "propup-version":
506 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
508 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
509 if self.pkg.changes["distribution"].has_key(args[1]):
510 self.pkg.changes.setdefault("distribution-version", {})
511 for suite in args[2:]:
512 self.pkg.changes["distribution-version"][suite] = suite
514 # Ensure there is (still) a target distribution
515 if len(self.pkg.changes["distribution"].keys()) < 1:
516 self.rejects.append("No valid distribution remaining.")
518 # Ensure target distributions exist
519 for suite in self.pkg.changes["distribution"].keys():
520 if not Cnf.has_key("Suite::%s" % (suite)):
521 self.rejects.append("Unknown distribution `%s'." % (suite))
523 ###########################################################################
525 def binary_file_checks(self, f, session):
527 entry = self.pkg.files[f]
529 # Extract package control information
530 deb_file = utils.open_file(f)
532 control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
534 self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
536 # Can't continue, none of the checks on control would work.
539 # Check for mandantory "Description:"
542 apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
544 self.rejects.append("%s: Missing Description in binary package" % (f))
549 # Check for mandatory fields
550 for field in [ "Package", "Architecture", "Version" ]:
551 if control.Find(field) == None:
553 self.rejects.append("%s: No %s field in control." % (f, field))
556 # Ensure the package name matches the one give in the .changes
557 if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
558 self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
560 # Validate the package field
561 package = control.Find("Package")
562 if not re_valid_pkg_name.match(package):
563 self.rejects.append("%s: invalid package name '%s'." % (f, package))
565 # Validate the version field
566 version = control.Find("Version")
567 if not re_valid_version.match(version):
568 self.rejects.append("%s: invalid version number '%s'." % (f, version))
570 # Ensure the architecture of the .deb is one we know about.
571 default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
572 architecture = control.Find("Architecture")
573 upload_suite = self.pkg.changes["distribution"].keys()[0]
575 if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
576 and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
577 self.rejects.append("Unknown architecture '%s'." % (architecture))
579 # Ensure the architecture of the .deb is one of the ones
580 # listed in the .changes.
581 if not self.pkg.changes["architecture"].has_key(architecture):
582 self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
584 # Sanity-check the Depends field
585 depends = control.Find("Depends")
587 self.rejects.append("%s: Depends field is empty." % (f))
589 # Sanity-check the Provides field
590 provides = control.Find("Provides")
592 provide = re_spacestrip.sub('', provides)
594 self.rejects.append("%s: Provides field is empty." % (f))
595 prov_list = provide.split(",")
596 for prov in prov_list:
597 if not re_valid_pkg_name.match(prov):
598 self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
600 # Check the section & priority match those given in the .changes (non-fatal)
601 if control.Find("Section") and entry["section"] != "" \
602 and entry["section"] != control.Find("Section"):
603 self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
604 (f, control.Find("Section", ""), entry["section"]))
605 if control.Find("Priority") and entry["priority"] != "" \
606 and entry["priority"] != control.Find("Priority"):
607 self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
608 (f, control.Find("Priority", ""), entry["priority"]))
610 entry["package"] = package
611 entry["architecture"] = architecture
612 entry["version"] = version
613 entry["maintainer"] = control.Find("Maintainer", "")
615 if f.endswith(".udeb"):
616 self.pkg.files[f]["dbtype"] = "udeb"
617 elif f.endswith(".deb"):
618 self.pkg.files[f]["dbtype"] = "deb"
620 self.rejects.append("%s is neither a .deb or a .udeb." % (f))
622 entry["source"] = control.Find("Source", entry["package"])
624 # Get the source version
625 source = entry["source"]
628 if source.find("(") != -1:
629 m = re_extract_src_version.match(source)
631 source_version = m.group(2)
633 if not source_version:
634 source_version = self.pkg.files[f]["version"]
636 entry["source package"] = source
637 entry["source version"] = source_version
639 # Ensure the filename matches the contents of the .deb
640 m = re_isadeb.match(f)
643 file_package = m.group(1)
644 if entry["package"] != file_package:
645 self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
646 (f, file_package, entry["dbtype"], entry["package"]))
647 epochless_version = re_no_epoch.sub('', control.Find("Version"))
650 file_version = m.group(2)
651 if epochless_version != file_version:
652 self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
653 (f, file_version, entry["dbtype"], epochless_version))
656 file_architecture = m.group(3)
657 if entry["architecture"] != file_architecture:
658 self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
659 (f, file_architecture, entry["dbtype"], entry["architecture"]))
661 # Check for existent source
662 source_version = entry["source version"]
663 source_package = entry["source package"]
664 if self.pkg.changes["architecture"].has_key("source"):
665 if source_version != self.pkg.changes["version"]:
666 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
667 (source_version, f, self.pkg.changes["version"]))
669 # Check in the SQL database
670 if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
671 # Check in one of the other directories
672 source_epochless_version = re_no_epoch.sub('', source_version)
673 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
674 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
676 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
679 dsc_file_exists = False
680 for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
681 if cnf.has_key("Dir::Queue::%s" % (myq)):
682 if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
683 dsc_file_exists = True
686 if not dsc_file_exists:
687 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
689 # Check the version and for file overwrites
690 self.check_binary_against_db(f, session)
692 # Temporarily disable contents generation until we change the table storage layout
695 #if len(b.rejects) > 0:
696 # for j in b.rejects:
697 # self.rejects.append(j)
699 def source_file_checks(self, f, session):
700 entry = self.pkg.files[f]
702 m = re_issource.match(f)
706 entry["package"] = m.group(1)
707 entry["version"] = m.group(2)
708 entry["type"] = m.group(3)
710 # Ensure the source package name matches the Source filed in the .changes
711 if self.pkg.changes["source"] != entry["package"]:
712 self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
714 # Ensure the source version matches the version in the .changes file
715 if re_is_orig_source.match(f):
716 changes_version = self.pkg.changes["chopversion2"]
718 changes_version = self.pkg.changes["chopversion"]
720 if changes_version != entry["version"]:
721 self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
723 # Ensure the .changes lists source in the Architecture field
724 if not self.pkg.changes["architecture"].has_key("source"):
725 self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
727 # Check the signature of a .dsc file
728 if entry["type"] == "dsc":
729 # check_signature returns either:
730 # (None, [list, of, rejects]) or (signature, [])
731 (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
733 self.rejects.append(j)
735 entry["architecture"] = "source"
737 def per_suite_file_checks(self, f, suite, session):
739 entry = self.pkg.files[f]
742 if entry.has_key("byhand"):
745 # Check we have fields we need to do these checks
747 for m in ['component', 'package', 'priority', 'size', 'md5sum']:
748 if not entry.has_key(m):
749 self.rejects.append("file '%s' does not have field %s set" % (f, m))
755 # Handle component mappings
756 for m in cnf.ValueList("ComponentMappings"):
757 (source, dest) = m.split()
758 if entry["component"] == source:
759 entry["original component"] = source
760 entry["component"] = dest
762 # Ensure the component is valid for the target suite
763 if cnf.has_key("Suite:%s::Components" % (suite)) and \
764 entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
765 self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
768 # Validate the component
769 if not get_component(entry["component"], session):
770 self.rejects.append("file '%s' has unknown component '%s'." % (f, entry["component"]))
773 # See if the package is NEW
774 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
777 # Validate the priority
778 if entry["priority"].find('/') != -1:
779 self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
781 # Determine the location
782 location = cnf["Dir::Pool"]
783 l = get_location(location, entry["component"], session=session)
785 self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %)" % entry["component"])
786 entry["location id"] = -1
788 entry["location id"] = l.location_id
790 # Check the md5sum & size against existing files (if any)
791 entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
793 found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
794 entry["size"], entry["md5sum"], entry["location id"])
797 self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
798 elif found is False and poolfile is not None:
799 self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
802 entry["files id"] = None
804 entry["files id"] = poolfile.file_id
806 # Check for packages that have moved from one component to another
807 entry['suite'] = suite
808 res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
810 entry["othercomponents"] = res.fetchone()[0]
812 def check_files(self, action=True):
813 file_keys = self.pkg.files.keys()
819 os.chdir(self.pkg.directory)
821 ret = holding.copy_to_holding(f)
823 # XXX: Should we bail out here or try and continue?
824 self.rejects.append(ret)
828 # check we already know the changes file
829 # [NB: this check must be done post-suite mapping]
830 base_filename = os.path.basename(self.pkg.changes_file)
832 session = DBConn().session()
835 dbc = session.query(DBChange).filter_by(changesname=base_filename).one()
836 # if in the pool or in a queue other than unchecked, reject
837 if (dbc.in_queue is None) \
838 or (dbc.in_queue is not None
839 and dbc.in_queue.queue_name not in ["unchecked", "newstage"]):
840 self.rejects.append("%s file already known to dak" % base_filename)
841 except NoResultFound, e:
848 for f, entry in self.pkg.files.items():
849 # Ensure the file does not already exist in one of the accepted directories
850 for d in [ "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
851 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
852 if os.path.exists(os.path.join(cnf["Dir::Queue::%s" % (d) ], f)):
853 self.rejects.append("%s file already exists in the %s directory." % (f, d))
855 if not re_taint_free.match(f):
856 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
858 # Check the file is readable
859 if os.access(f, os.R_OK) == 0:
860 # When running in -n, copy_to_holding() won't have
861 # generated the reject_message, so we need to.
863 if os.path.exists(f):
864 self.rejects.append("Can't read `%s'. [permission denied]" % (f))
866 self.rejects.append("Can't read `%s'. [file not found]" % (f))
867 entry["type"] = "unreadable"
870 # If it's byhand skip remaining checks
871 if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
873 entry["type"] = "byhand"
875 # Checks for a binary package...
876 elif re_isadeb.match(f):
878 entry["type"] = "deb"
880 # This routine appends to self.rejects/warnings as appropriate
881 self.binary_file_checks(f, session)
883 # Checks for a source package...
884 elif re_issource.match(f):
887 # This routine appends to self.rejects/warnings as appropriate
888 self.source_file_checks(f, session)
890 # Not a binary or source package? Assume byhand...
893 entry["type"] = "byhand"
895 # Per-suite file checks
896 entry["oldfiles"] = {}
897 for suite in self.pkg.changes["distribution"].keys():
898 self.per_suite_file_checks(f, suite, session)
902 # If the .changes file says it has source, it must have source.
903 if self.pkg.changes["architecture"].has_key("source"):
905 self.rejects.append("no source found and Architecture line in changes mention source.")
907 if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
908 self.rejects.append("source only uploads are not supported.")
910 ###########################################################################
911 def check_dsc(self, action=True, session=None):
912 """Returns bool indicating whether or not the source changes are valid"""
913 # Ensure there is source to check
914 if not self.pkg.changes["architecture"].has_key("source"):
919 for f, entry in self.pkg.files.items():
920 if entry["type"] == "dsc":
922 self.rejects.append("can not process a .changes file with multiple .dsc's.")
927 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
929 self.rejects.append("source uploads must contain a dsc file")
932 # Parse the .dsc file
934 self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
935 except CantOpenError:
936 # if not -n copy_to_holding() will have done this for us...
938 self.rejects.append("%s: can't read file." % (dsc_filename))
939 except ParseChangesError, line:
940 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
941 except InvalidDscError, line:
942 self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
943 except ChangesUnicodeError:
944 self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
946 # Build up the file list of files mentioned by the .dsc
948 self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
949 except NoFilesFieldError:
950 self.rejects.append("%s: no Files: field." % (dsc_filename))
952 except UnknownFormatError, format:
953 self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
955 except ParseChangesError, line:
956 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
959 # Enforce mandatory fields
960 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
961 if not self.pkg.dsc.has_key(i):
962 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
965 # Validate the source and version fields
966 if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
967 self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
968 if not re_valid_version.match(self.pkg.dsc["version"]):
969 self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
971 # Only a limited list of source formats are allowed in each suite
972 for dist in self.pkg.changes["distribution"].keys():
973 allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
974 if self.pkg.dsc["format"] not in allowed:
975 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
977 # Validate the Maintainer field
979 # We ignore the return value
980 fix_maintainer(self.pkg.dsc["maintainer"])
981 except ParseMaintError, msg:
982 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
983 % (dsc_filename, self.pkg.dsc["maintainer"], msg))
985 # Validate the build-depends field(s)
986 for field_name in [ "build-depends", "build-depends-indep" ]:
987 field = self.pkg.dsc.get(field_name)
989 # Have apt try to parse them...
991 apt_pkg.ParseSrcDepends(field)
993 self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
995 # Ensure the version number in the .dsc matches the version number in the .changes
996 epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
997 changes_version = self.pkg.files[dsc_filename]["version"]
999 if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1000 self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1002 # Ensure the Files field contain only what's expected
1003 self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
1005 # Ensure source is newer than existing source in target suites
1006 session = DBConn().session()
1007 self.check_source_against_db(dsc_filename, session)
1008 self.check_dsc_against_db(dsc_filename, session)
1013 ###########################################################################
1015 def get_changelog_versions(self, source_dir):
1016 """Extracts a the source package and (optionally) grabs the
1017 version history out of debian/changelog for the BTS."""
1021 # Find the .dsc (again)
1023 for f in self.pkg.files.keys():
1024 if self.pkg.files[f]["type"] == "dsc":
1027 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1028 if not dsc_filename:
1031 # Create a symlink mirror of the source files in our temporary directory
1032 for f in self.pkg.files.keys():
1033 m = re_issource.match(f)
1035 src = os.path.join(source_dir, f)
1036 # If a file is missing for whatever reason, give up.
1037 if not os.path.exists(src):
1040 if re_is_orig_source.match(f) and self.pkg.orig_files.has_key(f) and \
1041 self.pkg.orig_files[f].has_key("path"):
1043 dest = os.path.join(os.getcwd(), f)
1044 os.symlink(src, dest)
1046 # If the orig files are not a part of the upload, create symlinks to the
1048 for orig_file in self.pkg.orig_files.keys():
1049 if not self.pkg.orig_files[orig_file].has_key("path"):
1051 dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
1052 os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
1054 # Extract the source
1055 cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1056 (result, output) = commands.getstatusoutput(cmd)
1058 self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1059 self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "))
1062 if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1065 # Get the upstream version
1066 upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1067 if re_strip_revision.search(upstr_version):
1068 upstr_version = re_strip_revision.sub('', upstr_version)
1070 # Ensure the changelog file exists
1071 changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1072 if not os.path.exists(changelog_filename):
1073 self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1076 # Parse the changelog
1077 self.pkg.dsc["bts changelog"] = ""
1078 changelog_file = utils.open_file(changelog_filename)
1079 for line in changelog_file.readlines():
1080 m = re_changelog_versions.match(line)
1082 self.pkg.dsc["bts changelog"] += line
1083 changelog_file.close()
1085 # Check we found at least one revision in the changelog
1086 if not self.pkg.dsc["bts changelog"]:
1087 self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1089 def check_source(self):
1091 # a) there's no source
1092 # or c) the orig files are MIA
1093 if not self.pkg.changes["architecture"].has_key("source") \
1094 or len(self.pkg.orig_files) == 0:
1097 tmpdir = utils.temp_dirname()
1099 # Move into the temporary directory
1103 # Get the changelog version history
1104 self.get_changelog_versions(cwd)
1106 # Move back and cleanup the temporary tree
1110 shutil.rmtree(tmpdir)
1112 if e.errno != errno.EACCES:
1114 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1116 self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1117 # We probably have u-r or u-w directories so chmod everything
1119 cmd = "chmod -R u+rwx %s" % (tmpdir)
1120 result = os.system(cmd)
1122 utils.fubar("'%s' failed with result %s." % (cmd, result))
1123 shutil.rmtree(tmpdir)
1124 except Exception, e:
1125 print "foobar2 (%s)" % e
1126 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1128 ###########################################################################
1129 def ensure_hashes(self):
1130 # Make sure we recognise the format of the Files: field in the .changes
1131 format = self.pkg.changes.get("format", "0.0").split(".", 1)
1132 if len(format) == 2:
1133 format = int(format[0]), int(format[1])
1135 format = int(float(format[0])), 0
1137 # We need to deal with the original changes blob, as the fields we need
1138 # might not be in the changes dict serialised into the .dak anymore.
1139 orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1141 # Copy the checksums over to the current changes dict. This will keep
1142 # the existing modifications to it intact.
1143 for field in orig_changes:
1144 if field.startswith('checksums-'):
1145 self.pkg.changes[field] = orig_changes[field]
1147 # Check for unsupported hashes
1148 for j in utils.check_hash_fields(".changes", self.pkg.changes):
1149 self.rejects.append(j)
1151 for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1152 self.rejects.append(j)
1154 # We have to calculate the hash if we have an earlier changes version than
1155 # the hash appears in rather than require it exist in the changes file
1156 for hashname, hashfunc, version in utils.known_hashes:
1157 # TODO: Move _ensure_changes_hash into this class
1158 for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1159 self.rejects.append(j)
1160 if "source" in self.pkg.changes["architecture"]:
1161 # TODO: Move _ensure_dsc_hash into this class
1162 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1163 self.rejects.append(j)
1165 def check_hashes(self):
1166 for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1167 self.rejects.append(m)
1169 for m in utils.check_size(".changes", self.pkg.files):
1170 self.rejects.append(m)
1172 for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1173 self.rejects.append(m)
1175 for m in utils.check_size(".dsc", self.pkg.dsc_files):
1176 self.rejects.append(m)
1178 self.ensure_hashes()
1180 ###########################################################################
1182 def ensure_orig(self, target_dir='.', session=None):
1184 Ensures that all orig files mentioned in the changes file are present
1185 in target_dir. If they do not exist, they are symlinked into place.
1187 An list containing the symlinks that were created are returned (so they
1194 for filename, entry in self.pkg.dsc_files.iteritems():
1195 if not re_is_orig_source.match(filename):
1196 # File is not an orig; ignore
1199 if os.path.exists(filename):
1200 # File exists, no need to continue
1203 def symlink_if_valid(path):
1204 f = utils.open_file(path)
1205 md5sum = apt_pkg.md5sum(f)
1208 fingerprint = (os.stat(path)[stat.ST_SIZE], md5sum)
1209 expected = (int(entry['size']), entry['md5sum'])
1211 if fingerprint != expected:
1214 dest = os.path.join(target_dir, filename)
1216 os.symlink(path, dest)
1217 symlinked.append(dest)
1223 session_ = DBConn().session()
1228 for poolfile in get_poolfile_like_name('/%s' % filename, session_):
1229 poolfile_path = os.path.join(
1230 poolfile.location.path, poolfile.filename
1233 if symlink_if_valid(poolfile_path):
1243 # Look in some other queues for the file
1244 queues = ('New', 'Byhand', 'ProposedUpdates',
1245 'OldProposedUpdates', 'Embargoed', 'Unembargoed')
1247 for queue in queues:
1248 if not cnf.get('Dir::Queue::%s' % queue):
1251 queuefile_path = os.path.join(
1252 cnf['Dir::Queue::%s' % queue], filename
1255 if not os.path.exists(queuefile_path):
1256 # Does not exist in this queue
1259 if symlink_if_valid(queuefile_path):
1264 ###########################################################################
1266 def check_lintian(self):
1268 Extends self.rejects by checking the output of lintian against tags
1269 specified in Dinstall::LintianTags.
1274 # Don't reject binary uploads
1275 if not self.pkg.changes['architecture'].has_key('source'):
1278 # Only check some distributions
1279 for dist in ('unstable', 'experimental'):
1280 if dist in self.pkg.changes['distribution']:
1285 # If we do not have a tagfile, don't do anything
1286 tagfile = cnf.get("Dinstall::LintianTags")
1290 # Parse the yaml file
1291 sourcefile = file(tagfile, 'r')
1292 sourcecontent = sourcefile.read()
1296 lintiantags = yaml.load(sourcecontent)['lintian']
1297 except yaml.YAMLError, msg:
1298 utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
1301 # Try and find all orig mentioned in the .dsc
1302 symlinked = self.ensure_orig()
1304 # Setup the input file for lintian
1305 fd, temp_filename = utils.temp_filename()
1306 temptagfile = os.fdopen(fd, 'w')
1307 for tags in lintiantags.values():
1308 temptagfile.writelines(['%s\n' % x for x in tags])
1312 cmd = "lintian --show-overrides --tags-from-file %s %s" % \
1313 (temp_filename, self.pkg.changes_file)
1315 result, output = commands.getstatusoutput(cmd)
1317 # Remove our tempfile and any symlinks we created
1318 os.unlink(temp_filename)
1320 for symlink in symlinked:
1324 utils.warn("lintian failed for %s [return code: %s]." % \
1325 (self.pkg.changes_file, result))
1326 utils.warn(utils.prefix_multi_line_string(output, \
1327 " [possible output:] "))
1332 [self.pkg.changes_file, "check_lintian"] + list(txt)
1336 parsed_tags = parse_lintian_output(output)
1337 self.rejects.extend(
1338 generate_reject_messages(parsed_tags, lintiantags, log=log)
1341 ###########################################################################
1342 def check_urgency(self):
1344 if self.pkg.changes["architecture"].has_key("source"):
1345 if not self.pkg.changes.has_key("urgency"):
1346 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1347 self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1348 if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1349 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1350 (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1351 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1353 ###########################################################################
1355 # Sanity check the time stamps of files inside debs.
1356 # [Files in the near future cause ugly warnings and extreme time
1357 # travel can cause errors on extraction]
1359 def check_timestamps(self):
1362 future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1363 past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1364 tar = TarTime(future_cutoff, past_cutoff)
1366 for filename, entry in self.pkg.files.items():
1367 if entry["type"] == "deb":
1370 deb_file = utils.open_file(filename)
1371 apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1374 apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1375 except SystemError, e:
1376 # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1377 if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1380 apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1384 future_files = tar.future_files.keys()
1386 num_future_files = len(future_files)
1387 future_file = future_files[0]
1388 future_date = tar.future_files[future_file]
1389 self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1390 % (filename, num_future_files, future_file, time.ctime(future_date)))
1392 ancient_files = tar.ancient_files.keys()
1394 num_ancient_files = len(ancient_files)
1395 ancient_file = ancient_files[0]
1396 ancient_date = tar.ancient_files[ancient_file]
1397 self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1398 % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1400 self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1402 def check_if_upload_is_sponsored(self, uid_email, uid_name):
1403 if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1405 elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1411 if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
1412 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1413 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1414 self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1415 self.pkg.changes["sponsoremail"] = uid_email
1420 ###########################################################################
1421 # check_signed_by_key checks
1422 ###########################################################################
1424 def check_signed_by_key(self):
1425 """Ensure the .changes is signed by an authorized uploader."""
1426 session = DBConn().session()
1428 # First of all we check that the person has proper upload permissions
1429 # and that this upload isn't blocked
1430 fpr = get_fingerprint(self.pkg.changes['fingerprint'], session=session)
1433 self.rejects.append("Cannot find fingerprint %s" % self.pkg.changes["fingerprint"])
1436 # TODO: Check that import-keyring adds UIDs properly
1438 self.rejects.append("Cannot find uid for fingerprint %s. Please contact ftpmaster@debian.org" % fpr.fingerprint)
1441 # Check that the fingerprint which uploaded has permission to do so
1442 self.check_upload_permissions(fpr, session)
1444 # Check that this package is not in a transition
1445 self.check_transition(session)
1450 def check_upload_permissions(self, fpr, session):
1451 # Check any one-off upload blocks
1452 self.check_upload_blocks(fpr, session)
1454 # Start with DM as a special case
1455 # DM is a special case unfortunately, so we check it first
1456 # (keys with no source access get more access than DMs in one
1457 # way; DMs can only upload for their packages whether source
1458 # or binary, whereas keys with no access might be able to
1459 # upload some binaries)
1460 if fpr.source_acl.access_level == 'dm':
1461 self.check_dm_upload(fpr, session)
1463 # Check source-based permissions for other types
1464 if self.pkg.changes["architecture"].has_key("source") and \
1465 fpr.source_acl.access_level is None:
1466 rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
1467 rej += '\nPlease contact ftpmaster if you think this is incorrect'
1468 self.rejects.append(rej)
1470 # If not a DM, we allow full upload rights
1471 uid_email = "%s@debian.org" % (fpr.uid.uid)
1472 self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
1475 # Check binary upload permissions
1476 # By this point we know that DMs can't have got here unless they
1477 # are allowed to deal with the package concerned so just apply
1479 if fpr.binary_acl.access_level == 'full':
1482 # Otherwise we're in the map case
1483 tmparches = self.pkg.changes["architecture"].copy()
1484 tmparches.pop('source', None)
1486 for bam in fpr.binary_acl_map:
1487 tmparches.pop(bam.architecture.arch_string, None)
1489 if len(tmparches.keys()) > 0:
1490 if fpr.binary_reject:
1491 rej = ".changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
1492 rej += "\narchitectures involved are: ", ",".join(tmparches.keys())
1493 self.rejects.append(rej)
1495 # TODO: This is where we'll implement reject vs throw away binaries later
1496 rej = "Uhm. I'm meant to throw away the binaries now but that's not implemented yet"
1497 rej += "\nPlease complain to ftpmaster@debian.org as this shouldn't have been turned on"
1498 rej += "\nFingerprint: %s", (fpr.fingerprint)
1499 self.rejects.append(rej)
1502 def check_upload_blocks(self, fpr, session):
1503 """Check whether any upload blocks apply to this source, source
1504 version, uid / fpr combination"""
1506 def block_rej_template(fb):
1507 rej = 'Manual upload block in place for package %s' % fb.source
1508 if fb.version is not None:
1509 rej += ', version %s' % fb.version
1512 for fb in session.query(UploadBlock).filter_by(source = self.pkg.changes['source']).all():
1513 # version is None if the block applies to all versions
1514 if fb.version is None or fb.version == self.pkg.changes['version']:
1515 # Check both fpr and uid - either is enough to cause a reject
1516 if fb.fpr is not None:
1517 if fb.fpr.fingerprint == fpr.fingerprint:
1518 self.rejects.append(block_rej_template(fb) + ' for fingerprint %s\nReason: %s' % (fpr.fingerprint, fb.reason))
1519 if fb.uid is not None:
1520 if fb.uid == fpr.uid:
1521 self.rejects.append(block_rej_template(fb) + ' for uid %s\nReason: %s' % (fb.uid.uid, fb.reason))
1524 def check_dm_upload(self, fpr, session):
1525 # Quoth the GR (http://www.debian.org/vote/2007/vote_003):
1526 ## none of the uploaded packages are NEW
1528 for f in self.pkg.files.keys():
1529 if self.pkg.files[f].has_key("byhand"):
1530 self.rejects.append("%s may not upload BYHAND file %s" % (fpr.uid.uid, f))
1532 if self.pkg.files[f].has_key("new"):
1533 self.rejects.append("%s may not upload NEW file %s" % (fpr.uid.uid, f))
1539 ## the most recent version of the package uploaded to unstable or
1540 ## experimental includes the field "DM-Upload-Allowed: yes" in the source
1541 ## section of its control file
1542 q = session.query(DBSource).filter_by(source=self.pkg.changes["source"])
1543 q = q.join(SrcAssociation)
1544 q = q.join(Suite).filter(Suite.suite_name.in_(['unstable', 'experimental']))
1545 q = q.order_by(desc('source.version')).limit(1)
1550 rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"]
1551 self.rejects.append(rej)
1555 if not r.dm_upload_allowed:
1556 rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version)
1557 self.rejects.append(rej)
1560 ## the Maintainer: field of the uploaded .changes file corresponds with
1561 ## the owner of the key used (ie, non-developer maintainers may not sponsor
1563 if self.check_if_upload_is_sponsored(fpr.uid.uid, fpr.uid.name):
1564 self.rejects.append("%s (%s) is not authorised to sponsor uploads" % (fpr.uid.uid, fpr.fingerprint))
1566 ## the most recent version of the package uploaded to unstable or
1567 ## experimental lists the uploader in the Maintainer: or Uploaders: fields (ie,
1568 ## non-developer maintainers cannot NMU or hijack packages)
1570 # srcuploaders includes the maintainer
1572 for sup in r.srcuploaders:
1573 (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1574 # Eww - I hope we never have two people with the same name in Debian
1575 if email == fpr.uid.uid or name == fpr.uid.name:
1580 self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (fpr.uid.uid, self.pkg.changes["source"]))
1583 ## none of the packages are being taken over from other source packages
1584 for b in self.pkg.changes["binary"].keys():
1585 for suite in self.pkg.changes["distribution"].keys():
1586 q = session.query(DBSource)
1587 q = q.join(DBBinary).filter_by(package=b)
1588 q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1591 if s.source != self.pkg.changes["source"]:
1592 self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite))
1596 def check_transition(self, session):
1599 sourcepkg = self.pkg.changes["source"]
1601 # No sourceful upload -> no need to do anything else, direct return
1602 # We also work with unstable uploads, not experimental or those going to some
1603 # proposed-updates queue
1604 if "source" not in self.pkg.changes["architecture"] or \
1605 "unstable" not in self.pkg.changes["distribution"]:
1608 # Also only check if there is a file defined (and existant) with
1610 transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1611 if transpath == "" or not os.path.exists(transpath):
1614 # Parse the yaml file
1615 sourcefile = file(transpath, 'r')
1616 sourcecontent = sourcefile.read()
1618 transitions = yaml.load(sourcecontent)
1619 except yaml.YAMLError, msg:
1620 # This shouldn't happen, there is a wrapper to edit the file which
1621 # checks it, but we prefer to be safe than ending up rejecting
1623 utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1626 # Now look through all defined transitions
1627 for trans in transitions:
1628 t = transitions[trans]
1629 source = t["source"]
1632 # Will be None if nothing is in testing.
1633 current = get_source_in_suite(source, "testing", session)
1634 if current is not None:
1635 compare = apt_pkg.VersionCompare(current.version, expected)
1637 if current is None or compare < 0:
1638 # This is still valid, the current version in testing is older than
1639 # the new version we wait for, or there is none in testing yet
1641 # Check if the source we look at is affected by this.
1642 if sourcepkg in t['packages']:
1643 # The source is affected, lets reject it.
1645 rejectmsg = "%s: part of the %s transition.\n\n" % (
1648 if current is not None:
1649 currentlymsg = "at version %s" % (current.version)
1651 currentlymsg = "not present in testing"
1653 rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1655 rejectmsg += "\n".join(textwrap.wrap("""Your package
1656 is part of a testing transition designed to get %s migrated (it is
1657 currently %s, we need version %s). This transition is managed by the
1658 Release Team, and %s is the Release-Team member responsible for it.
1659 Please mail debian-release@lists.debian.org or contact %s directly if you
1660 need further assistance. You might want to upload to experimental until this
1661 transition is done."""
1662 % (source, currentlymsg, expected,t["rm"], t["rm"])))
1664 self.rejects.append(rejectmsg)
1667 ###########################################################################
1668 # End check_signed_by_key checks
1669 ###########################################################################
1671 def build_summaries(self):
1672 """ Build a summary of changes the upload introduces. """
1674 (byhand, new, summary, override_summary) = self.pkg.file_summary()
1676 short_summary = summary
1678 # This is for direport's benefit...
1679 f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1682 summary += "Changes: " + f
1684 summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1686 summary += self.announce(short_summary, 0)
1688 return (summary, short_summary)
1690 ###########################################################################
1692 def close_bugs(self, summary, action):
1694 Send mail to close bugs as instructed by the closes field in the changes file.
1695 Also add a line to summary if any work was done.
1697 @type summary: string
1698 @param summary: summary text, as given by L{build_summaries}
1701 @param action: Set to false no real action will be done.
1704 @return: summary. If action was taken, extended by the list of closed bugs.
1708 template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1710 bugs = self.pkg.changes["closes"].keys()
1716 summary += "Closing bugs: "
1718 summary += "%s " % (bug)
1721 self.Subst["__BUG_NUMBER__"] = bug
1722 if self.pkg.changes["distribution"].has_key("stable"):
1723 self.Subst["__STABLE_WARNING__"] = """
1724 Note that this package is not part of the released stable Debian
1725 distribution. It may have dependencies on other unreleased software,
1726 or other instabilities. Please take care if you wish to install it.
1727 The update will eventually make its way into the next released Debian
1730 self.Subst["__STABLE_WARNING__"] = ""
1731 mail_message = utils.TemplateSubst(self.Subst, template)
1732 utils.send_mail(mail_message)
1734 # Clear up after ourselves
1735 del self.Subst["__BUG_NUMBER__"]
1736 del self.Subst["__STABLE_WARNING__"]
1738 if action and self.logger:
1739 self.logger.log(["closing bugs"] + bugs)
1745 ###########################################################################
1747 def announce(self, short_summary, action):
1749 Send an announce mail about a new upload.
1751 @type short_summary: string
1752 @param short_summary: Short summary text to include in the mail
1755 @param action: Set to false no real action will be done.
1758 @return: Textstring about action taken.
1763 announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1765 # Only do announcements for source uploads with a recent dpkg-dev installed
1766 if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1767 self.pkg.changes["architecture"].has_key("source"):
1773 self.Subst["__SHORT_SUMMARY__"] = short_summary
1775 for dist in self.pkg.changes["distribution"].keys():
1776 announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1777 if announce_list == "" or lists_done.has_key(announce_list):
1780 lists_done[announce_list] = 1
1781 summary += "Announcing to %s\n" % (announce_list)
1785 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1786 if cnf.get("Dinstall::TrackingServer") and \
1787 self.pkg.changes["architecture"].has_key("source"):
1788 trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1789 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1791 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1792 utils.send_mail(mail_message)
1794 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1796 if cnf.FindB("Dinstall::CloseBugs"):
1797 summary = self.close_bugs(summary, action)
1799 del self.Subst["__SHORT_SUMMARY__"]
1803 ###########################################################################
1805 def accept (self, summary, short_summary, session=None):
1809 This moves all files referenced from the .changes into the pool,
1810 sends the accepted mail, announces to lists, closes bugs and
1811 also checks for override disparities. If enabled it will write out
1812 the version history for the BTS Version Tracking and will finally call
1815 @type summary: string
1816 @param summary: Summary text
1818 @type short_summary: string
1819 @param short_summary: Short summary
1823 stats = SummaryStats()
1826 self.logger.log(["installing changes", self.pkg.changes_file])
1830 # Add the .dsc file to the DB first
1831 for newfile, entry in self.pkg.files.items():
1832 if entry["type"] == "dsc":
1833 dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
1837 # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
1838 for newfile, entry in self.pkg.files.items():
1839 if entry["type"] == "deb":
1840 poolfiles.append(add_deb_to_db(self, newfile, session))
1842 # If this is a sourceful diff only upload that is moving
1843 # cross-component we need to copy the .orig files into the new
1844 # component too for the same reasons as above.
1845 if self.pkg.changes["architecture"].has_key("source"):
1846 for orig_file in self.pkg.orig_files.keys():
1847 if not self.pkg.orig_files[orig_file].has_key("id"):
1848 continue # Skip if it's not in the pool
1849 orig_file_id = self.pkg.orig_files[orig_file]["id"]
1850 if self.pkg.orig_files[orig_file]["location"] == dsc_location_id:
1851 continue # Skip if the location didn't change
1854 oldf = get_poolfile_by_id(orig_file_id, session)
1855 old_filename = os.path.join(oldf.location.path, oldf.filename)
1856 old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
1857 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
1859 new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
1861 # TODO: Care about size/md5sum collisions etc
1862 (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
1865 utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
1866 newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
1868 # TODO: Check that there's only 1 here
1869 source = get_sources_from_name(self.pkg.changes["source"], self.pkg.changes["version"])[0]
1870 dscf = get_dscfiles(source_id=source.source_id, poolfile_id=orig_file_id, session=session)[0]
1871 dscf.poolfile_id = newf.file_id
1875 poolfiles.append(newf)
1877 # Install the files into the pool
1878 for newfile, entry in self.pkg.files.items():
1879 destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
1880 utils.move(newfile, destination)
1881 self.logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
1882 stats.accept_bytes += float(entry["size"])
1884 # Copy the .changes file across for suite which need it.
1886 for suite_name in self.pkg.changes["distribution"].keys():
1887 if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
1888 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
1890 for dest in copy_changes.keys():
1891 utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
1893 # We're done - commit the database changes
1895 # Our SQL session will automatically start a new transaction after
1898 # Move the .changes into the 'done' directory
1899 utils.move(self.pkg.changes_file,
1900 os.path.join(cnf["Dir::Queue::Done"], os.path.basename(self.pkg.changes_file)))
1902 if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"):
1903 UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])
1905 # Send accept mail, announce to lists, close bugs and check for
1906 # override disparities
1907 if not cnf["Dinstall::Options::No-Mail"]:
1909 self.Subst["__SUITE__"] = ""
1910 self.Subst["__SUMMARY__"] = summary
1911 mail_message = utils.TemplateSubst(self.Subst,
1912 os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted'))
1913 utils.send_mail(mail_message)
1914 self.announce(short_summary, 1)
1916 ## Helper stuff for DebBugs Version Tracking
1917 if cnf.Find("Dir::Queue::BTSVersionTrack"):
1918 # ??? once queue/* is cleared on *.d.o and/or reprocessed
1919 # the conditionalization on dsc["bts changelog"] should be
1922 # Write out the version history from the changelog
1923 if self.pkg.changes["architecture"].has_key("source") and \
1924 self.pkg.dsc.has_key("bts changelog"):
1926 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1927 version_history = os.fdopen(fd, 'w')
1928 version_history.write(self.pkg.dsc["bts changelog"])
1929 version_history.close()
1930 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1931 self.pkg.changes_file[:-8]+".versions")
1932 os.rename(temp_filename, filename)
1933 os.chmod(filename, 0644)
1935 # Write out the binary -> source mapping.
1936 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1937 debinfo = os.fdopen(fd, 'w')
1938 for name, entry in sorted(self.pkg.files.items()):
1939 if entry["type"] == "deb":
1940 line = " ".join([entry["package"], entry["version"],
1941 entry["architecture"], entry["source package"],
1942 entry["source version"]])
1943 debinfo.write(line+"\n")
1945 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1946 self.pkg.changes_file[:-8]+".debinfo")
1947 os.rename(temp_filename, filename)
1948 os.chmod(filename, 0644)
1952 # Set up our copy queues (e.g. buildd queues)
1953 for suite_name in self.pkg.changes["distribution"].keys():
1954 suite = get_suite(suite_name, session)
1955 for q in suite.copy_queues:
1957 q.add_file_from_pool(f)
1962 stats.accept_count += 1
1964 def check_override(self):
1966 Checks override entries for validity. Mails "Override disparity" warnings,
1967 if that feature is enabled.
1969 Abandons the check if
1970 - override disparity checks are disabled
1971 - mail sending is disabled
1976 # Abandon the check if:
1977 # a) override disparity checks have been disabled
1978 # b) we're not sending mail
1979 if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
1980 cnf["Dinstall::Options::No-Mail"]:
1983 summary = self.pkg.check_override()
1988 overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
1991 self.Subst["__SUMMARY__"] = summary
1992 mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
1993 utils.send_mail(mail_message)
1994 del self.Subst["__SUMMARY__"]
1996 ###########################################################################
1998 def remove(self, from_dir=None):
2000 Used (for instance) in p-u to remove the package from unchecked
2002 Also removes the package from holding area.
2004 if from_dir is None:
2005 from_dir = self.pkg.directory
2008 for f in self.pkg.files.keys():
2009 os.unlink(os.path.join(from_dir, f))
2010 if os.path.exists(os.path.join(h.holding_dir, f)):
2011 os.unlink(os.path.join(h.holding_dir, f))
2013 os.unlink(os.path.join(from_dir, self.pkg.changes_file))
2014 if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
2015 os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
2017 ###########################################################################
2019 def move_to_queue (self, queue):
2021 Move files to a destination queue using the permissions in the table
2024 utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
2025 queue.path, perms=int(queue.change_perms, 8))
2026 for f in self.pkg.files.keys():
2027 utils.move(os.path.join(h.holding_dir, f), queue.path, perms=int(queue.perms, 8))
2029 ###########################################################################
2031 def force_reject(self, reject_files):
2033 Forcefully move files from the current directory to the
2034 reject directory. If any file already exists in the reject
2035 directory it will be moved to the morgue to make way for
2039 @param files: file dictionary
2045 for file_entry in reject_files:
2046 # Skip any files which don't exist or which we don't have permission to copy.
2047 if os.access(file_entry, os.R_OK) == 0:
2050 dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
2053 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
2055 # File exists? Let's try and move it to the morgue
2056 if e.errno == errno.EEXIST:
2057 morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
2059 morgue_file = utils.find_next_free(morgue_file)
2060 except NoFreeFilenameError:
2061 # Something's either gone badly Pete Tong, or
2062 # someone is trying to exploit us.
2063 utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
2065 utils.move(dest_file, morgue_file, perms=0660)
2067 dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2070 utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
2074 # If we got here, we own the destination file, so we can
2075 # safely overwrite it.
2076 utils.move(file_entry, dest_file, 1, perms=0660)
2079 ###########################################################################
2080 def do_reject (self, manual=0, reject_message="", notes=""):
2082 Reject an upload. If called without a reject message or C{manual} is
2083 true, spawn an editor so the user can write one.
2086 @param manual: manual or automated rejection
2088 @type reject_message: string
2089 @param reject_message: A reject message
2094 # If we weren't given a manual rejection message, spawn an
2095 # editor so the user can add one in...
2096 if manual and not reject_message:
2097 (fd, temp_filename) = utils.temp_filename()
2098 temp_file = os.fdopen(fd, 'w')
2101 temp_file.write("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
2102 % (note.author, note.version, note.notedate, note.comment))
2104 editor = os.environ.get("EDITOR","vi")
2106 while answer == 'E':
2107 os.system("%s %s" % (editor, temp_filename))
2108 temp_fh = utils.open_file(temp_filename)
2109 reject_message = "".join(temp_fh.readlines())
2111 print "Reject message:"
2112 print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1)
2113 prompt = "[R]eject, Edit, Abandon, Quit ?"
2115 while prompt.find(answer) == -1:
2116 answer = utils.our_raw_input(prompt)
2117 m = re_default_answer.search(prompt)
2120 answer = answer[:1].upper()
2121 os.unlink(temp_filename)
2127 print "Rejecting.\n"
2131 reason_filename = self.pkg.changes_file[:-8] + ".reason"
2132 reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2134 # Move all the files into the reject directory
2135 reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
2136 self.force_reject(reject_files)
2138 # If we fail here someone is probably trying to exploit the race
2139 # so let's just raise an exception ...
2140 if os.path.exists(reason_filename):
2141 os.unlink(reason_filename)
2142 reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2144 rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
2148 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2149 self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
2150 self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)"
2151 os.write(reason_fd, reject_message)
2152 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2154 # Build up the rejection email
2155 user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
2156 self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
2157 self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
2158 self.Subst["__REJECT_MESSAGE__"] = ""
2159 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2160 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
2161 # Write the rejection email out as the <foo>.reason file
2162 os.write(reason_fd, reject_mail_message)
2164 del self.Subst["__REJECTOR_ADDRESS__"]
2165 del self.Subst["__MANUAL_REJECT_MESSAGE__"]
2166 del self.Subst["__CC__"]
2170 # Send the rejection mail if appropriate
2171 if not cnf["Dinstall::Options::No-Mail"]:
2172 utils.send_mail(reject_mail_message)
2175 self.logger.log(["rejected", self.pkg.changes_file])
2179 ################################################################################
2180 def in_override_p(self, package, component, suite, binary_type, filename, session):
2182 Check if a package already has override entries in the DB
2184 @type package: string
2185 @param package: package name
2187 @type component: string
2188 @param component: database id of the component
2191 @param suite: database id of the suite
2193 @type binary_type: string
2194 @param binary_type: type of the package
2196 @type filename: string
2197 @param filename: filename we check
2199 @return: the database result. But noone cares anyway.
2205 if binary_type == "": # must be source
2208 file_type = binary_type
2210 # Override suite name; used for example with proposed-updates
2211 if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
2212 suite = cnf["Suite::%s::OverrideSuite" % (suite)]
2214 result = get_override(package, suite, component, file_type, session)
2216 # If checking for a source package fall back on the binary override type
2217 if file_type == "dsc" and len(result) < 1:
2218 result = get_override(package, suite, component, ['deb', 'udeb'], session)
2220 # Remember the section and priority so we can check them later if appropriate
2223 self.pkg.files[filename]["override section"] = result.section.section
2224 self.pkg.files[filename]["override priority"] = result.priority.priority
2229 ################################################################################
2230 def get_anyversion(self, sv_list, suite):
2233 @param sv_list: list of (suite, version) tuples to check
2236 @param suite: suite name
2242 anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
2243 for (s, v) in sv_list:
2244 if s in [ x.lower() for x in anysuite ]:
2245 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
2250 ################################################################################
2252 def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
2255 @param sv_list: list of (suite, version) tuples to check
2257 @type filename: string
2258 @param filename: XXX
2260 @type new_version: string
2261 @param new_version: XXX
2263 Ensure versions are newer than existing packages in target
2264 suites and that cross-suite version checking rules as
2265 set out in the conf file are satisfied.
2270 # Check versions for each target suite
2271 for target_suite in self.pkg.changes["distribution"].keys():
2272 must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
2273 must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
2275 # Enforce "must be newer than target suite" even if conffile omits it
2276 if target_suite not in must_be_newer_than:
2277 must_be_newer_than.append(target_suite)
2279 for (suite, existent_version) in sv_list:
2280 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
2282 if suite in must_be_newer_than and sourceful and vercmp < 1:
2283 self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2285 if suite in must_be_older_than and vercmp > -1:
2288 if self.pkg.changes.get('distribution-version', {}).has_key(suite):
2289 # we really use the other suite, ignoring the conflicting one ...
2290 addsuite = self.pkg.changes["distribution-version"][suite]
2292 add_version = self.get_anyversion(sv_list, addsuite)
2293 target_version = self.get_anyversion(sv_list, target_suite)
2296 # not add_version can only happen if we map to a suite
2297 # that doesn't enhance the suite we're propup'ing from.
2298 # so "propup-ver x a b c; map a d" is a problem only if
2299 # d doesn't enhance a.
2301 # i think we could always propagate in this case, rather
2302 # than complaining. either way, this isn't a REJECT issue
2304 # And - we really should complain to the dorks who configured dak
2305 self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
2306 self.pkg.changes.setdefault("propdistribution", {})
2307 self.pkg.changes["propdistribution"][addsuite] = 1
2309 elif not target_version:
2310 # not targets_version is true when the package is NEW
2311 # we could just stick with the "...old version..." REJECT
2312 # for this, I think.
2313 self.rejects.append("Won't propogate NEW packages.")
2314 elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2315 # propogation would be redundant. no need to reject though.
2316 self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2318 elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2319 apt_pkg.VersionCompare(add_version, target_version) >= 0:
2321 self.warnings.append("Propogating upload to %s" % (addsuite))
2322 self.pkg.changes.setdefault("propdistribution", {})
2323 self.pkg.changes["propdistribution"][addsuite] = 1
2327 self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
2329 ################################################################################
2330 def check_binary_against_db(self, filename, session):
2331 # Ensure version is sane
2332 q = session.query(BinAssociation)
2333 q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[filename]["package"])
2334 q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[filename]["architecture"], 'all']))
2336 self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2337 filename, self.pkg.files[filename]["version"], sourceful=False)
2339 # Check for any existing copies of the file
2340 q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"])
2341 q = q.filter_by(version=self.pkg.files[filename]["version"])
2342 q = q.join(Architecture).filter_by(arch_string=self.pkg.files[filename]["architecture"])
2345 self.rejects.append("%s: can not overwrite existing copy already in the archive." % filename)
2347 ################################################################################
2349 def check_source_against_db(self, filename, session):
2350 source = self.pkg.dsc.get("source")
2351 version = self.pkg.dsc.get("version")
2353 # Ensure version is sane
2354 q = session.query(SrcAssociation)
2355 q = q.join(DBSource).filter(DBSource.source==source)
2357 self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2358 filename, version, sourceful=True)
2360 ################################################################################
2361 def check_dsc_against_db(self, filename, session):
2364 @warning: NB: this function can remove entries from the 'files' index [if
2365 the orig tarball is a duplicate of the one in the archive]; if
2366 you're iterating over 'files' and call this function as part of
2367 the loop, be sure to add a check to the top of the loop to
2368 ensure you haven't just tried to dereference the deleted entry.
2373 self.pkg.orig_files = {} # XXX: do we need to clear it?
2374 orig_files = self.pkg.orig_files
2376 # Try and find all files mentioned in the .dsc. This has
2377 # to work harder to cope with the multiple possible
2378 # locations of an .orig.tar.gz.
2379 # The ordering on the select is needed to pick the newest orig
2380 # when it exists in multiple places.
2381 for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2383 if self.pkg.files.has_key(dsc_name):
2384 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2385 actual_size = int(self.pkg.files[dsc_name]["size"])
2386 found = "%s in incoming" % (dsc_name)
2388 # Check the file does not already exist in the archive
2389 ql = get_poolfile_like_name(dsc_name, session)
2391 # Strip out anything that isn't '%s' or '/%s$'
2393 if not i.filename.endswith(dsc_name):
2396 # "[dak] has not broken them. [dak] has fixed a
2397 # brokenness. Your crappy hack exploited a bug in
2400 # "(Come on! I thought it was always obvious that
2401 # one just doesn't release different files with
2402 # the same name and version.)"
2403 # -- ajk@ on d-devel@l.d.o
2406 # Ignore exact matches for .orig.tar.gz
2408 if re_is_orig_source.match(dsc_name):
2410 if self.pkg.files.has_key(dsc_name) and \
2411 int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2412 self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2413 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2414 # TODO: Don't delete the entry, just mark it as not needed
2415 # This would fix the stupidity of changing something we often iterate over
2416 # whilst we're doing it
2417 del self.pkg.files[dsc_name]
2418 dsc_entry["files id"] = i.file_id
2419 if not orig_files.has_key(dsc_name):
2420 orig_files[dsc_name] = {}
2421 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
2425 self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2427 elif re_is_orig_source.match(dsc_name):
2429 ql = get_poolfile_like_name(dsc_name, session)
2431 # Strip out anything that isn't '%s' or '/%s$'
2432 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2434 if not i.filename.endswith(dsc_name):
2438 # Unfortunately, we may get more than one match here if,
2439 # for example, the package was in potato but had an -sa
2440 # upload in woody. So we need to choose the right one.
2442 # default to something sane in case we don't match any or have only one
2447 old_file = os.path.join(i.location.path, i.filename)
2448 old_file_fh = utils.open_file(old_file)
2449 actual_md5 = apt_pkg.md5sum(old_file_fh)
2451 actual_size = os.stat(old_file)[stat.ST_SIZE]
2452 if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2455 old_file = os.path.join(i.location.path, i.filename)
2456 old_file_fh = utils.open_file(old_file)
2457 actual_md5 = apt_pkg.md5sum(old_file_fh)
2459 actual_size = os.stat(old_file)[stat.ST_SIZE]
2461 suite_type = x.location.archive_type
2462 # need this for updating dsc_files in install()
2463 dsc_entry["files id"] = x.file_id
2464 # See install() in process-accepted...
2465 if not orig_files.has_key(dsc_name):
2466 orig_files[dsc_name] = {}
2467 orig_files[dsc_name]["id"] = x.file_id
2468 orig_files[dsc_name]["path"] = old_file
2469 orig_files[dsc_name]["location"] = x.location.location_id
2471 # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2472 # Not there? Check the queue directories...
2473 for directory in [ "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2474 if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2476 in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2477 if os.path.exists(in_otherdir):
2478 in_otherdir_fh = utils.open_file(in_otherdir)
2479 actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2480 in_otherdir_fh.close()
2481 actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2483 if not orig_files.has_key(dsc_name):
2484 orig_files[dsc_name] = {}
2485 orig_files[dsc_name]["path"] = in_otherdir
2488 self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (filename, dsc_name))
2491 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (filename, dsc_name))
2493 if actual_md5 != dsc_entry["md5sum"]:
2494 self.rejects.append("md5sum for %s doesn't match %s." % (found, filename))
2495 if actual_size != int(dsc_entry["size"]):
2496 self.rejects.append("size for %s doesn't match %s." % (found, filename))
2498 ################################################################################
2499 # This is used by process-new and process-holding to recheck a changes file
2500 # at the time we're running. It mainly wraps various other internal functions
2501 # and is similar to accepted_checks - these should probably be tidied up
2503 def recheck(self, session):
2505 for f in self.pkg.files.keys():
2506 # The .orig.tar.gz can disappear out from under us is it's a
2507 # duplicate of one in the archive.
2508 if not self.pkg.files.has_key(f):
2511 entry = self.pkg.files[f]
2513 # Check that the source still exists
2514 if entry["type"] == "deb":
2515 source_version = entry["source version"]
2516 source_package = entry["source package"]
2517 if not self.pkg.changes["architecture"].has_key("source") \
2518 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
2519 source_epochless_version = re_no_epoch.sub('', source_version)
2520 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
2522 for q in ["Embargoed", "Unembargoed", "Newstage"]:
2523 if cnf.has_key("Dir::Queue::%s" % (q)):
2524 if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
2527 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
2529 # Version and file overwrite checks
2530 if entry["type"] == "deb":
2531 self.check_binary_against_db(f, session)
2532 elif entry["type"] == "dsc":
2533 self.check_source_against_db(f, session)
2534 self.check_dsc_against_db(f, session)
2536 ################################################################################
2537 def accepted_checks(self, overwrite_checks, session):
2538 # Recheck anything that relies on the database; since that's not
2539 # frozen between accept and our run time when called from p-a.
2541 # overwrite_checks is set to False when installing to stable/oldstable
2546 # Find the .dsc (again)
2548 for f in self.pkg.files.keys():
2549 if self.pkg.files[f]["type"] == "dsc":
2552 for checkfile in self.pkg.files.keys():
2553 # The .orig.tar.gz can disappear out from under us is it's a
2554 # duplicate of one in the archive.
2555 if not self.pkg.files.has_key(checkfile):
2558 entry = self.pkg.files[checkfile]
2560 # Check that the source still exists
2561 if entry["type"] == "deb":
2562 source_version = entry["source version"]
2563 source_package = entry["source package"]
2564 if not self.pkg.changes["architecture"].has_key("source") \
2565 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
2566 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2568 # Version and file overwrite checks
2569 if overwrite_checks:
2570 if entry["type"] == "deb":
2571 self.check_binary_against_db(checkfile, session)
2572 elif entry["type"] == "dsc":
2573 self.check_source_against_db(checkfile, session)
2574 self.check_dsc_against_db(dsc_filename, session)
2576 # propogate in the case it is in the override tables:
2577 for suite in self.pkg.changes.get("propdistribution", {}).keys():
2578 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2579 propogate[suite] = 1
2581 nopropogate[suite] = 1
2583 for suite in propogate.keys():
2584 if suite in nopropogate:
2586 self.pkg.changes["distribution"][suite] = 1
2588 for checkfile in self.pkg.files.keys():
2589 # Check the package is still in the override tables
2590 for suite in self.pkg.changes["distribution"].keys():
2591 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2592 self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2594 ################################################################################
2595 # This is not really a reject, but an unaccept, but since a) the code for
2596 # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2597 # extremely rare, for now we'll go with whining at our admin folks...
2599 def do_unaccept(self):
2603 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2604 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2605 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2606 self.Subst["__BCC__"] = "X-DAK: dak process-accepted"
2607 if cnf.has_key("Dinstall::Bcc"):
2608 self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2610 template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2612 reject_mail_message = utils.TemplateSubst(self.Subst, template)
2614 # Write the rejection email out as the <foo>.reason file
2615 reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2616 reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2618 # If we fail here someone is probably trying to exploit the race
2619 # so let's just raise an exception ...
2620 if os.path.exists(reject_filename):
2621 os.unlink(reject_filename)
2623 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2624 os.write(fd, reject_mail_message)
2627 utils.send_mail(reject_mail_message)
2629 del self.Subst["__REJECTOR_ADDRESS__"]
2630 del self.Subst["__REJECT_MESSAGE__"]
2631 del self.Subst["__CC__"]
2633 ################################################################################
2634 # If any file of an upload has a recent mtime then chances are good
2635 # the file is still being uploaded.
2637 def upload_too_new(self):
2640 # Move back to the original directory to get accurate time stamps
2642 os.chdir(self.pkg.directory)
2643 file_list = self.pkg.files.keys()
2644 file_list.extend(self.pkg.dsc_files.keys())
2645 file_list.append(self.pkg.changes_file)
2648 last_modified = time.time()-os.path.getmtime(f)
2649 if last_modified < int(cnf["Dinstall::SkipTime"]):