5 Queue utility functions for dak
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ###############################################################################
46 from dak_exceptions import *
49 from config import Config
50 from holding import Holding
52 from summarystats import SummaryStats
53 from utils import parse_changes
54 from textutils import fix_maintainer
55 from binary import Binary
57 ###############################################################################
59 def get_type(f, session):
61 Get the file type of C{f}
64 @param f: file entry from Changes object
66 @type session: SQLA Session
67 @param session: SQL Alchemy session object
74 if f.has_key("dbtype"):
75 file_type = f["dbtype"]
76 elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
79 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
81 # Validate the override type
82 type_id = get_override_type(file_type, session)
84 utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
88 ################################################################################
90 # Determine what parts in a .changes are NEW
92 def determine_new(changes, files, warn=1):
94 Determine what parts in a C{changes} file are NEW.
96 @type changes: Upload.Pkg.changes dict
97 @param changes: Changes dictionary
99 @type files: Upload.Pkg.files dict
100 @param files: Files dictionary
103 @param warn: Warn if overrides are added for (old)stable
106 @return: dictionary of NEW components.
111 session = DBConn().session()
113 # Build up a list of potentially new things
114 for name, f in files.items():
115 # Skip byhand elements
116 if f["type"] == "byhand":
119 priority = f["priority"]
120 section = f["section"]
121 file_type = get_type(f, session)
122 component = f["component"]
124 if file_type == "dsc":
127 if not new.has_key(pkg):
129 new[pkg]["priority"] = priority
130 new[pkg]["section"] = section
131 new[pkg]["type"] = file_type
132 new[pkg]["component"] = component
133 new[pkg]["files"] = []
135 old_type = new[pkg]["type"]
136 if old_type != file_type:
137 # source gets trumped by deb or udeb
138 if old_type == "dsc":
139 new[pkg]["priority"] = priority
140 new[pkg]["section"] = section
141 new[pkg]["type"] = file_type
142 new[pkg]["component"] = component
144 new[pkg]["files"].append(name)
146 if f.has_key("othercomponents"):
147 new[pkg]["othercomponents"] = f["othercomponents"]
149 for suite in changes["suite"].keys():
150 for pkg in new.keys():
151 ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
153 for file_entry in new[pkg]["files"]:
154 if files[file_entry].has_key("new"):
155 del files[file_entry]["new"]
159 for s in ['stable', 'oldstable']:
160 if changes["suite"].has_key(s):
161 print "WARNING: overrides will be added for %s!" % s
162 for pkg in new.keys():
163 if new[pkg].has_key("othercomponents"):
164 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
170 ################################################################################
172 def check_valid(new):
174 Check if section and priority for NEW packages exist in database.
175 Additionally does sanity checks:
176 - debian-installer packages have to be udeb (or source)
177 - non debian-installer packages can not be udeb
178 - source priority can only be assigned to dsc file types
181 @param new: Dict of new packages with their section, priority and type.
184 for pkg in new.keys():
185 section_name = new[pkg]["section"]
186 priority_name = new[pkg]["priority"]
187 file_type = new[pkg]["type"]
189 section = get_section(section_name)
191 new[pkg]["section id"] = -1
193 new[pkg]["section id"] = section.section_id
195 priority = get_priority(priority_name)
197 new[pkg]["priority id"] = -1
199 new[pkg]["priority id"] = priority.priority_id
202 di = section_name.find("debian-installer") != -1
204 # If d-i, we must be udeb and vice-versa
205 if (di and file_type not in ("udeb", "dsc")) or \
206 (not di and file_type == "udeb"):
207 new[pkg]["section id"] = -1
209 # If dsc we need to be source and vice-versa
210 if (priority == "source" and file_type != "dsc") or \
211 (priority != "source" and file_type == "dsc"):
212 new[pkg]["priority id"] = -1
214 ###############################################################################
216 def lookup_uid_from_fingerprint(fpr, session):
219 # This is a stupid default, but see the comments below
222 user = get_uid_from_fingerprint(fpr, session)
226 if user.name is None:
231 # Check the relevant fingerprint (which we have to have)
232 for f in user.fingerprint:
233 if f.fingerprint == fpr:
234 is_dm = f.keyring.debian_maintainer
237 return (uid, uid_name, is_dm)
239 ###############################################################################
241 def check_status(files):
243 for f in files.keys():
244 if files[f]["type"] == "byhand":
246 elif files[f].has_key("new"):
250 ###############################################################################
252 # Used by Upload.check_timestamps
253 class TarTime(object):
254 def __init__(self, future_cutoff, past_cutoff):
256 self.future_cutoff = future_cutoff
257 self.past_cutoff = past_cutoff
260 self.future_files = {}
261 self.ancient_files = {}
263 def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
264 if MTime > self.future_cutoff:
265 self.future_files[Name] = MTime
266 if MTime < self.past_cutoff:
267 self.ancient_files[Name] = MTime
269 ###############################################################################
271 class Upload(object):
273 Everything that has to do with an upload processed.
281 ###########################################################################
284 """ Reset a number of internal variables."""
286 # Initialize the substitution template map
289 self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
290 self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
291 self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
292 self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
300 def package_info(self):
303 if len(self.rejects) > 0:
304 msg += "Reject Reasons:\n"
305 msg += "\n".join(self.rejects)
307 if len(self.warnings) > 0:
309 msg += "\n".join(self.warnings)
311 if len(self.notes) > 0:
313 msg += "\n".join(self.notes)
317 ###########################################################################
318 def update_subst(self):
319 """ Set up the per-package template substitution mappings """
323 # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
324 if not self.pkg.changes.has_key("architecture") or not \
325 isinstance(self.pkg.changes["architecture"], DictType):
326 self.pkg.changes["architecture"] = { "Unknown" : "" }
328 # and maintainer2047 may not exist.
329 if not self.pkg.changes.has_key("maintainer2047"):
330 self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
332 self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
333 self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
334 self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
336 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
337 if self.pkg.changes["architecture"].has_key("source") and \
338 self.pkg.changes["changedby822"] != "" and \
339 (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
341 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
342 self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
343 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
345 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
346 self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
347 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
349 if "sponsoremail" in self.pkg.changes:
350 self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
352 if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
353 self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
355 # Apply any global override of the Maintainer field
356 if cnf.get("Dinstall::OverrideMaintainer"):
357 self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
358 self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
360 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
361 self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
362 self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
364 ###########################################################################
365 def load_changes(self, filename):
368 @rvalue: whether the changes file was valid or not. We may want to
369 reject even if this is True (see what gets put in self.rejects).
370 This is simply to prevent us even trying things later which will
371 fail because we couldn't properly parse the file.
374 self.pkg.changes_file = filename
376 # Parse the .changes field into a dictionary
378 self.pkg.changes.update(parse_changes(filename))
379 except CantOpenError:
380 self.rejects.append("%s: can't read file." % (filename))
382 except ParseChangesError, line:
383 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
385 except ChangesUnicodeError:
386 self.rejects.append("%s: changes file not proper utf-8" % (filename))
389 # Parse the Files field from the .changes into another dictionary
391 self.pkg.files.update(utils.build_file_list(self.pkg.changes))
392 except ParseChangesError, line:
393 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
395 except UnknownFormatError, format:
396 self.rejects.append("%s: unknown format '%s'." % (filename, format))
399 # Check for mandatory fields
400 for i in ("distribution", "source", "binary", "architecture",
401 "version", "maintainer", "files", "changes", "description"):
402 if not self.pkg.changes.has_key(i):
403 # Avoid undefined errors later
404 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
407 # Strip a source version in brackets from the source field
408 if re_strip_srcver.search(self.pkg.changes["source"]):
409 self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
411 # Ensure the source field is a valid package name.
412 if not re_valid_pkg_name.match(self.pkg.changes["source"]):
413 self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
415 # Split multi-value fields into a lower-level dictionary
416 for i in ("architecture", "distribution", "binary", "closes"):
417 o = self.pkg.changes.get(i, "")
419 del self.pkg.changes[i]
421 self.pkg.changes[i] = {}
424 self.pkg.changes[i][j] = 1
426 # Fix the Maintainer: field to be RFC822/2047 compatible
428 (self.pkg.changes["maintainer822"],
429 self.pkg.changes["maintainer2047"],
430 self.pkg.changes["maintainername"],
431 self.pkg.changes["maintaineremail"]) = \
432 fix_maintainer (self.pkg.changes["maintainer"])
433 except ParseMaintError, msg:
434 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
435 % (filename, changes["maintainer"], msg))
437 # ...likewise for the Changed-By: field if it exists.
439 (self.pkg.changes["changedby822"],
440 self.pkg.changes["changedby2047"],
441 self.pkg.changes["changedbyname"],
442 self.pkg.changes["changedbyemail"]) = \
443 fix_maintainer (self.pkg.changes.get("changed-by", ""))
444 except ParseMaintError, msg:
445 self.pkg.changes["changedby822"] = ""
446 self.pkg.changes["changedby2047"] = ""
447 self.pkg.changes["changedbyname"] = ""
448 self.pkg.changes["changedbyemail"] = ""
450 self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
451 % (filename, changes["changed-by"], msg))
453 # Ensure all the values in Closes: are numbers
454 if self.pkg.changes.has_key("closes"):
455 for i in self.pkg.changes["closes"].keys():
456 if re_isanum.match (i) == None:
457 self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
459 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
460 self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
461 self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
463 # Check there isn't already a changes file of the same name in one
464 # of the queue directories.
465 base_filename = os.path.basename(filename)
466 for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
467 if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
468 self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
470 # Check the .changes is non-empty
471 if not self.pkg.files:
472 self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
475 # Changes was syntactically valid even if we'll reject
478 ###########################################################################
480 def check_distributions(self):
481 "Check and map the Distribution field"
485 # Handle suite mappings
486 for m in Cnf.ValueList("SuiteMappings"):
489 if mtype == "map" or mtype == "silent-map":
490 (source, dest) = args[1:3]
491 if self.pkg.changes["distribution"].has_key(source):
492 del self.pkg.changes["distribution"][source]
493 self.pkg.changes["distribution"][dest] = 1
494 if mtype != "silent-map":
495 self.notes.append("Mapping %s to %s." % (source, dest))
496 if self.pkg.changes.has_key("distribution-version"):
497 if self.pkg.changes["distribution-version"].has_key(source):
498 self.pkg.changes["distribution-version"][source]=dest
499 elif mtype == "map-unreleased":
500 (source, dest) = args[1:3]
501 if self.pkg.changes["distribution"].has_key(source):
502 for arch in self.pkg.changes["architecture"].keys():
503 if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
504 self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
505 del self.pkg.changes["distribution"][source]
506 self.pkg.changes["distribution"][dest] = 1
508 elif mtype == "ignore":
510 if self.pkg.changes["distribution"].has_key(suite):
511 del self.pkg.changes["distribution"][suite]
512 self.warnings.append("Ignoring %s as a target suite." % (suite))
513 elif mtype == "reject":
515 if self.pkg.changes["distribution"].has_key(suite):
516 self.rejects.append("Uploads to %s are not accepted." % (suite))
517 elif mtype == "propup-version":
518 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
520 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
521 if self.pkg.changes["distribution"].has_key(args[1]):
522 self.pkg.changes.setdefault("distribution-version", {})
523 for suite in args[2:]:
524 self.pkg.changes["distribution-version"][suite] = suite
526 # Ensure there is (still) a target distribution
527 if len(self.pkg.changes["distribution"].keys()) < 1:
528 self.rejects.append("No valid distribution remaining.")
530 # Ensure target distributions exist
531 for suite in self.pkg.changes["distribution"].keys():
532 if not Cnf.has_key("Suite::%s" % (suite)):
533 self.rejects.append("Unknown distribution `%s'." % (suite))
535 ###########################################################################
537 def binary_file_checks(self, f, session):
539 entry = self.pkg.files[f]
541 # Extract package control information
542 deb_file = utils.open_file(f)
544 control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
546 self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
548 # Can't continue, none of the checks on control would work.
551 # Check for mandantory "Description:"
554 apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
556 self.rejects.append("%s: Missing Description in binary package" % (f))
561 # Check for mandatory fields
562 for field in [ "Package", "Architecture", "Version" ]:
563 if control.Find(field) == None:
565 self.rejects.append("%s: No %s field in control." % (f, field))
568 # Ensure the package name matches the one give in the .changes
569 if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
570 self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
572 # Validate the package field
573 package = control.Find("Package")
574 if not re_valid_pkg_name.match(package):
575 self.rejects.append("%s: invalid package name '%s'." % (f, package))
577 # Validate the version field
578 version = control.Find("Version")
579 if not re_valid_version.match(version):
580 self.rejects.append("%s: invalid version number '%s'." % (f, version))
582 # Ensure the architecture of the .deb is one we know about.
583 default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
584 architecture = control.Find("Architecture")
585 upload_suite = self.pkg.changes["distribution"].keys()[0]
587 if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
588 and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
589 self.rejects.append("Unknown architecture '%s'." % (architecture))
591 # Ensure the architecture of the .deb is one of the ones
592 # listed in the .changes.
593 if not self.pkg.changes["architecture"].has_key(architecture):
594 self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
596 # Sanity-check the Depends field
597 depends = control.Find("Depends")
599 self.rejects.append("%s: Depends field is empty." % (f))
601 # Sanity-check the Provides field
602 provides = control.Find("Provides")
604 provide = re_spacestrip.sub('', provides)
606 self.rejects.append("%s: Provides field is empty." % (f))
607 prov_list = provide.split(",")
608 for prov in prov_list:
609 if not re_valid_pkg_name.match(prov):
610 self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
612 # Check the section & priority match those given in the .changes (non-fatal)
613 if control.Find("Section") and entry["section"] != "" \
614 and entry["section"] != control.Find("Section"):
615 self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
616 (f, control.Find("Section", ""), entry["section"]))
617 if control.Find("Priority") and entry["priority"] != "" \
618 and entry["priority"] != control.Find("Priority"):
619 self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
620 (f, control.Find("Priority", ""), entry["priority"]))
622 entry["package"] = package
623 entry["architecture"] = architecture
624 entry["version"] = version
625 entry["maintainer"] = control.Find("Maintainer", "")
627 if f.endswith(".udeb"):
628 self.pkg.files[f]["dbtype"] = "udeb"
629 elif f.endswith(".deb"):
630 self.pkg.files[f]["dbtype"] = "deb"
632 self.rejects.append("%s is neither a .deb or a .udeb." % (f))
634 entry["source"] = control.Find("Source", entry["package"])
636 # Get the source version
637 source = entry["source"]
640 if source.find("(") != -1:
641 m = re_extract_src_version.match(source)
643 source_version = m.group(2)
645 if not source_version:
646 source_version = self.pkg.files[f]["version"]
648 entry["source package"] = source
649 entry["source version"] = source_version
651 # Ensure the filename matches the contents of the .deb
652 m = re_isadeb.match(f)
655 file_package = m.group(1)
656 if entry["package"] != file_package:
657 self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
658 (f, file_package, entry["dbtype"], entry["package"]))
659 epochless_version = re_no_epoch.sub('', control.Find("Version"))
662 file_version = m.group(2)
663 if epochless_version != file_version:
664 self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
665 (f, file_version, entry["dbtype"], epochless_version))
668 file_architecture = m.group(3)
669 if entry["architecture"] != file_architecture:
670 self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
671 (f, file_architecture, entry["dbtype"], entry["architecture"]))
673 # Check for existent source
674 source_version = entry["source version"]
675 source_package = entry["source package"]
676 if self.pkg.changes["architecture"].has_key("source"):
677 if source_version != self.pkg.changes["version"]:
678 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
679 (source_version, f, self.pkg.changes["version"]))
681 # Check in the SQL database
682 if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
683 # Check in one of the other directories
684 source_epochless_version = re_no_epoch.sub('', source_version)
685 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
686 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
688 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
691 dsc_file_exists = False
692 for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
693 if cnf.has_key("Dir::Queue::%s" % (myq)):
694 if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
695 dsc_file_exists = True
698 if not dsc_file_exists:
699 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
701 # Check the version and for file overwrites
702 self.check_binary_against_db(f, session)
704 # Temporarily disable contents generation until we change the table storage layout
707 #if len(b.rejects) > 0:
708 # for j in b.rejects:
709 # self.rejects.append(j)
711 def source_file_checks(self, f, session):
712 entry = self.pkg.files[f]
714 m = re_issource.match(f)
718 entry["package"] = m.group(1)
719 entry["version"] = m.group(2)
720 entry["type"] = m.group(3)
722 # Ensure the source package name matches the Source filed in the .changes
723 if self.pkg.changes["source"] != entry["package"]:
724 self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
726 # Ensure the source version matches the version in the .changes file
727 if entry["type"] == "orig.tar.gz":
728 changes_version = self.pkg.changes["chopversion2"]
730 changes_version = self.pkg.changes["chopversion"]
732 if changes_version != entry["version"]:
733 self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
735 # Ensure the .changes lists source in the Architecture field
736 if not self.pkg.changes["architecture"].has_key("source"):
737 self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
739 # Check the signature of a .dsc file
740 if entry["type"] == "dsc":
741 # check_signature returns either:
742 # (None, [list, of, rejects]) or (signature, [])
743 (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
745 self.rejects.append(j)
747 entry["architecture"] = "source"
749 def per_suite_file_checks(self, f, suite, session):
751 entry = self.pkg.files[f]
752 archive = utils.where_am_i()
755 if entry.has_key("byhand"):
758 # Check we have fields we need to do these checks
760 for m in ['component', 'package', 'priority', 'size', 'md5sum']:
761 if not entry.has_key(m):
762 self.rejects.append("file '%s' does not have field %s set" % (f, m))
768 # Handle component mappings
769 for m in cnf.ValueList("ComponentMappings"):
770 (source, dest) = m.split()
771 if entry["component"] == source:
772 entry["original component"] = source
773 entry["component"] = dest
775 # Ensure the component is valid for the target suite
776 if cnf.has_key("Suite:%s::Components" % (suite)) and \
777 entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
778 self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
781 # Validate the component
782 if not get_component(entry["component"], session):
783 self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
786 # See if the package is NEW
787 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
790 # Validate the priority
791 if entry["priority"].find('/') != -1:
792 self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
794 # Determine the location
795 location = cnf["Dir::Pool"]
796 l = get_location(location, entry["component"], archive, session)
798 self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
799 entry["location id"] = -1
801 entry["location id"] = l.location_id
803 # Check the md5sum & size against existing files (if any)
804 entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
806 found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
807 entry["size"], entry["md5sum"], entry["location id"])
810 self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
811 elif found is False and poolfile is not None:
812 self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
815 entry["files id"] = None
817 entry["files id"] = poolfile.file_id
819 # Check for packages that have moved from one component to another
820 entry['suite'] = suite
821 res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
823 entry["othercomponents"] = res.fetchone()[0]
825 def check_files(self, action=True):
826 archive = utils.where_am_i()
827 file_keys = self.pkg.files.keys()
831 # XXX: As far as I can tell, this can no longer happen - see
832 # comments by AJ in old revisions - mhy
833 # if reprocess is 2 we've already done this and we're checking
834 # things again for the new .orig.tar.gz.
835 # [Yes, I'm fully aware of how disgusting this is]
836 if action and self.reprocess < 2:
838 os.chdir(self.pkg.directory)
840 ret = holding.copy_to_holding(f)
842 # XXX: Should we bail out here or try and continue?
843 self.rejects.append(ret)
847 # Check there isn't already a .changes or .dak file of the same name in
848 # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
849 # [NB: this check must be done post-suite mapping]
850 base_filename = os.path.basename(self.pkg.changes_file)
851 dot_dak_filename = base_filename[:-8] + ".dak"
853 for suite in self.pkg.changes["distribution"].keys():
854 copychanges = "Suite::%s::CopyChanges" % (suite)
855 if cnf.has_key(copychanges) and \
856 os.path.exists(os.path.join(cnf[copychanges], base_filename)):
857 self.rejects.append("%s: a file with this name already exists in %s" \
858 % (base_filename, cnf[copychanges]))
860 copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
861 if cnf.has_key(copy_dot_dak) and \
862 os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
863 self.rejects.append("%s: a file with this name already exists in %s" \
864 % (dot_dak_filename, Cnf[copy_dot_dak]))
870 session = DBConn().session()
872 for f, entry in self.pkg.files.items():
873 # Ensure the file does not already exist in one of the accepted directories
874 for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
875 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
876 if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
877 self.rejects.append("%s file already exists in the %s directory." % (f, d))
879 if not re_taint_free.match(f):
880 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
882 # Check the file is readable
883 if os.access(f, os.R_OK) == 0:
884 # When running in -n, copy_to_holding() won't have
885 # generated the reject_message, so we need to.
887 if os.path.exists(f):
888 self.rejects.append("Can't read `%s'. [permission denied]" % (f))
890 self.rejects.append("Can't read `%s'. [file not found]" % (f))
891 entry["type"] = "unreadable"
894 # If it's byhand skip remaining checks
895 if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
897 entry["type"] = "byhand"
899 # Checks for a binary package...
900 elif re_isadeb.match(f):
902 entry["type"] = "deb"
904 # This routine appends to self.rejects/warnings as appropriate
905 self.binary_file_checks(f, session)
907 # Checks for a source package...
908 elif re_issource.match(f):
911 # This routine appends to self.rejects/warnings as appropriate
912 self.source_file_checks(f, session)
914 # Not a binary or source package? Assume byhand...
917 entry["type"] = "byhand"
919 # Per-suite file checks
920 entry["oldfiles"] = {}
921 for suite in self.pkg.changes["distribution"].keys():
922 self.per_suite_file_checks(f, suite, session)
926 # If the .changes file says it has source, it must have source.
927 if self.pkg.changes["architecture"].has_key("source"):
929 self.rejects.append("no source found and Architecture line in changes mention source.")
931 if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
932 self.rejects.append("source only uploads are not supported.")
934 ###########################################################################
935 def check_dsc(self, action=True):
936 """Returns bool indicating whether or not the source changes are valid"""
937 # Ensure there is source to check
938 if not self.pkg.changes["architecture"].has_key("source"):
943 for f, entry in self.pkg.files.items():
944 if entry["type"] == "dsc":
946 self.rejects.append("can not process a .changes file with multiple .dsc's.")
951 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
953 self.rejects.append("source uploads must contain a dsc file")
956 # Parse the .dsc file
958 self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
959 except CantOpenError:
960 # if not -n copy_to_holding() will have done this for us...
962 self.rejects.append("%s: can't read file." % (dsc_filename))
963 except ParseChangesError, line:
964 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
965 except InvalidDscError, line:
966 self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
967 except ChangesUnicodeError:
968 self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
970 # Build up the file list of files mentioned by the .dsc
972 self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
973 except NoFilesFieldError:
974 self.rejects.append("%s: no Files: field." % (dsc_filename))
976 except UnknownFormatError, format:
977 self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
979 except ParseChangesError, line:
980 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
983 # Enforce mandatory fields
984 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
985 if not self.pkg.dsc.has_key(i):
986 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
989 # Validate the source and version fields
990 if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
991 self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
992 if not re_valid_version.match(self.pkg.dsc["version"]):
993 self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
995 # Bumping the version number of the .dsc breaks extraction by stable's
996 # dpkg-source. So let's not do that...
997 if self.pkg.dsc["format"] != "1.0":
998 self.rejects.append("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename))
1000 # Validate the Maintainer field
1002 # We ignore the return value
1003 fix_maintainer(self.pkg.dsc["maintainer"])
1004 except ParseMaintError, msg:
1005 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
1006 % (dsc_filename, self.pkg.dsc["maintainer"], msg))
1008 # Validate the build-depends field(s)
1009 for field_name in [ "build-depends", "build-depends-indep" ]:
1010 field = self.pkg.dsc.get(field_name)
1012 # Check for broken dpkg-dev lossage...
1013 if field.startswith("ARRAY"):
1014 self.rejects.append("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % \
1015 (dsc_filename, field_name.title()))
1017 # Have apt try to parse them...
1019 apt_pkg.ParseSrcDepends(field)
1021 self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
1023 # Ensure the version number in the .dsc matches the version number in the .changes
1024 epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1025 changes_version = self.pkg.files[dsc_filename]["version"]
1027 if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1028 self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1030 # Ensure there is a .tar.gz in the .dsc file
1032 for f in self.pkg.dsc_files.keys():
1033 m = re_issource.match(f)
1035 self.rejects.append("%s: %s in Files field not recognised as source." % (dsc_filename, f))
1038 if ftype == "orig.tar.gz" or ftype == "tar.gz":
1042 self.rejects.append("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
1044 # Ensure source is newer than existing source in target suites
1045 session = DBConn().session()
1046 self.check_source_against_db(dsc_filename, session)
1047 self.check_dsc_against_db(dsc_filename, session)
1052 ###########################################################################
1054 def get_changelog_versions(self, source_dir):
1055 """Extracts a the source package and (optionally) grabs the
1056 version history out of debian/changelog for the BTS."""
1060 # Find the .dsc (again)
1062 for f in self.pkg.files.keys():
1063 if self.pkg.files[f]["type"] == "dsc":
1066 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1067 if not dsc_filename:
1070 # Create a symlink mirror of the source files in our temporary directory
1071 for f in self.pkg.files.keys():
1072 m = re_issource.match(f)
1074 src = os.path.join(source_dir, f)
1075 # If a file is missing for whatever reason, give up.
1076 if not os.path.exists(src):
1079 if ftype == "orig.tar.gz" and self.pkg.orig_tar_gz:
1081 dest = os.path.join(os.getcwd(), f)
1082 os.symlink(src, dest)
1084 # If the orig.tar.gz is not a part of the upload, create a symlink to the
1086 if self.pkg.orig_tar_gz:
1087 dest = os.path.join(os.getcwd(), os.path.basename(self.pkg.orig_tar_gz))
1088 os.symlink(self.pkg.orig_tar_gz, dest)
1090 # Extract the source
1091 cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1092 (result, output) = commands.getstatusoutput(cmd)
1094 self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1095 self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
1098 if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1101 # Get the upstream version
1102 upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1103 if re_strip_revision.search(upstr_version):
1104 upstr_version = re_strip_revision.sub('', upstr_version)
1106 # Ensure the changelog file exists
1107 changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1108 if not os.path.exists(changelog_filename):
1109 self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1112 # Parse the changelog
1113 self.pkg.dsc["bts changelog"] = ""
1114 changelog_file = utils.open_file(changelog_filename)
1115 for line in changelog_file.readlines():
1116 m = re_changelog_versions.match(line)
1118 self.pkg.dsc["bts changelog"] += line
1119 changelog_file.close()
1121 # Check we found at least one revision in the changelog
1122 if not self.pkg.dsc["bts changelog"]:
1123 self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1125 def check_source(self):
1126 # XXX: I'm fairly sure reprocess == 2 can never happen
1127 # AJT disabled the is_incoming check years ago - mhy
1128 # We should probably scrap or rethink the whole reprocess thing
1130 # a) there's no source
1131 # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
1132 # or c) the orig.tar.gz is MIA
1133 if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
1134 or self.pkg.orig_tar_gz == -1:
1137 tmpdir = utils.temp_dirname()
1139 # Move into the temporary directory
1143 # Get the changelog version history
1144 self.get_changelog_versions(cwd)
1146 # Move back and cleanup the temporary tree
1150 shutil.rmtree(tmpdir)
1152 if e.errno != errno.EACCES:
1154 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1156 self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1157 # We probably have u-r or u-w directories so chmod everything
1159 cmd = "chmod -R u+rwx %s" % (tmpdir)
1160 result = os.system(cmd)
1162 utils.fubar("'%s' failed with result %s." % (cmd, result))
1163 shutil.rmtree(tmpdir)
1164 except Exception, e:
1165 print "foobar2 (%s)" % e
1166 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1168 ###########################################################################
1169 def ensure_hashes(self):
1170 # Make sure we recognise the format of the Files: field in the .changes
1171 format = self.pkg.changes.get("format", "0.0").split(".", 1)
1172 if len(format) == 2:
1173 format = int(format[0]), int(format[1])
1175 format = int(float(format[0])), 0
1177 # We need to deal with the original changes blob, as the fields we need
1178 # might not be in the changes dict serialised into the .dak anymore.
1179 orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1181 # Copy the checksums over to the current changes dict. This will keep
1182 # the existing modifications to it intact.
1183 for field in orig_changes:
1184 if field.startswith('checksums-'):
1185 self.pkg.changes[field] = orig_changes[field]
1187 # Check for unsupported hashes
1188 for j in utils.check_hash_fields(".changes", self.pkg.changes):
1189 self.rejects.append(j)
1191 for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1192 self.rejects.append(j)
1194 # We have to calculate the hash if we have an earlier changes version than
1195 # the hash appears in rather than require it exist in the changes file
1196 for hashname, hashfunc, version in utils.known_hashes:
1197 # TODO: Move _ensure_changes_hash into this class
1198 for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1199 self.rejects.append(j)
1200 if "source" in self.pkg.changes["architecture"]:
1201 # TODO: Move _ensure_dsc_hash into this class
1202 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1203 self.rejects.append(j)
1205 def check_hashes(self):
1206 for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1207 self.rejects.append(m)
1209 for m in utils.check_size(".changes", self.pkg.files):
1210 self.rejects.append(m)
1212 for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1213 self.rejects.append(m)
1215 for m in utils.check_size(".dsc", self.pkg.dsc_files):
1216 self.rejects.append(m)
1218 self.ensure_hashes()
1220 ###########################################################################
1221 def check_urgency(self):
1223 if self.pkg.changes["architecture"].has_key("source"):
1224 if not self.pkg.changes.has_key("urgency"):
1225 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1226 self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1227 if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1228 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1229 (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1230 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1232 ###########################################################################
1234 # Sanity check the time stamps of files inside debs.
1235 # [Files in the near future cause ugly warnings and extreme time
1236 # travel can cause errors on extraction]
1238 def check_timestamps(self):
1241 future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1242 past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1243 tar = TarTime(future_cutoff, past_cutoff)
1245 for filename, entry in self.pkg.files.items():
1246 if entry["type"] == "deb":
1249 deb_file = utils.open_file(filename)
1250 apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1253 apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1254 except SystemError, e:
1255 # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1256 if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1259 apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1263 future_files = tar.future_files.keys()
1265 num_future_files = len(future_files)
1266 future_file = future_files[0]
1267 future_date = tar.future_files[future_file]
1268 self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1269 % (filename, num_future_files, future_file, time.ctime(future_date)))
1271 ancient_files = tar.ancient_files.keys()
1273 num_ancient_files = len(ancient_files)
1274 ancient_file = ancient_files[0]
1275 ancient_date = tar.ancient_files[ancient_file]
1276 self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1277 % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1279 self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1281 ###########################################################################
1282 def check_transition(self, session):
1285 sourcepkg = self.pkg.changes["source"]
1287 # No sourceful upload -> no need to do anything else, direct return
1288 # We also work with unstable uploads, not experimental or those going to some
1289 # proposed-updates queue
1290 if "source" not in self.pkg.changes["architecture"] or \
1291 "unstable" not in self.pkg.changes["distribution"]:
1294 # Also only check if there is a file defined (and existant) with
1296 transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1297 if transpath == "" or not os.path.exists(transpath):
1300 # Parse the yaml file
1301 sourcefile = file(transpath, 'r')
1302 sourcecontent = sourcefile.read()
1304 transitions = yaml.load(sourcecontent)
1305 except yaml.YAMLError, msg:
1306 # This shouldn't happen, there is a wrapper to edit the file which
1307 # checks it, but we prefer to be safe than ending up rejecting
1309 utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1312 # Now look through all defined transitions
1313 for trans in transitions:
1314 t = transitions[trans]
1315 source = t["source"]
1318 # Will be None if nothing is in testing.
1319 current = get_source_in_suite(source, "testing", session)
1320 if current is not None:
1321 compare = apt_pkg.VersionCompare(current.version, expected)
1323 if current is None or compare < 0:
1324 # This is still valid, the current version in testing is older than
1325 # the new version we wait for, or there is none in testing yet
1327 # Check if the source we look at is affected by this.
1328 if sourcepkg in t['packages']:
1329 # The source is affected, lets reject it.
1331 rejectmsg = "%s: part of the %s transition.\n\n" % (
1334 if current is not None:
1335 currentlymsg = "at version %s" % (current.version)
1337 currentlymsg = "not present in testing"
1339 rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1341 rejectmsg += "\n".join(textwrap.wrap("""Your package
1342 is part of a testing transition designed to get %s migrated (it is
1343 currently %s, we need version %s). This transition is managed by the
1344 Release Team, and %s is the Release-Team member responsible for it.
1345 Please mail debian-release@lists.debian.org or contact %s directly if you
1346 need further assistance. You might want to upload to experimental until this
1347 transition is done."""
1348 % (source, currentlymsg, expected,t["rm"], t["rm"])))
1350 self.rejects.append(rejectmsg)
1353 ###########################################################################
1354 def check_signed_by_key(self):
1355 """Ensure the .changes is signed by an authorized uploader."""
1356 session = DBConn().session()
1358 self.check_transition(session)
1360 (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
1362 # match claimed name with actual name:
1364 # This is fundamentally broken but need us to refactor how we get
1365 # the UIDs/Fingerprints in order for us to fix it properly
1366 uid, uid_email = self.pkg.changes["fingerprint"], uid
1367 may_nmu, may_sponsor = 1, 1
1368 # XXX by default new dds don't have a fingerprint/uid in the db atm,
1369 # and can't get one in there if we don't allow nmu/sponsorship
1370 elif is_dm is False:
1371 # If is_dm is False, we allow full upload rights
1372 uid_email = "%s@debian.org" % (uid)
1373 may_nmu, may_sponsor = 1, 1
1375 # Assume limited upload rights unless we've discovered otherwise
1377 may_nmu, may_sponsor = 0, 0
1379 if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1381 elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1383 if uid_name == "": sponsored = 1
1386 if ("source" in self.pkg.changes["architecture"] and
1387 uid_email and utils.is_email_alias(uid_email)):
1388 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1389 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1390 self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1391 self.pkg.changes["sponsoremail"] = uid_email
1393 if sponsored and not may_sponsor:
1394 self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
1396 if not sponsored and not may_nmu:
1397 should_reject = True
1398 highest_sid, highest_version = None, None
1400 # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
1401 # It ignores higher versions with the dm_upload_allowed flag set to false
1402 # I'm keeping the existing behaviour for now until I've gone back and
1403 # checked exactly what the GR says - mhy
1404 for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
1405 if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
1406 highest_sid = si.source_id
1407 highest_version = si.version
1409 if highest_sid is None:
1410 self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
1412 for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
1413 (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1414 if email == uid_email or name == uid_name:
1415 should_reject = False
1418 if should_reject is True:
1419 self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
1421 for b in self.pkg.changes["binary"].keys():
1422 for suite in self.pkg.changes["distribution"].keys():
1423 q = session.query(DBSource)
1424 q = q.join(DBBinary).filter_by(package=b)
1425 q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1428 if s.source != self.pkg.changes["source"]:
1429 self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
1431 for f in self.pkg.files.keys():
1432 if self.pkg.files[f].has_key("byhand"):
1433 self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
1434 if self.pkg.files[f].has_key("new"):
1435 self.rejects.append("%s may not upload NEW file %s" % (uid, f))
1439 ###########################################################################
1440 def build_summaries(self):
1441 """ Build a summary of changes the upload introduces. """
1443 (byhand, new, summary, override_summary) = self.pkg.file_summary()
1445 short_summary = summary
1447 # This is for direport's benefit...
1448 f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1451 summary += "Changes: " + f
1453 summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1455 summary += self.announce(short_summary, 0)
1457 return (summary, short_summary)
1459 ###########################################################################
1461 def close_bugs(self, summary, action):
1463 Send mail to close bugs as instructed by the closes field in the changes file.
1464 Also add a line to summary if any work was done.
1466 @type summary: string
1467 @param summary: summary text, as given by L{build_summaries}
1470 @param action: Set to false no real action will be done.
1473 @return: summary. If action was taken, extended by the list of closed bugs.
1477 template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1479 bugs = self.pkg.changes["closes"].keys()
1485 summary += "Closing bugs: "
1487 summary += "%s " % (bug)
1490 self.Subst["__BUG_NUMBER__"] = bug
1491 if self.pkg.changes["distribution"].has_key("stable"):
1492 self.Subst["__STABLE_WARNING__"] = """
1493 Note that this package is not part of the released stable Debian
1494 distribution. It may have dependencies on other unreleased software,
1495 or other instabilities. Please take care if you wish to install it.
1496 The update will eventually make its way into the next released Debian
1499 self.Subst["__STABLE_WARNING__"] = ""
1500 mail_message = utils.TemplateSubst(self.Subst, template)
1501 utils.send_mail(mail_message)
1503 # Clear up after ourselves
1504 del self.Subst["__BUG_NUMBER__"]
1505 del self.Subst["__STABLE_WARNING__"]
1507 if action and self.logger:
1508 self.logger.log(["closing bugs"] + bugs)
1514 ###########################################################################
1516 def announce(self, short_summary, action):
1518 Send an announce mail about a new upload.
1520 @type short_summary: string
1521 @param short_summary: Short summary text to include in the mail
1524 @param action: Set to false no real action will be done.
1527 @return: Textstring about action taken.
1532 announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1534 # Only do announcements for source uploads with a recent dpkg-dev installed
1535 if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1536 self.pkg.changes["architecture"].has_key("source"):
1542 self.Subst["__SHORT_SUMMARY__"] = short_summary
1544 for dist in self.pkg.changes["distribution"].keys():
1545 announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1546 if announce_list == "" or lists_done.has_key(announce_list):
1549 lists_done[announce_list] = 1
1550 summary += "Announcing to %s\n" % (announce_list)
1554 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1555 if cnf.get("Dinstall::TrackingServer") and \
1556 self.pkg.changes["architecture"].has_key("source"):
1557 trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1558 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1560 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1561 utils.send_mail(mail_message)
1563 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1565 if cnf.FindB("Dinstall::CloseBugs"):
1566 summary = self.close_bugs(summary, action)
1568 del self.Subst["__SHORT_SUMMARY__"]
1572 ###########################################################################
1574 def accept (self, summary, short_summary, targetdir=None):
1578 This moves all files referenced from the .changes into the I{accepted}
1579 queue, sends the accepted mail, announces to lists, closes bugs and
1580 also checks for override disparities. If enabled it will write out
1581 the version history for the BTS Version Tracking and will finally call
1584 @type summary: string
1585 @param summary: Summary text
1587 @type short_summary: string
1588 @param short_summary: Short summary
1593 stats = SummaryStats()
1595 accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
1597 if targetdir is None:
1598 targetdir = cnf["Dir::Queue::Accepted"]
1602 self.logger.log(["Accepting changes", self.pkg.changes_file])
1604 self.pkg.write_dot_dak(targetdir)
1606 # Move all the files into the accepted directory
1607 utils.move(self.pkg.changes_file, targetdir)
1609 for name, entry in sorted(self.pkg.files.items()):
1610 utils.move(name, targetdir)
1611 stats.accept_bytes += float(entry["size"])
1613 stats.accept_count += 1
1615 # Send accept mail, announce to lists, close bugs and check for
1616 # override disparities
1617 if not cnf["Dinstall::Options::No-Mail"]:
1619 self.Subst["__SUITE__"] = ""
1620 self.Subst["__SUMMARY__"] = summary
1621 mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
1622 utils.send_mail(mail_message)
1623 self.announce(short_summary, 1)
1625 ## Helper stuff for DebBugs Version Tracking
1626 if cnf.Find("Dir::Queue::BTSVersionTrack"):
1627 # ??? once queue/* is cleared on *.d.o and/or reprocessed
1628 # the conditionalization on dsc["bts changelog"] should be
1631 # Write out the version history from the changelog
1632 if self.pkg.changes["architecture"].has_key("source") and \
1633 self.pkg.dsc.has_key("bts changelog"):
1635 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1636 version_history = os.fdopen(fd, 'w')
1637 version_history.write(self.pkg.dsc["bts changelog"])
1638 version_history.close()
1639 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1640 self.pkg.changes_file[:-8]+".versions")
1641 os.rename(temp_filename, filename)
1642 os.chmod(filename, 0644)
1644 # Write out the binary -> source mapping.
1645 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1646 debinfo = os.fdopen(fd, 'w')
1647 for name, entry in sorted(self.pkg.files.items()):
1648 if entry["type"] == "deb":
1649 line = " ".join([entry["package"], entry["version"],
1650 entry["architecture"], entry["source package"],
1651 entry["source version"]])
1652 debinfo.write(line+"\n")
1654 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1655 self.pkg.changes_file[:-8]+".debinfo")
1656 os.rename(temp_filename, filename)
1657 os.chmod(filename, 0644)
1659 # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
1660 # <Ganneff> we do call queue_build too
1661 # <mhy> well yes, we'd have had to if we were inserting into accepted
1662 # <Ganneff> now. thats database only.
1663 # <mhy> urgh, that's going to get messy
1664 # <Ganneff> so i make the p-n call to it *also* using accepted/
1665 # <mhy> but then the packages will be in the queue_build table without the files being there
1666 # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
1667 # <mhy> ah, good point
1668 # <Ganneff> so it will work out, as unchecked move it over
1669 # <mhy> that's all completely sick
1672 # This routine returns None on success or an error on failure
1673 res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
1678 def check_override(self):
1680 Checks override entries for validity. Mails "Override disparity" warnings,
1681 if that feature is enabled.
1683 Abandons the check if
1684 - override disparity checks are disabled
1685 - mail sending is disabled
1690 # Abandon the check if:
1691 # a) override disparity checks have been disabled
1692 # b) we're not sending mail
1693 if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
1694 cnf["Dinstall::Options::No-Mail"]:
1697 summary = self.pkg.check_override()
1702 overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
1705 self.Subst["__SUMMARY__"] = summary
1706 mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
1707 utils.send_mail(mail_message)
1708 del self.Subst["__SUMMARY__"]
1710 ###########################################################################
1712 def remove(self, dir=None):
1714 Used (for instance) in p-u to remove the package from unchecked
1717 os.chdir(self.pkg.directory)
1721 for f in self.pkg.files.keys():
1723 os.unlink(self.pkg.changes_file)
1725 ###########################################################################
1727 def move_to_dir (self, dest, perms=0660, changesperms=0664):
1729 Move files to dest with certain perms/changesperms
1731 utils.move(self.pkg.changes_file, dest, perms=changesperms)
1732 for f in self.pkg.files.keys():
1733 utils.move(f, dest, perms=perms)
1735 ###########################################################################
1737 def force_reject(self, reject_files):
1739 Forcefully move files from the current directory to the
1740 reject directory. If any file already exists in the reject
1741 directory it will be moved to the morgue to make way for
1745 @param files: file dictionary
1751 for file_entry in reject_files:
1752 # Skip any files which don't exist or which we don't have permission to copy.
1753 if os.access(file_entry, os.R_OK) == 0:
1756 dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
1759 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
1761 # File exists? Let's try and move it to the morgue
1762 if e.errno == errno.EEXIST:
1763 morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
1765 morgue_file = utils.find_next_free(morgue_file)
1766 except NoFreeFilenameError:
1767 # Something's either gone badly Pete Tong, or
1768 # someone is trying to exploit us.
1769 utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
1771 utils.move(dest_file, morgue_file, perms=0660)
1773 dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1776 utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
1780 # If we got here, we own the destination file, so we can
1781 # safely overwrite it.
1782 utils.move(file_entry, dest_file, 1, perms=0660)
1785 ###########################################################################
1786 def do_reject (self, manual=0, reject_message="", note=""):
1788 Reject an upload. If called without a reject message or C{manual} is
1789 true, spawn an editor so the user can write one.
1792 @param manual: manual or automated rejection
1794 @type reject_message: string
1795 @param reject_message: A reject message
1800 # If we weren't given a manual rejection message, spawn an
1801 # editor so the user can add one in...
1802 if manual and not reject_message:
1803 (fd, temp_filename) = utils.temp_filename()
1804 temp_file = os.fdopen(fd, 'w')
1807 temp_file.write(line)
1809 editor = os.environ.get("EDITOR","vi")
1811 while answer == 'E':
1812 os.system("%s %s" % (editor, temp_filename))
1813 temp_fh = utils.open_file(temp_filename)
1814 reject_message = "".join(temp_fh.readlines())
1816 print "Reject message:"
1817 print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1)
1818 prompt = "[R]eject, Edit, Abandon, Quit ?"
1820 while prompt.find(answer) == -1:
1821 answer = utils.our_raw_input(prompt)
1822 m = re_default_answer.search(prompt)
1825 answer = answer[:1].upper()
1826 os.unlink(temp_filename)
1832 print "Rejecting.\n"
1836 reason_filename = self.pkg.changes_file[:-8] + ".reason"
1837 reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
1839 # Move all the files into the reject directory
1840 reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
1841 self.force_reject(reject_files)
1843 # If we fail here someone is probably trying to exploit the race
1844 # so let's just raise an exception ...
1845 if os.path.exists(reason_filename):
1846 os.unlink(reason_filename)
1847 reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1849 rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
1853 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
1854 self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
1855 self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
1856 os.write(reason_fd, reject_message)
1857 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1859 # Build up the rejection email
1860 user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
1861 self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
1862 self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
1863 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
1864 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1865 # Write the rejection email out as the <foo>.reason file
1866 os.write(reason_fd, reject_mail_message)
1868 del self.Subst["__REJECTOR_ADDRESS__"]
1869 del self.Subst["__MANUAL_REJECT_MESSAGE__"]
1870 del self.Subst["__CC__"]
1874 # Send the rejection mail if appropriate
1875 if not cnf["Dinstall::Options::No-Mail"]:
1876 utils.send_mail(reject_mail_message)
1879 self.logger.log(["rejected", self.pkg.changes_file])
1883 ################################################################################
1884 def in_override_p(self, package, component, suite, binary_type, file, session):
1886 Check if a package already has override entries in the DB
1888 @type package: string
1889 @param package: package name
1891 @type component: string
1892 @param component: database id of the component
1895 @param suite: database id of the suite
1897 @type binary_type: string
1898 @param binary_type: type of the package
1901 @param file: filename we check
1903 @return: the database result. But noone cares anyway.
1909 if binary_type == "": # must be source
1912 file_type = binary_type
1914 # Override suite name; used for example with proposed-updates
1915 if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
1916 suite = cnf["Suite::%s::OverrideSuite" % (suite)]
1918 result = get_override(package, suite, component, file_type, session)
1920 # If checking for a source package fall back on the binary override type
1921 if file_type == "dsc" and len(result) < 1:
1922 result = get_override(package, suite, component, ['deb', 'udeb'], session)
1924 # Remember the section and priority so we can check them later if appropriate
1927 self.pkg.files[file]["override section"] = result.section.section
1928 self.pkg.files[file]["override priority"] = result.priority.priority
1933 ################################################################################
1934 def get_anyversion(self, sv_list, suite):
1937 @param sv_list: list of (suite, version) tuples to check
1940 @param suite: suite name
1946 anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
1947 for (s, v) in sv_list:
1948 if s in [ x.lower() for x in anysuite ]:
1949 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
1954 ################################################################################
1956 def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
1959 @param sv_list: list of (suite, version) tuples to check
1964 @type new_version: string
1965 @param new_version: XXX
1967 Ensure versions are newer than existing packages in target
1968 suites and that cross-suite version checking rules as
1969 set out in the conf file are satisfied.
1974 # Check versions for each target suite
1975 for target_suite in self.pkg.changes["distribution"].keys():
1976 must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
1977 must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
1979 # Enforce "must be newer than target suite" even if conffile omits it
1980 if target_suite not in must_be_newer_than:
1981 must_be_newer_than.append(target_suite)
1983 for (suite, existent_version) in sv_list:
1984 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
1986 if suite in must_be_newer_than and sourceful and vercmp < 1:
1987 self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
1989 if suite in must_be_older_than and vercmp > -1:
1992 if self.pkg.changes.get('distribution-version', {}).has_key(suite):
1993 # we really use the other suite, ignoring the conflicting one ...
1994 addsuite = self.pkg.changes["distribution-version"][suite]
1996 add_version = self.get_anyversion(sv_list, addsuite)
1997 target_version = self.get_anyversion(sv_list, target_suite)
2000 # not add_version can only happen if we map to a suite
2001 # that doesn't enhance the suite we're propup'ing from.
2002 # so "propup-ver x a b c; map a d" is a problem only if
2003 # d doesn't enhance a.
2005 # i think we could always propagate in this case, rather
2006 # than complaining. either way, this isn't a REJECT issue
2008 # And - we really should complain to the dorks who configured dak
2009 self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
2010 self.pkg.changes.setdefault("propdistribution", {})
2011 self.pkg.changes["propdistribution"][addsuite] = 1
2013 elif not target_version:
2014 # not targets_version is true when the package is NEW
2015 # we could just stick with the "...old version..." REJECT
2016 # for this, I think.
2017 self.rejects.append("Won't propogate NEW packages.")
2018 elif apt_pkg.VersionCompare(new_version, add_version) < 0:
2019 # propogation would be redundant. no need to reject though.
2020 self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2022 elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2023 apt_pkg.VersionCompare(add_version, target_version) >= 0:
2025 self.warnings.append("Propogating upload to %s" % (addsuite))
2026 self.pkg.changes.setdefault("propdistribution", {})
2027 self.pkg.changes["propdistribution"][addsuite] = 1
2031 self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2033 ################################################################################
2034 def check_binary_against_db(self, file, session):
2035 # Ensure version is sane
2036 q = session.query(BinAssociation)
2037 q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
2038 q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
2040 self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2041 file, self.pkg.files[file]["version"], sourceful=False)
2043 # Check for any existing copies of the file
2044 q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
2045 q = q.filter_by(version=self.pkg.files[file]["version"])
2046 q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
2049 self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
2051 ################################################################################
2053 def check_source_against_db(self, file, session):
2056 source = self.pkg.dsc.get("source")
2057 version = self.pkg.dsc.get("version")
2059 # Ensure version is sane
2060 q = session.query(SrcAssociation)
2061 q = q.join(DBSource).filter(DBSource.source==source)
2063 self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2064 file, version, sourceful=True)
2066 ################################################################################
2067 def check_dsc_against_db(self, file, session):
2070 @warning: NB: this function can remove entries from the 'files' index [if
2071 the .orig.tar.gz is a duplicate of the one in the archive]; if
2072 you're iterating over 'files' and call this function as part of
2073 the loop, be sure to add a check to the top of the loop to
2074 ensure you haven't just tried to dereference the deleted entry.
2079 self.pkg.orig_tar_gz = None
2081 # Try and find all files mentioned in the .dsc. This has
2082 # to work harder to cope with the multiple possible
2083 # locations of an .orig.tar.gz.
2084 # The ordering on the select is needed to pick the newest orig
2085 # when it exists in multiple places.
2086 for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2088 if self.pkg.files.has_key(dsc_name):
2089 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2090 actual_size = int(self.pkg.files[dsc_name]["size"])
2091 found = "%s in incoming" % (dsc_name)
2093 # Check the file does not already exist in the archive
2094 ql = get_poolfile_like_name(dsc_name, session)
2096 # Strip out anything that isn't '%s' or '/%s$'
2098 if not i.filename.endswith(dsc_name):
2101 # "[dak] has not broken them. [dak] has fixed a
2102 # brokenness. Your crappy hack exploited a bug in
2105 # "(Come on! I thought it was always obvious that
2106 # one just doesn't release different files with
2107 # the same name and version.)"
2108 # -- ajk@ on d-devel@l.d.o
2111 # Ignore exact matches for .orig.tar.gz
2113 if dsc_name.endswith(".orig.tar.gz"):
2115 if self.pkg.files.has_key(dsc_name) and \
2116 int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2117 self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2118 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2119 # TODO: Don't delete the entry, just mark it as not needed
2120 # This would fix the stupidity of changing something we often iterate over
2121 # whilst we're doing it
2122 del self.pkg.files[dsc_name]
2123 self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename)
2127 self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2129 elif dsc_name.endswith(".orig.tar.gz"):
2131 ql = get_poolfile_like_name(dsc_name, session)
2133 # Strip out anything that isn't '%s' or '/%s$'
2134 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2136 if not i.filename.endswith(dsc_name):
2140 # Unfortunately, we may get more than one match here if,
2141 # for example, the package was in potato but had an -sa
2142 # upload in woody. So we need to choose the right one.
2144 # default to something sane in case we don't match any or have only one
2149 old_file = os.path.join(i.location.path, i.filename)
2150 old_file_fh = utils.open_file(old_file)
2151 actual_md5 = apt_pkg.md5sum(old_file_fh)
2153 actual_size = os.stat(old_file)[stat.ST_SIZE]
2154 if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2157 old_file = os.path.join(i.location.path, i.filename)
2158 old_file_fh = utils.open_file(old_file)
2159 actual_md5 = apt_pkg.md5sum(old_file_fh)
2161 actual_size = os.stat(old_file)[stat.ST_SIZE]
2163 suite_type = x.location.archive_type
2164 # need this for updating dsc_files in install()
2165 dsc_entry["files id"] = x.file_id
2166 # See install() in process-accepted...
2167 self.pkg.orig_tar_id = x.file_id
2168 self.pkg.orig_tar_gz = old_file
2169 self.pkg.orig_tar_location = x.location.location_id
2171 # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2172 # Not there? Check the queue directories...
2173 for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2174 if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2176 in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2177 if os.path.exists(in_otherdir):
2178 in_otherdir_fh = utils.open_file(in_otherdir)
2179 actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2180 in_otherdir_fh.close()
2181 actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2183 self.pkg.orig_tar_gz = in_otherdir
2186 self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
2187 self.pkg.orig_tar_gz = -1
2190 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
2192 if actual_md5 != dsc_entry["md5sum"]:
2193 self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
2194 if actual_size != int(dsc_entry["size"]):
2195 self.rejects.append("size for %s doesn't match %s." % (found, file))
2197 ################################################################################
2198 # This is used by process-new and process-holding to recheck a changes file
2199 # at the time we're running. It mainly wraps various other internal functions
2200 # and is similar to accepted_checks - these should probably be tidied up
2202 def recheck(self, session):
2204 for f in self.pkg.files.keys():
2205 # The .orig.tar.gz can disappear out from under us is it's a
2206 # duplicate of one in the archive.
2207 if not self.pkg.files.has_key(f):
2210 entry = self.pkg.files[f]
2212 # Check that the source still exists
2213 if entry["type"] == "deb":
2214 source_version = entry["source version"]
2215 source_package = entry["source package"]
2216 if not self.pkg.changes["architecture"].has_key("source") \
2217 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
2218 source_epochless_version = re_no_epoch.sub('', source_version)
2219 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
2221 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
2222 if cnf.has_key("Dir::Queue::%s" % (q)):
2223 if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
2226 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
2228 # Version and file overwrite checks
2229 if entry["type"] == "deb":
2230 self.check_binary_against_db(f, session)
2231 elif entry["type"] == "dsc":
2232 self.check_source_against_db(f, session)
2233 self.check_dsc_against_db(f, session)
2235 ################################################################################
2236 def accepted_checks(self, overwrite_checks, session):
2237 # Recheck anything that relies on the database; since that's not
2238 # frozen between accept and our run time when called from p-a.
2240 # overwrite_checks is set to False when installing to stable/oldstable
2245 # Find the .dsc (again)
2247 for f in self.pkg.files.keys():
2248 if self.pkg.files[f]["type"] == "dsc":
2251 for checkfile in self.pkg.files.keys():
2252 # The .orig.tar.gz can disappear out from under us is it's a
2253 # duplicate of one in the archive.
2254 if not self.pkg.files.has_key(checkfile):
2257 entry = self.pkg.files[checkfile]
2259 # Check that the source still exists
2260 if entry["type"] == "deb":
2261 source_version = entry["source version"]
2262 source_package = entry["source package"]
2263 if not self.pkg.changes["architecture"].has_key("source") \
2264 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
2265 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2267 # Version and file overwrite checks
2268 if overwrite_checks:
2269 if entry["type"] == "deb":
2270 self.check_binary_against_db(checkfile, session)
2271 elif entry["type"] == "dsc":
2272 self.check_source_against_db(checkfile, session)
2273 self.check_dsc_against_db(dsc_filename, session)
2275 # propogate in the case it is in the override tables:
2276 for suite in self.pkg.changes.get("propdistribution", {}).keys():
2277 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2278 propogate[suite] = 1
2280 nopropogate[suite] = 1
2282 for suite in propogate.keys():
2283 if suite in nopropogate:
2285 self.pkg.changes["distribution"][suite] = 1
2287 for checkfile in self.pkg.files.keys():
2288 # Check the package is still in the override tables
2289 for suite in self.pkg.changes["distribution"].keys():
2290 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2291 self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2293 ################################################################################
2294 # This is not really a reject, but an unaccept, but since a) the code for
2295 # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2296 # extremely rare, for now we'll go with whining at our admin folks...
2298 def do_unaccept(self):
2302 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2303 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2304 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2305 self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
2306 if cnf.has_key("Dinstall::Bcc"):
2307 self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2309 template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2311 reject_mail_message = utils.TemplateSubst(self.Subst, template)
2313 # Write the rejection email out as the <foo>.reason file
2314 reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2315 reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2317 # If we fail here someone is probably trying to exploit the race
2318 # so let's just raise an exception ...
2319 if os.path.exists(reject_filename):
2320 os.unlink(reject_filename)
2322 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2323 os.write(fd, reject_mail_message)
2326 utils.send_mail(reject_mail_message)
2328 del self.Subst["__REJECTOR_ADDRESS__"]
2329 del self.Subst["__REJECT_MESSAGE__"]
2330 del self.Subst["__CC__"]
2332 ################################################################################
2333 # If any file of an upload has a recent mtime then chances are good
2334 # the file is still being uploaded.
2336 def upload_too_new(self):
2339 # Move back to the original directory to get accurate time stamps
2341 os.chdir(self.pkg.directory)
2342 file_list = self.pkg.files.keys()
2343 file_list.extend(self.pkg.dsc_files.keys())
2344 file_list.append(self.pkg.changes_file)
2347 last_modified = time.time()-os.path.getmtime(f)
2348 if last_modified < int(cnf["Dinstall::SkipTime"]):