5 Queue utility functions for dak
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ###############################################################################
45 from dak_exceptions import *
48 from config import Config
49 from holding import Holding
51 from summarystats import SummaryStats
52 from utils import parse_changes, check_dsc_files
53 from textutils import fix_maintainer
54 from binary import Binary
56 ###############################################################################
58 def get_type(f, session):
60 Get the file type of C{f}
63 @param f: file entry from Changes object
65 @type session: SQLA Session
66 @param session: SQL Alchemy session object
73 if f.has_key("dbtype"):
74 file_type = f["dbtype"]
75 elif re_source_ext.match(f["type"]):
78 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
80 # Validate the override type
81 type_id = get_override_type(file_type, session)
83 utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
87 ################################################################################
89 # Determine what parts in a .changes are NEW
91 def determine_new(changes, files, warn=1):
93 Determine what parts in a C{changes} file are NEW.
95 @type changes: Upload.Pkg.changes dict
96 @param changes: Changes dictionary
98 @type files: Upload.Pkg.files dict
99 @param files: Files dictionary
102 @param warn: Warn if overrides are added for (old)stable
105 @return: dictionary of NEW components.
110 session = DBConn().session()
112 # Build up a list of potentially new things
113 for name, f in files.items():
114 # Skip byhand elements
115 if f["type"] == "byhand":
118 priority = f["priority"]
119 section = f["section"]
120 file_type = get_type(f, session)
121 component = f["component"]
123 if file_type == "dsc":
126 if not new.has_key(pkg):
128 new[pkg]["priority"] = priority
129 new[pkg]["section"] = section
130 new[pkg]["type"] = file_type
131 new[pkg]["component"] = component
132 new[pkg]["files"] = []
134 old_type = new[pkg]["type"]
135 if old_type != file_type:
136 # source gets trumped by deb or udeb
137 if old_type == "dsc":
138 new[pkg]["priority"] = priority
139 new[pkg]["section"] = section
140 new[pkg]["type"] = file_type
141 new[pkg]["component"] = component
143 new[pkg]["files"].append(name)
145 if f.has_key("othercomponents"):
146 new[pkg]["othercomponents"] = f["othercomponents"]
148 for suite in changes["suite"].keys():
149 for pkg in new.keys():
150 ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
152 for file_entry in new[pkg]["files"]:
153 if files[file_entry].has_key("new"):
154 del files[file_entry]["new"]
158 for s in ['stable', 'oldstable']:
159 if changes["suite"].has_key(s):
160 print "WARNING: overrides will be added for %s!" % s
161 for pkg in new.keys():
162 if new[pkg].has_key("othercomponents"):
163 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
169 ################################################################################
171 def check_valid(new):
173 Check if section and priority for NEW packages exist in database.
174 Additionally does sanity checks:
175 - debian-installer packages have to be udeb (or source)
176 - non debian-installer packages can not be udeb
177 - source priority can only be assigned to dsc file types
180 @param new: Dict of new packages with their section, priority and type.
183 for pkg in new.keys():
184 section_name = new[pkg]["section"]
185 priority_name = new[pkg]["priority"]
186 file_type = new[pkg]["type"]
188 section = get_section(section_name)
190 new[pkg]["section id"] = -1
192 new[pkg]["section id"] = section.section_id
194 priority = get_priority(priority_name)
196 new[pkg]["priority id"] = -1
198 new[pkg]["priority id"] = priority.priority_id
201 di = section_name.find("debian-installer") != -1
203 # If d-i, we must be udeb and vice-versa
204 if (di and file_type not in ("udeb", "dsc")) or \
205 (not di and file_type == "udeb"):
206 new[pkg]["section id"] = -1
208 # If dsc we need to be source and vice-versa
209 if (priority == "source" and file_type != "dsc") or \
210 (priority != "source" and file_type == "dsc"):
211 new[pkg]["priority id"] = -1
213 ###############################################################################
215 def lookup_uid_from_fingerprint(fpr, session):
218 # This is a stupid default, but see the comments below
221 user = get_uid_from_fingerprint(fpr, session)
225 if user.name is None:
230 # Check the relevant fingerprint (which we have to have)
231 for f in user.fingerprint:
232 if f.fingerprint == fpr:
233 is_dm = f.keyring.debian_maintainer
236 return (uid, uid_name, is_dm)
238 ###############################################################################
240 # Used by Upload.check_timestamps
241 class TarTime(object):
242 def __init__(self, future_cutoff, past_cutoff):
244 self.future_cutoff = future_cutoff
245 self.past_cutoff = past_cutoff
248 self.future_files = {}
249 self.ancient_files = {}
251 def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
252 if MTime > self.future_cutoff:
253 self.future_files[Name] = MTime
254 if MTime < self.past_cutoff:
255 self.ancient_files[Name] = MTime
257 ###############################################################################
259 class Upload(object):
261 Everything that has to do with an upload processed.
269 ###########################################################################
272 """ Reset a number of internal variables."""
274 # Initialize the substitution template map
277 self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
278 self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
279 self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
280 self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
288 def package_info(self):
291 if len(self.rejects) > 0:
292 msg += "Reject Reasons:\n"
293 msg += "\n".join(self.rejects)
295 if len(self.warnings) > 0:
297 msg += "\n".join(self.warnings)
299 if len(self.notes) > 0:
301 msg += "\n".join(self.notes)
305 ###########################################################################
306 def update_subst(self):
307 """ Set up the per-package template substitution mappings """
311 # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
312 if not self.pkg.changes.has_key("architecture") or not \
313 isinstance(self.pkg.changes["architecture"], DictType):
314 self.pkg.changes["architecture"] = { "Unknown" : "" }
316 # and maintainer2047 may not exist.
317 if not self.pkg.changes.has_key("maintainer2047"):
318 self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
320 self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
321 self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
322 self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
324 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
325 if self.pkg.changes["architecture"].has_key("source") and \
326 self.pkg.changes["changedby822"] != "" and \
327 (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
329 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
330 self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
331 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
333 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
334 self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
335 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
337 if "sponsoremail" in self.pkg.changes:
338 self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
340 if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
341 self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
343 # Apply any global override of the Maintainer field
344 if cnf.get("Dinstall::OverrideMaintainer"):
345 self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
346 self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
348 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
349 self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
350 self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
352 ###########################################################################
353 def load_changes(self, filename):
356 @rvalue: whether the changes file was valid or not. We may want to
357 reject even if this is True (see what gets put in self.rejects).
358 This is simply to prevent us even trying things later which will
359 fail because we couldn't properly parse the file.
362 self.pkg.changes_file = filename
364 # Parse the .changes field into a dictionary
366 self.pkg.changes.update(parse_changes(filename))
367 except CantOpenError:
368 self.rejects.append("%s: can't read file." % (filename))
370 except ParseChangesError, line:
371 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
373 except ChangesUnicodeError:
374 self.rejects.append("%s: changes file not proper utf-8" % (filename))
377 # Parse the Files field from the .changes into another dictionary
379 self.pkg.files.update(utils.build_file_list(self.pkg.changes))
380 except ParseChangesError, line:
381 self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
383 except UnknownFormatError, format:
384 self.rejects.append("%s: unknown format '%s'." % (filename, format))
387 # Check for mandatory fields
388 for i in ("distribution", "source", "binary", "architecture",
389 "version", "maintainer", "files", "changes", "description"):
390 if not self.pkg.changes.has_key(i):
391 # Avoid undefined errors later
392 self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
395 # Strip a source version in brackets from the source field
396 if re_strip_srcver.search(self.pkg.changes["source"]):
397 self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
399 # Ensure the source field is a valid package name.
400 if not re_valid_pkg_name.match(self.pkg.changes["source"]):
401 self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
403 # Split multi-value fields into a lower-level dictionary
404 for i in ("architecture", "distribution", "binary", "closes"):
405 o = self.pkg.changes.get(i, "")
407 del self.pkg.changes[i]
409 self.pkg.changes[i] = {}
412 self.pkg.changes[i][j] = 1
414 # Fix the Maintainer: field to be RFC822/2047 compatible
416 (self.pkg.changes["maintainer822"],
417 self.pkg.changes["maintainer2047"],
418 self.pkg.changes["maintainername"],
419 self.pkg.changes["maintaineremail"]) = \
420 fix_maintainer (self.pkg.changes["maintainer"])
421 except ParseMaintError, msg:
422 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
423 % (filename, changes["maintainer"], msg))
425 # ...likewise for the Changed-By: field if it exists.
427 (self.pkg.changes["changedby822"],
428 self.pkg.changes["changedby2047"],
429 self.pkg.changes["changedbyname"],
430 self.pkg.changes["changedbyemail"]) = \
431 fix_maintainer (self.pkg.changes.get("changed-by", ""))
432 except ParseMaintError, msg:
433 self.pkg.changes["changedby822"] = ""
434 self.pkg.changes["changedby2047"] = ""
435 self.pkg.changes["changedbyname"] = ""
436 self.pkg.changes["changedbyemail"] = ""
438 self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
439 % (filename, changes["changed-by"], msg))
441 # Ensure all the values in Closes: are numbers
442 if self.pkg.changes.has_key("closes"):
443 for i in self.pkg.changes["closes"].keys():
444 if re_isanum.match (i) == None:
445 self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
447 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
448 self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
449 self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
451 # Check there isn't already a changes file of the same name in one
452 # of the queue directories.
453 base_filename = os.path.basename(filename)
454 for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
455 if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
456 self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
458 # Check the .changes is non-empty
459 if not self.pkg.files:
460 self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
463 # Changes was syntactically valid even if we'll reject
466 ###########################################################################
468 def check_distributions(self):
469 "Check and map the Distribution field"
473 # Handle suite mappings
474 for m in Cnf.ValueList("SuiteMappings"):
477 if mtype == "map" or mtype == "silent-map":
478 (source, dest) = args[1:3]
479 if self.pkg.changes["distribution"].has_key(source):
480 del self.pkg.changes["distribution"][source]
481 self.pkg.changes["distribution"][dest] = 1
482 if mtype != "silent-map":
483 self.notes.append("Mapping %s to %s." % (source, dest))
484 if self.pkg.changes.has_key("distribution-version"):
485 if self.pkg.changes["distribution-version"].has_key(source):
486 self.pkg.changes["distribution-version"][source]=dest
487 elif mtype == "map-unreleased":
488 (source, dest) = args[1:3]
489 if self.pkg.changes["distribution"].has_key(source):
490 for arch in self.pkg.changes["architecture"].keys():
491 if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
492 self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
493 del self.pkg.changes["distribution"][source]
494 self.pkg.changes["distribution"][dest] = 1
496 elif mtype == "ignore":
498 if self.pkg.changes["distribution"].has_key(suite):
499 del self.pkg.changes["distribution"][suite]
500 self.warnings.append("Ignoring %s as a target suite." % (suite))
501 elif mtype == "reject":
503 if self.pkg.changes["distribution"].has_key(suite):
504 self.rejects.append("Uploads to %s are not accepted." % (suite))
505 elif mtype == "propup-version":
506 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
508 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
509 if self.pkg.changes["distribution"].has_key(args[1]):
510 self.pkg.changes.setdefault("distribution-version", {})
511 for suite in args[2:]:
512 self.pkg.changes["distribution-version"][suite] = suite
514 # Ensure there is (still) a target distribution
515 if len(self.pkg.changes["distribution"].keys()) < 1:
516 self.rejects.append("No valid distribution remaining.")
518 # Ensure target distributions exist
519 for suite in self.pkg.changes["distribution"].keys():
520 if not Cnf.has_key("Suite::%s" % (suite)):
521 self.rejects.append("Unknown distribution `%s'." % (suite))
523 ###########################################################################
525 def binary_file_checks(self, f, session):
527 entry = self.pkg.files[f]
529 # Extract package control information
530 deb_file = utils.open_file(f)
532 control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
534 self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
536 # Can't continue, none of the checks on control would work.
539 # Check for mandantory "Description:"
542 apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
544 self.rejects.append("%s: Missing Description in binary package" % (f))
549 # Check for mandatory fields
550 for field in [ "Package", "Architecture", "Version" ]:
551 if control.Find(field) == None:
553 self.rejects.append("%s: No %s field in control." % (f, field))
556 # Ensure the package name matches the one give in the .changes
557 if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
558 self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
560 # Validate the package field
561 package = control.Find("Package")
562 if not re_valid_pkg_name.match(package):
563 self.rejects.append("%s: invalid package name '%s'." % (f, package))
565 # Validate the version field
566 version = control.Find("Version")
567 if not re_valid_version.match(version):
568 self.rejects.append("%s: invalid version number '%s'." % (f, version))
570 # Ensure the architecture of the .deb is one we know about.
571 default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
572 architecture = control.Find("Architecture")
573 upload_suite = self.pkg.changes["distribution"].keys()[0]
575 if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
576 and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
577 self.rejects.append("Unknown architecture '%s'." % (architecture))
579 # Ensure the architecture of the .deb is one of the ones
580 # listed in the .changes.
581 if not self.pkg.changes["architecture"].has_key(architecture):
582 self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
584 # Sanity-check the Depends field
585 depends = control.Find("Depends")
587 self.rejects.append("%s: Depends field is empty." % (f))
589 # Sanity-check the Provides field
590 provides = control.Find("Provides")
592 provide = re_spacestrip.sub('', provides)
594 self.rejects.append("%s: Provides field is empty." % (f))
595 prov_list = provide.split(",")
596 for prov in prov_list:
597 if not re_valid_pkg_name.match(prov):
598 self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
600 # Check the section & priority match those given in the .changes (non-fatal)
601 if control.Find("Section") and entry["section"] != "" \
602 and entry["section"] != control.Find("Section"):
603 self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
604 (f, control.Find("Section", ""), entry["section"]))
605 if control.Find("Priority") and entry["priority"] != "" \
606 and entry["priority"] != control.Find("Priority"):
607 self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
608 (f, control.Find("Priority", ""), entry["priority"]))
610 entry["package"] = package
611 entry["architecture"] = architecture
612 entry["version"] = version
613 entry["maintainer"] = control.Find("Maintainer", "")
615 if f.endswith(".udeb"):
616 self.pkg.files[f]["dbtype"] = "udeb"
617 elif f.endswith(".deb"):
618 self.pkg.files[f]["dbtype"] = "deb"
620 self.rejects.append("%s is neither a .deb or a .udeb." % (f))
622 entry["source"] = control.Find("Source", entry["package"])
624 # Get the source version
625 source = entry["source"]
628 if source.find("(") != -1:
629 m = re_extract_src_version.match(source)
631 source_version = m.group(2)
633 if not source_version:
634 source_version = self.pkg.files[f]["version"]
636 entry["source package"] = source
637 entry["source version"] = source_version
639 # Ensure the filename matches the contents of the .deb
640 m = re_isadeb.match(f)
643 file_package = m.group(1)
644 if entry["package"] != file_package:
645 self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
646 (f, file_package, entry["dbtype"], entry["package"]))
647 epochless_version = re_no_epoch.sub('', control.Find("Version"))
650 file_version = m.group(2)
651 if epochless_version != file_version:
652 self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
653 (f, file_version, entry["dbtype"], epochless_version))
656 file_architecture = m.group(3)
657 if entry["architecture"] != file_architecture:
658 self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
659 (f, file_architecture, entry["dbtype"], entry["architecture"]))
661 # Check for existent source
662 source_version = entry["source version"]
663 source_package = entry["source package"]
664 if self.pkg.changes["architecture"].has_key("source"):
665 if source_version != self.pkg.changes["version"]:
666 self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
667 (source_version, f, self.pkg.changes["version"]))
669 # Check in the SQL database
670 if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
671 # Check in one of the other directories
672 source_epochless_version = re_no_epoch.sub('', source_version)
673 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
674 if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
676 elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
679 dsc_file_exists = False
680 for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
681 if cnf.has_key("Dir::Queue::%s" % (myq)):
682 if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
683 dsc_file_exists = True
686 if not dsc_file_exists:
687 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
689 # Check the version and for file overwrites
690 self.check_binary_against_db(f, session)
692 # Temporarily disable contents generation until we change the table storage layout
695 #if len(b.rejects) > 0:
696 # for j in b.rejects:
697 # self.rejects.append(j)
699 def source_file_checks(self, f, session):
700 entry = self.pkg.files[f]
702 m = re_issource.match(f)
706 entry["package"] = m.group(1)
707 entry["version"] = m.group(2)
708 entry["type"] = m.group(3)
710 # Ensure the source package name matches the Source filed in the .changes
711 if self.pkg.changes["source"] != entry["package"]:
712 self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
714 # Ensure the source version matches the version in the .changes file
715 if re_is_orig_source.match(f):
716 changes_version = self.pkg.changes["chopversion2"]
718 changes_version = self.pkg.changes["chopversion"]
720 if changes_version != entry["version"]:
721 self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
723 # Ensure the .changes lists source in the Architecture field
724 if not self.pkg.changes["architecture"].has_key("source"):
725 self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
727 # Check the signature of a .dsc file
728 if entry["type"] == "dsc":
729 # check_signature returns either:
730 # (None, [list, of, rejects]) or (signature, [])
731 (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
733 self.rejects.append(j)
735 entry["architecture"] = "source"
737 def per_suite_file_checks(self, f, suite, session):
739 entry = self.pkg.files[f]
740 archive = utils.where_am_i()
743 if entry.has_key("byhand"):
746 # Check we have fields we need to do these checks
748 for m in ['component', 'package', 'priority', 'size', 'md5sum']:
749 if not entry.has_key(m):
750 self.rejects.append("file '%s' does not have field %s set" % (f, m))
756 # Handle component mappings
757 for m in cnf.ValueList("ComponentMappings"):
758 (source, dest) = m.split()
759 if entry["component"] == source:
760 entry["original component"] = source
761 entry["component"] = dest
763 # Ensure the component is valid for the target suite
764 if cnf.has_key("Suite:%s::Components" % (suite)) and \
765 entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
766 self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
769 # Validate the component
770 if not get_component(entry["component"], session):
771 self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
774 # See if the package is NEW
775 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
778 # Validate the priority
779 if entry["priority"].find('/') != -1:
780 self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
782 # Determine the location
783 location = cnf["Dir::Pool"]
784 l = get_location(location, entry["component"], archive, session)
786 self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
787 entry["location id"] = -1
789 entry["location id"] = l.location_id
791 # Check the md5sum & size against existing files (if any)
792 entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
794 found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
795 entry["size"], entry["md5sum"], entry["location id"])
798 self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
799 elif found is False and poolfile is not None:
800 self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
803 entry["files id"] = None
805 entry["files id"] = poolfile.file_id
807 # Check for packages that have moved from one component to another
808 entry['suite'] = suite
809 res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
811 entry["othercomponents"] = res.fetchone()[0]
813 def check_files(self, action=True):
814 archive = utils.where_am_i()
815 file_keys = self.pkg.files.keys()
819 # XXX: As far as I can tell, this can no longer happen - see
820 # comments by AJ in old revisions - mhy
821 # if reprocess is 2 we've already done this and we're checking
822 # things again for the new .orig.tar.gz.
823 # [Yes, I'm fully aware of how disgusting this is]
824 if action and self.reprocess < 2:
826 os.chdir(self.pkg.directory)
828 ret = holding.copy_to_holding(f)
830 # XXX: Should we bail out here or try and continue?
831 self.rejects.append(ret)
835 # Check there isn't already a .changes or .dak file of the same name in
836 # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
837 # [NB: this check must be done post-suite mapping]
838 base_filename = os.path.basename(self.pkg.changes_file)
839 dot_dak_filename = base_filename[:-8] + ".dak"
841 for suite in self.pkg.changes["distribution"].keys():
842 copychanges = "Suite::%s::CopyChanges" % (suite)
843 if cnf.has_key(copychanges) and \
844 os.path.exists(os.path.join(cnf[copychanges], base_filename)):
845 self.rejects.append("%s: a file with this name already exists in %s" \
846 % (base_filename, cnf[copychanges]))
848 copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
849 if cnf.has_key(copy_dot_dak) and \
850 os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
851 self.rejects.append("%s: a file with this name already exists in %s" \
852 % (dot_dak_filename, Cnf[copy_dot_dak]))
858 session = DBConn().session()
860 for f, entry in self.pkg.files.items():
861 # Ensure the file does not already exist in one of the accepted directories
862 for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
863 if not cnf.has_key("Dir::Queue::%s" % (d)): continue
864 if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
865 self.rejects.append("%s file already exists in the %s directory." % (f, d))
867 if not re_taint_free.match(f):
868 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
870 # Check the file is readable
871 if os.access(f, os.R_OK) == 0:
872 # When running in -n, copy_to_holding() won't have
873 # generated the reject_message, so we need to.
875 if os.path.exists(f):
876 self.rejects.append("Can't read `%s'. [permission denied]" % (f))
878 self.rejects.append("Can't read `%s'. [file not found]" % (f))
879 entry["type"] = "unreadable"
882 # If it's byhand skip remaining checks
883 if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
885 entry["type"] = "byhand"
887 # Checks for a binary package...
888 elif re_isadeb.match(f):
890 entry["type"] = "deb"
892 # This routine appends to self.rejects/warnings as appropriate
893 self.binary_file_checks(f, session)
895 # Checks for a source package...
896 elif re_issource.match(f):
899 # This routine appends to self.rejects/warnings as appropriate
900 self.source_file_checks(f, session)
902 # Not a binary or source package? Assume byhand...
905 entry["type"] = "byhand"
907 # Per-suite file checks
908 entry["oldfiles"] = {}
909 for suite in self.pkg.changes["distribution"].keys():
910 self.per_suite_file_checks(f, suite, session)
914 # If the .changes file says it has source, it must have source.
915 if self.pkg.changes["architecture"].has_key("source"):
917 self.rejects.append("no source found and Architecture line in changes mention source.")
919 if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
920 self.rejects.append("source only uploads are not supported.")
922 ###########################################################################
923 def check_dsc(self, action=True, session=None):
924 """Returns bool indicating whether or not the source changes are valid"""
925 # Ensure there is source to check
926 if not self.pkg.changes["architecture"].has_key("source"):
931 for f, entry in self.pkg.files.items():
932 if entry["type"] == "dsc":
934 self.rejects.append("can not process a .changes file with multiple .dsc's.")
939 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
941 self.rejects.append("source uploads must contain a dsc file")
944 # Parse the .dsc file
946 self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
947 except CantOpenError:
948 # if not -n copy_to_holding() will have done this for us...
950 self.rejects.append("%s: can't read file." % (dsc_filename))
951 except ParseChangesError, line:
952 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
953 except InvalidDscError, line:
954 self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
955 except ChangesUnicodeError:
956 self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
958 # Build up the file list of files mentioned by the .dsc
960 self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
961 except NoFilesFieldError:
962 self.rejects.append("%s: no Files: field." % (dsc_filename))
964 except UnknownFormatError, format:
965 self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
967 except ParseChangesError, line:
968 self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
971 # Enforce mandatory fields
972 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
973 if not self.pkg.dsc.has_key(i):
974 self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
977 # Validate the source and version fields
978 if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
979 self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
980 if not re_valid_version.match(self.pkg.dsc["version"]):
981 self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
983 # Only a limited list of source formats are allowed in each suite
984 for dist in self.pkg.changes["distribution"].keys():
985 allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
986 if self.pkg.dsc["format"] not in allowed:
987 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
989 # Validate the Maintainer field
991 # We ignore the return value
992 fix_maintainer(self.pkg.dsc["maintainer"])
993 except ParseMaintError, msg:
994 self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
995 % (dsc_filename, self.pkg.dsc["maintainer"], msg))
997 # Validate the build-depends field(s)
998 for field_name in [ "build-depends", "build-depends-indep" ]:
999 field = self.pkg.dsc.get(field_name)
1001 # Have apt try to parse them...
1003 apt_pkg.ParseSrcDepends(field)
1005 self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
1007 # Ensure the version number in the .dsc matches the version number in the .changes
1008 epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1009 changes_version = self.pkg.files[dsc_filename]["version"]
1011 if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
1012 self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
1014 # Ensure the Files field contain only what's expected
1015 self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
1017 # Ensure source is newer than existing source in target suites
1018 session = DBConn().session()
1019 self.check_source_against_db(dsc_filename, session)
1020 self.check_dsc_against_db(dsc_filename, session)
1025 ###########################################################################
1027 def get_changelog_versions(self, source_dir):
1028 """Extracts a the source package and (optionally) grabs the
1029 version history out of debian/changelog for the BTS."""
1033 # Find the .dsc (again)
1035 for f in self.pkg.files.keys():
1036 if self.pkg.files[f]["type"] == "dsc":
1039 # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
1040 if not dsc_filename:
1043 # Create a symlink mirror of the source files in our temporary directory
1044 for f in self.pkg.files.keys():
1045 m = re_issource.match(f)
1047 src = os.path.join(source_dir, f)
1048 # If a file is missing for whatever reason, give up.
1049 if not os.path.exists(src):
1052 if re_is_orig_source.match(f) and pkg.orig_files.has_key(f) and \
1053 pkg.orig_files[f].has_key("path"):
1055 dest = os.path.join(os.getcwd(), f)
1056 os.symlink(src, dest)
1058 # If the orig files are not a part of the upload, create symlinks to the
1060 for orig_file in self.pkg.orig_files.keys():
1061 if not self.pkg.orig_files[orig_file].has_key("path"):
1063 dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
1064 os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
1066 # Extract the source
1067 cmd = "dpkg-source -sn -x %s" % (dsc_filename)
1068 (result, output) = commands.getstatusoutput(cmd)
1070 self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
1071 self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
1074 if not cnf.Find("Dir::Queue::BTSVersionTrack"):
1077 # Get the upstream version
1078 upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
1079 if re_strip_revision.search(upstr_version):
1080 upstr_version = re_strip_revision.sub('', upstr_version)
1082 # Ensure the changelog file exists
1083 changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
1084 if not os.path.exists(changelog_filename):
1085 self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
1088 # Parse the changelog
1089 self.pkg.dsc["bts changelog"] = ""
1090 changelog_file = utils.open_file(changelog_filename)
1091 for line in changelog_file.readlines():
1092 m = re_changelog_versions.match(line)
1094 self.pkg.dsc["bts changelog"] += line
1095 changelog_file.close()
1097 # Check we found at least one revision in the changelog
1098 if not self.pkg.dsc["bts changelog"]:
1099 self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
1101 def check_source(self):
1102 # XXX: I'm fairly sure reprocess == 2 can never happen
1103 # AJT disabled the is_incoming check years ago - mhy
1104 # We should probably scrap or rethink the whole reprocess thing
1106 # a) there's no source
1107 # or b) reprocess is 2 - we will do this check next time when orig
1108 # tarball is in 'files'
1109 # or c) the orig files are MIA
1110 if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
1111 or len(self.pkg.orig_files) == 0:
1114 tmpdir = utils.temp_dirname()
1116 # Move into the temporary directory
1120 # Get the changelog version history
1121 self.get_changelog_versions(cwd)
1123 # Move back and cleanup the temporary tree
1127 shutil.rmtree(tmpdir)
1129 if e.errno != errno.EACCES:
1131 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1133 self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
1134 # We probably have u-r or u-w directories so chmod everything
1136 cmd = "chmod -R u+rwx %s" % (tmpdir)
1137 result = os.system(cmd)
1139 utils.fubar("'%s' failed with result %s." % (cmd, result))
1140 shutil.rmtree(tmpdir)
1141 except Exception, e:
1142 print "foobar2 (%s)" % e
1143 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
1145 ###########################################################################
1146 def ensure_hashes(self):
1147 # Make sure we recognise the format of the Files: field in the .changes
1148 format = self.pkg.changes.get("format", "0.0").split(".", 1)
1149 if len(format) == 2:
1150 format = int(format[0]), int(format[1])
1152 format = int(float(format[0])), 0
1154 # We need to deal with the original changes blob, as the fields we need
1155 # might not be in the changes dict serialised into the .dak anymore.
1156 orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
1158 # Copy the checksums over to the current changes dict. This will keep
1159 # the existing modifications to it intact.
1160 for field in orig_changes:
1161 if field.startswith('checksums-'):
1162 self.pkg.changes[field] = orig_changes[field]
1164 # Check for unsupported hashes
1165 for j in utils.check_hash_fields(".changes", self.pkg.changes):
1166 self.rejects.append(j)
1168 for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
1169 self.rejects.append(j)
1171 # We have to calculate the hash if we have an earlier changes version than
1172 # the hash appears in rather than require it exist in the changes file
1173 for hashname, hashfunc, version in utils.known_hashes:
1174 # TODO: Move _ensure_changes_hash into this class
1175 for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
1176 self.rejects.append(j)
1177 if "source" in self.pkg.changes["architecture"]:
1178 # TODO: Move _ensure_dsc_hash into this class
1179 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
1180 self.rejects.append(j)
1182 def check_hashes(self):
1183 for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
1184 self.rejects.append(m)
1186 for m in utils.check_size(".changes", self.pkg.files):
1187 self.rejects.append(m)
1189 for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
1190 self.rejects.append(m)
1192 for m in utils.check_size(".dsc", self.pkg.dsc_files):
1193 self.rejects.append(m)
1195 self.ensure_hashes()
1197 ###########################################################################
1198 def check_urgency(self):
1200 if self.pkg.changes["architecture"].has_key("source"):
1201 if not self.pkg.changes.has_key("urgency"):
1202 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1203 self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
1204 if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
1205 self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
1206 (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
1207 self.pkg.changes["urgency"] = cnf["Urgency::Default"]
1209 ###########################################################################
1211 # Sanity check the time stamps of files inside debs.
1212 # [Files in the near future cause ugly warnings and extreme time
1213 # travel can cause errors on extraction]
1215 def check_timestamps(self):
1218 future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
1219 past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
1220 tar = TarTime(future_cutoff, past_cutoff)
1222 for filename, entry in self.pkg.files.items():
1223 if entry["type"] == "deb":
1226 deb_file = utils.open_file(filename)
1227 apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
1230 apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
1231 except SystemError, e:
1232 # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
1233 if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
1236 apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
1240 future_files = tar.future_files.keys()
1242 num_future_files = len(future_files)
1243 future_file = future_files[0]
1244 future_date = tar.future_files[future_file]
1245 self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
1246 % (filename, num_future_files, future_file, time.ctime(future_date)))
1248 ancient_files = tar.ancient_files.keys()
1250 num_ancient_files = len(ancient_files)
1251 ancient_file = ancient_files[0]
1252 ancient_date = tar.ancient_files[ancient_file]
1253 self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
1254 % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
1256 self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
1258 ###########################################################################
1259 def check_transition(self, session):
1262 sourcepkg = self.pkg.changes["source"]
1264 # No sourceful upload -> no need to do anything else, direct return
1265 # We also work with unstable uploads, not experimental or those going to some
1266 # proposed-updates queue
1267 if "source" not in self.pkg.changes["architecture"] or \
1268 "unstable" not in self.pkg.changes["distribution"]:
1271 # Also only check if there is a file defined (and existant) with
1273 transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
1274 if transpath == "" or not os.path.exists(transpath):
1277 # Parse the yaml file
1278 sourcefile = file(transpath, 'r')
1279 sourcecontent = sourcefile.read()
1281 transitions = yaml.load(sourcecontent)
1282 except yaml.YAMLError, msg:
1283 # This shouldn't happen, there is a wrapper to edit the file which
1284 # checks it, but we prefer to be safe than ending up rejecting
1286 utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
1289 # Now look through all defined transitions
1290 for trans in transitions:
1291 t = transitions[trans]
1292 source = t["source"]
1295 # Will be None if nothing is in testing.
1296 current = get_source_in_suite(source, "testing", session)
1297 if current is not None:
1298 compare = apt_pkg.VersionCompare(current.version, expected)
1300 if current is None or compare < 0:
1301 # This is still valid, the current version in testing is older than
1302 # the new version we wait for, or there is none in testing yet
1304 # Check if the source we look at is affected by this.
1305 if sourcepkg in t['packages']:
1306 # The source is affected, lets reject it.
1308 rejectmsg = "%s: part of the %s transition.\n\n" % (
1311 if current is not None:
1312 currentlymsg = "at version %s" % (current.version)
1314 currentlymsg = "not present in testing"
1316 rejectmsg += "Transition description: %s\n\n" % (t["reason"])
1318 rejectmsg += "\n".join(textwrap.wrap("""Your package
1319 is part of a testing transition designed to get %s migrated (it is
1320 currently %s, we need version %s). This transition is managed by the
1321 Release Team, and %s is the Release-Team member responsible for it.
1322 Please mail debian-release@lists.debian.org or contact %s directly if you
1323 need further assistance. You might want to upload to experimental until this
1324 transition is done."""
1325 % (source, currentlymsg, expected,t["rm"], t["rm"])))
1327 self.rejects.append(rejectmsg)
1330 ###########################################################################
1331 def check_signed_by_key(self):
1332 """Ensure the .changes is signed by an authorized uploader."""
1333 session = DBConn().session()
1335 self.check_transition(session)
1337 (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
1339 # match claimed name with actual name:
1341 # This is fundamentally broken but need us to refactor how we get
1342 # the UIDs/Fingerprints in order for us to fix it properly
1343 uid, uid_email = self.pkg.changes["fingerprint"], uid
1344 may_nmu, may_sponsor = 1, 1
1345 # XXX by default new dds don't have a fingerprint/uid in the db atm,
1346 # and can't get one in there if we don't allow nmu/sponsorship
1347 elif is_dm is False:
1348 # If is_dm is False, we allow full upload rights
1349 uid_email = "%s@debian.org" % (uid)
1350 may_nmu, may_sponsor = 1, 1
1352 # Assume limited upload rights unless we've discovered otherwise
1354 may_nmu, may_sponsor = 0, 0
1356 if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
1358 elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
1360 if uid_name == "": sponsored = 1
1363 if ("source" in self.pkg.changes["architecture"] and
1364 uid_email and utils.is_email_alias(uid_email)):
1365 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
1366 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
1367 self.pkg.changes["changedbyemail"] not in sponsor_addresses):
1368 self.pkg.changes["sponsoremail"] = uid_email
1370 if sponsored and not may_sponsor:
1371 self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
1373 if not sponsored and not may_nmu:
1374 should_reject = True
1375 highest_sid, highest_version = None, None
1377 # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
1378 # It ignores higher versions with the dm_upload_allowed flag set to false
1379 # I'm keeping the existing behaviour for now until I've gone back and
1380 # checked exactly what the GR says - mhy
1381 for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
1382 if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
1383 highest_sid = si.source_id
1384 highest_version = si.version
1386 if highest_sid is None:
1387 self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
1389 for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
1390 (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
1391 if email == uid_email or name == uid_name:
1392 should_reject = False
1395 if should_reject is True:
1396 self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
1398 for b in self.pkg.changes["binary"].keys():
1399 for suite in self.pkg.changes["distribution"].keys():
1400 q = session.query(DBSource)
1401 q = q.join(DBBinary).filter_by(package=b)
1402 q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
1405 if s.source != self.pkg.changes["source"]:
1406 self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
1408 for f in self.pkg.files.keys():
1409 if self.pkg.files[f].has_key("byhand"):
1410 self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
1411 if self.pkg.files[f].has_key("new"):
1412 self.rejects.append("%s may not upload NEW file %s" % (uid, f))
1416 ###########################################################################
1417 def build_summaries(self):
1418 """ Build a summary of changes the upload introduces. """
1420 (byhand, new, summary, override_summary) = self.pkg.file_summary()
1422 short_summary = summary
1424 # This is for direport's benefit...
1425 f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
1428 summary += "Changes: " + f
1430 summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
1432 summary += self.announce(short_summary, 0)
1434 return (summary, short_summary)
1436 ###########################################################################
1438 def close_bugs(self, summary, action):
1440 Send mail to close bugs as instructed by the closes field in the changes file.
1441 Also add a line to summary if any work was done.
1443 @type summary: string
1444 @param summary: summary text, as given by L{build_summaries}
1447 @param action: Set to false no real action will be done.
1450 @return: summary. If action was taken, extended by the list of closed bugs.
1454 template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
1456 bugs = self.pkg.changes["closes"].keys()
1462 summary += "Closing bugs: "
1464 summary += "%s " % (bug)
1467 self.Subst["__BUG_NUMBER__"] = bug
1468 if self.pkg.changes["distribution"].has_key("stable"):
1469 self.Subst["__STABLE_WARNING__"] = """
1470 Note that this package is not part of the released stable Debian
1471 distribution. It may have dependencies on other unreleased software,
1472 or other instabilities. Please take care if you wish to install it.
1473 The update will eventually make its way into the next released Debian
1476 self.Subst["__STABLE_WARNING__"] = ""
1477 mail_message = utils.TemplateSubst(self.Subst, template)
1478 utils.send_mail(mail_message)
1480 # Clear up after ourselves
1481 del self.Subst["__BUG_NUMBER__"]
1482 del self.Subst["__STABLE_WARNING__"]
1484 if action and self.logger:
1485 self.logger.log(["closing bugs"] + bugs)
1491 ###########################################################################
1493 def announce(self, short_summary, action):
1495 Send an announce mail about a new upload.
1497 @type short_summary: string
1498 @param short_summary: Short summary text to include in the mail
1501 @param action: Set to false no real action will be done.
1504 @return: Textstring about action taken.
1509 announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
1511 # Only do announcements for source uploads with a recent dpkg-dev installed
1512 if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
1513 self.pkg.changes["architecture"].has_key("source"):
1519 self.Subst["__SHORT_SUMMARY__"] = short_summary
1521 for dist in self.pkg.changes["distribution"].keys():
1522 announce_list = cnf.Find("Suite::%s::Announce" % (dist))
1523 if announce_list == "" or lists_done.has_key(announce_list):
1526 lists_done[announce_list] = 1
1527 summary += "Announcing to %s\n" % (announce_list)
1531 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
1532 if cnf.get("Dinstall::TrackingServer") and \
1533 self.pkg.changes["architecture"].has_key("source"):
1534 trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
1535 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
1537 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
1538 utils.send_mail(mail_message)
1540 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
1542 if cnf.FindB("Dinstall::CloseBugs"):
1543 summary = self.close_bugs(summary, action)
1545 del self.Subst["__SHORT_SUMMARY__"]
1549 ###########################################################################
1551 def accept (self, summary, short_summary, targetdir=None):
1555 This moves all files referenced from the .changes into the I{accepted}
1556 queue, sends the accepted mail, announces to lists, closes bugs and
1557 also checks for override disparities. If enabled it will write out
1558 the version history for the BTS Version Tracking and will finally call
1561 @type summary: string
1562 @param summary: Summary text
1564 @type short_summary: string
1565 @param short_summary: Short summary
1570 stats = SummaryStats()
1572 accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
1574 if targetdir is None:
1575 targetdir = cnf["Dir::Queue::Accepted"]
1579 self.logger.log(["Accepting changes", self.pkg.changes_file])
1581 self.pkg.write_dot_dak(targetdir)
1583 # Move all the files into the accepted directory
1584 utils.move(self.pkg.changes_file, targetdir)
1586 for name, entry in sorted(self.pkg.files.items()):
1587 utils.move(name, targetdir)
1588 stats.accept_bytes += float(entry["size"])
1590 stats.accept_count += 1
1592 # Send accept mail, announce to lists, close bugs and check for
1593 # override disparities
1594 if not cnf["Dinstall::Options::No-Mail"]:
1596 self.Subst["__SUITE__"] = ""
1597 self.Subst["__SUMMARY__"] = summary
1598 mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
1599 utils.send_mail(mail_message)
1600 self.announce(short_summary, 1)
1602 ## Helper stuff for DebBugs Version Tracking
1603 if cnf.Find("Dir::Queue::BTSVersionTrack"):
1604 # ??? once queue/* is cleared on *.d.o and/or reprocessed
1605 # the conditionalization on dsc["bts changelog"] should be
1608 # Write out the version history from the changelog
1609 if self.pkg.changes["architecture"].has_key("source") and \
1610 self.pkg.dsc.has_key("bts changelog"):
1612 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1613 version_history = os.fdopen(fd, 'w')
1614 version_history.write(self.pkg.dsc["bts changelog"])
1615 version_history.close()
1616 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1617 self.pkg.changes_file[:-8]+".versions")
1618 os.rename(temp_filename, filename)
1619 os.chmod(filename, 0644)
1621 # Write out the binary -> source mapping.
1622 (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
1623 debinfo = os.fdopen(fd, 'w')
1624 for name, entry in sorted(self.pkg.files.items()):
1625 if entry["type"] == "deb":
1626 line = " ".join([entry["package"], entry["version"],
1627 entry["architecture"], entry["source package"],
1628 entry["source version"]])
1629 debinfo.write(line+"\n")
1631 filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
1632 self.pkg.changes_file[:-8]+".debinfo")
1633 os.rename(temp_filename, filename)
1634 os.chmod(filename, 0644)
1636 # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
1637 # <Ganneff> we do call queue_build too
1638 # <mhy> well yes, we'd have had to if we were inserting into accepted
1639 # <Ganneff> now. thats database only.
1640 # <mhy> urgh, that's going to get messy
1641 # <Ganneff> so i make the p-n call to it *also* using accepted/
1642 # <mhy> but then the packages will be in the queue_build table without the files being there
1643 # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
1644 # <mhy> ah, good point
1645 # <Ganneff> so it will work out, as unchecked move it over
1646 # <mhy> that's all completely sick
1649 # This routine returns None on success or an error on failure
1650 res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
1655 def check_override(self):
1657 Checks override entries for validity. Mails "Override disparity" warnings,
1658 if that feature is enabled.
1660 Abandons the check if
1661 - override disparity checks are disabled
1662 - mail sending is disabled
1667 # Abandon the check if:
1668 # a) override disparity checks have been disabled
1669 # b) we're not sending mail
1670 if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
1671 cnf["Dinstall::Options::No-Mail"]:
1674 summary = self.pkg.check_override()
1679 overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
1682 self.Subst["__SUMMARY__"] = summary
1683 mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
1684 utils.send_mail(mail_message)
1685 del self.Subst["__SUMMARY__"]
1687 ###########################################################################
1689 def remove(self, dir=None):
1691 Used (for instance) in p-u to remove the package from unchecked
1694 os.chdir(self.pkg.directory)
1698 for f in self.pkg.files.keys():
1700 os.unlink(self.pkg.changes_file)
1702 ###########################################################################
1704 def move_to_dir (self, dest, perms=0660, changesperms=0664):
1706 Move files to dest with certain perms/changesperms
1708 utils.move(self.pkg.changes_file, dest, perms=changesperms)
1709 for f in self.pkg.files.keys():
1710 utils.move(f, dest, perms=perms)
1712 ###########################################################################
1714 def force_reject(self, reject_files):
1716 Forcefully move files from the current directory to the
1717 reject directory. If any file already exists in the reject
1718 directory it will be moved to the morgue to make way for
1722 @param files: file dictionary
1728 for file_entry in reject_files:
1729 # Skip any files which don't exist or which we don't have permission to copy.
1730 if os.access(file_entry, os.R_OK) == 0:
1733 dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
1736 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
1738 # File exists? Let's try and move it to the morgue
1739 if e.errno == errno.EEXIST:
1740 morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
1742 morgue_file = utils.find_next_free(morgue_file)
1743 except NoFreeFilenameError:
1744 # Something's either gone badly Pete Tong, or
1745 # someone is trying to exploit us.
1746 utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
1748 utils.move(dest_file, morgue_file, perms=0660)
1750 dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1753 utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
1757 # If we got here, we own the destination file, so we can
1758 # safely overwrite it.
1759 utils.move(file_entry, dest_file, 1, perms=0660)
1762 ###########################################################################
1763 def do_reject (self, manual=0, reject_message="", note=""):
1765 Reject an upload. If called without a reject message or C{manual} is
1766 true, spawn an editor so the user can write one.
1769 @param manual: manual or automated rejection
1771 @type reject_message: string
1772 @param reject_message: A reject message
1777 # If we weren't given a manual rejection message, spawn an
1778 # editor so the user can add one in...
1779 if manual and not reject_message:
1780 (fd, temp_filename) = utils.temp_filename()
1781 temp_file = os.fdopen(fd, 'w')
1784 temp_file.write(line)
1786 editor = os.environ.get("EDITOR","vi")
1788 while answer == 'E':
1789 os.system("%s %s" % (editor, temp_filename))
1790 temp_fh = utils.open_file(temp_filename)
1791 reject_message = "".join(temp_fh.readlines())
1793 print "Reject message:"
1794 print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1)
1795 prompt = "[R]eject, Edit, Abandon, Quit ?"
1797 while prompt.find(answer) == -1:
1798 answer = utils.our_raw_input(prompt)
1799 m = re_default_answer.search(prompt)
1802 answer = answer[:1].upper()
1803 os.unlink(temp_filename)
1809 print "Rejecting.\n"
1813 reason_filename = self.pkg.changes_file[:-8] + ".reason"
1814 reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
1816 # Move all the files into the reject directory
1817 reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
1818 self.force_reject(reject_files)
1820 # If we fail here someone is probably trying to exploit the race
1821 # so let's just raise an exception ...
1822 if os.path.exists(reason_filename):
1823 os.unlink(reason_filename)
1824 reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
1826 rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
1830 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
1831 self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
1832 self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
1833 os.write(reason_fd, reject_message)
1834 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1836 # Build up the rejection email
1837 user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
1838 self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
1839 self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
1840 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
1841 reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
1842 # Write the rejection email out as the <foo>.reason file
1843 os.write(reason_fd, reject_mail_message)
1845 del self.Subst["__REJECTOR_ADDRESS__"]
1846 del self.Subst["__MANUAL_REJECT_MESSAGE__"]
1847 del self.Subst["__CC__"]
1851 # Send the rejection mail if appropriate
1852 if not cnf["Dinstall::Options::No-Mail"]:
1853 utils.send_mail(reject_mail_message)
1856 self.logger.log(["rejected", self.pkg.changes_file])
1860 ################################################################################
1861 def in_override_p(self, package, component, suite, binary_type, file, session):
1863 Check if a package already has override entries in the DB
1865 @type package: string
1866 @param package: package name
1868 @type component: string
1869 @param component: database id of the component
1872 @param suite: database id of the suite
1874 @type binary_type: string
1875 @param binary_type: type of the package
1878 @param file: filename we check
1880 @return: the database result. But noone cares anyway.
1886 if binary_type == "": # must be source
1889 file_type = binary_type
1891 # Override suite name; used for example with proposed-updates
1892 if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
1893 suite = cnf["Suite::%s::OverrideSuite" % (suite)]
1895 result = get_override(package, suite, component, file_type, session)
1897 # If checking for a source package fall back on the binary override type
1898 if file_type == "dsc" and len(result) < 1:
1899 result = get_override(package, suite, component, ['deb', 'udeb'], session)
1901 # Remember the section and priority so we can check them later if appropriate
1904 self.pkg.files[file]["override section"] = result.section.section
1905 self.pkg.files[file]["override priority"] = result.priority.priority
1910 ################################################################################
1911 def get_anyversion(self, sv_list, suite):
1914 @param sv_list: list of (suite, version) tuples to check
1917 @param suite: suite name
1923 anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
1924 for (s, v) in sv_list:
1925 if s in [ x.lower() for x in anysuite ]:
1926 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
1931 ################################################################################
1933 def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
1936 @param sv_list: list of (suite, version) tuples to check
1941 @type new_version: string
1942 @param new_version: XXX
1944 Ensure versions are newer than existing packages in target
1945 suites and that cross-suite version checking rules as
1946 set out in the conf file are satisfied.
1951 # Check versions for each target suite
1952 for target_suite in self.pkg.changes["distribution"].keys():
1953 must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
1954 must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
1956 # Enforce "must be newer than target suite" even if conffile omits it
1957 if target_suite not in must_be_newer_than:
1958 must_be_newer_than.append(target_suite)
1960 for (suite, existent_version) in sv_list:
1961 vercmp = apt_pkg.VersionCompare(new_version, existent_version)
1963 if suite in must_be_newer_than and sourceful and vercmp < 1:
1964 self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
1966 if suite in must_be_older_than and vercmp > -1:
1969 if self.pkg.changes.get('distribution-version', {}).has_key(suite):
1970 # we really use the other suite, ignoring the conflicting one ...
1971 addsuite = self.pkg.changes["distribution-version"][suite]
1973 add_version = self.get_anyversion(sv_list, addsuite)
1974 target_version = self.get_anyversion(sv_list, target_suite)
1977 # not add_version can only happen if we map to a suite
1978 # that doesn't enhance the suite we're propup'ing from.
1979 # so "propup-ver x a b c; map a d" is a problem only if
1980 # d doesn't enhance a.
1982 # i think we could always propagate in this case, rather
1983 # than complaining. either way, this isn't a REJECT issue
1985 # And - we really should complain to the dorks who configured dak
1986 self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
1987 self.pkg.changes.setdefault("propdistribution", {})
1988 self.pkg.changes["propdistribution"][addsuite] = 1
1990 elif not target_version:
1991 # not targets_version is true when the package is NEW
1992 # we could just stick with the "...old version..." REJECT
1993 # for this, I think.
1994 self.rejects.append("Won't propogate NEW packages.")
1995 elif apt_pkg.VersionCompare(new_version, add_version) < 0:
1996 # propogation would be redundant. no need to reject though.
1997 self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
1999 elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
2000 apt_pkg.VersionCompare(add_version, target_version) >= 0:
2002 self.warnings.append("Propogating upload to %s" % (addsuite))
2003 self.pkg.changes.setdefault("propdistribution", {})
2004 self.pkg.changes["propdistribution"][addsuite] = 1
2008 self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
2010 ################################################################################
2011 def check_binary_against_db(self, file, session):
2012 # Ensure version is sane
2013 q = session.query(BinAssociation)
2014 q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
2015 q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
2017 self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
2018 file, self.pkg.files[file]["version"], sourceful=False)
2020 # Check for any existing copies of the file
2021 q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
2022 q = q.filter_by(version=self.pkg.files[file]["version"])
2023 q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
2026 self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
2028 ################################################################################
2030 def check_source_against_db(self, file, session):
2033 source = self.pkg.dsc.get("source")
2034 version = self.pkg.dsc.get("version")
2036 # Ensure version is sane
2037 q = session.query(SrcAssociation)
2038 q = q.join(DBSource).filter(DBSource.source==source)
2040 self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
2041 file, version, sourceful=True)
2043 ################################################################################
2044 def check_dsc_against_db(self, file, session):
2047 @warning: NB: this function can remove entries from the 'files' index [if
2048 the orig tarball is a duplicate of the one in the archive]; if
2049 you're iterating over 'files' and call this function as part of
2050 the loop, be sure to add a check to the top of the loop to
2051 ensure you haven't just tried to dereference the deleted entry.
2056 self.pkg.orig_files = {} # XXX: do we need to clear it?
2057 orig_files = self.pkg.orig_files
2059 # Try and find all files mentioned in the .dsc. This has
2060 # to work harder to cope with the multiple possible
2061 # locations of an .orig.tar.gz.
2062 # The ordering on the select is needed to pick the newest orig
2063 # when it exists in multiple places.
2064 for dsc_name, dsc_entry in self.pkg.dsc_files.items():
2066 if self.pkg.files.has_key(dsc_name):
2067 actual_md5 = self.pkg.files[dsc_name]["md5sum"]
2068 actual_size = int(self.pkg.files[dsc_name]["size"])
2069 found = "%s in incoming" % (dsc_name)
2071 # Check the file does not already exist in the archive
2072 ql = get_poolfile_like_name(dsc_name, session)
2074 # Strip out anything that isn't '%s' or '/%s$'
2076 if not i.filename.endswith(dsc_name):
2079 # "[dak] has not broken them. [dak] has fixed a
2080 # brokenness. Your crappy hack exploited a bug in
2083 # "(Come on! I thought it was always obvious that
2084 # one just doesn't release different files with
2085 # the same name and version.)"
2086 # -- ajk@ on d-devel@l.d.o
2089 # Ignore exact matches for .orig.tar.gz
2091 if re_is_orig_source.match(dsc_name):
2093 if self.pkg.files.has_key(dsc_name) and \
2094 int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
2095 self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
2096 self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
2097 # TODO: Don't delete the entry, just mark it as not needed
2098 # This would fix the stupidity of changing something we often iterate over
2099 # whilst we're doing it
2100 del self.pkg.files[dsc_name]
2101 if not orig_files.has_key(dsc_name):
2102 orig_files[dsc_name] = {}
2103 orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
2107 self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
2109 elif re_is_orig_source.match(dsc_name):
2111 ql = get_poolfile_like_name(dsc_name, session)
2113 # Strip out anything that isn't '%s' or '/%s$'
2114 # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
2116 if not i.filename.endswith(dsc_name):
2120 # Unfortunately, we may get more than one match here if,
2121 # for example, the package was in potato but had an -sa
2122 # upload in woody. So we need to choose the right one.
2124 # default to something sane in case we don't match any or have only one
2129 old_file = os.path.join(i.location.path, i.filename)
2130 old_file_fh = utils.open_file(old_file)
2131 actual_md5 = apt_pkg.md5sum(old_file_fh)
2133 actual_size = os.stat(old_file)[stat.ST_SIZE]
2134 if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
2137 old_file = os.path.join(i.location.path, i.filename)
2138 old_file_fh = utils.open_file(old_file)
2139 actual_md5 = apt_pkg.md5sum(old_file_fh)
2141 actual_size = os.stat(old_file)[stat.ST_SIZE]
2143 suite_type = x.location.archive_type
2144 # need this for updating dsc_files in install()
2145 dsc_entry["files id"] = x.file_id
2146 # See install() in process-accepted...
2147 if not orig_files.has_key(dsc_name):
2148 orig_files[dsc_name] = {}
2149 orig_files[dsc_name]["id"] = x.file_id
2150 orig_files[dsc_name]["path"] = old_file
2151 orig_files[dsc_name]["location"] = x.location.location_id
2153 # TODO: Record the queues and info in the DB so we don't hardcode all this crap
2154 # Not there? Check the queue directories...
2155 for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
2156 if not Cnf.has_key("Dir::Queue::%s" % (directory)):
2158 in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
2159 if os.path.exists(in_otherdir):
2160 in_otherdir_fh = utils.open_file(in_otherdir)
2161 actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
2162 in_otherdir_fh.close()
2163 actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
2165 if not orig_files.has_key(dsc_name):
2166 orig_files[dsc_name] = {}
2167 orig_files[dsc_name]["path"] = in_otherdir
2170 self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
2173 self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
2175 if actual_md5 != dsc_entry["md5sum"]:
2176 self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
2177 if actual_size != int(dsc_entry["size"]):
2178 self.rejects.append("size for %s doesn't match %s." % (found, file))
2180 ################################################################################
2181 def accepted_checks(self, overwrite_checks, session):
2182 # Recheck anything that relies on the database; since that's not
2183 # frozen between accept and our run time when called from p-a.
2185 # overwrite_checks is set to False when installing to stable/oldstable
2190 # Find the .dsc (again)
2192 for f in self.pkg.files.keys():
2193 if self.pkg.files[f]["type"] == "dsc":
2196 for checkfile in self.pkg.files.keys():
2197 # The .orig.tar.gz can disappear out from under us is it's a
2198 # duplicate of one in the archive.
2199 if not self.pkg.files.has_key(checkfile):
2202 entry = self.pkg.files[checkfile]
2204 # Check that the source still exists
2205 if entry["type"] == "deb":
2206 source_version = entry["source version"]
2207 source_package = entry["source package"]
2208 if not self.pkg.changes["architecture"].has_key("source") \
2209 and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
2210 self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
2212 # Version and file overwrite checks
2213 if overwrite_checks:
2214 if entry["type"] == "deb":
2215 self.check_binary_against_db(checkfile, session)
2216 elif entry["type"] == "dsc":
2217 self.check_source_against_db(checkfile, session)
2218 self.check_dsc_against_db(dsc_filename, session)
2220 # propogate in the case it is in the override tables:
2221 for suite in self.pkg.changes.get("propdistribution", {}).keys():
2222 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2223 propogate[suite] = 1
2225 nopropogate[suite] = 1
2227 for suite in propogate.keys():
2228 if suite in nopropogate:
2230 self.pkg.changes["distribution"][suite] = 1
2232 for checkfile in self.pkg.files.keys():
2233 # Check the package is still in the override tables
2234 for suite in self.pkg.changes["distribution"].keys():
2235 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
2236 self.rejects.append("%s is NEW for %s." % (checkfile, suite))
2238 ################################################################################
2239 # This is not really a reject, but an unaccept, but since a) the code for
2240 # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
2241 # extremely rare, for now we'll go with whining at our admin folks...
2243 def do_unaccept(self):
2247 self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
2248 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
2249 self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
2250 self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
2251 if cnf.has_key("Dinstall::Bcc"):
2252 self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
2254 template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
2256 reject_mail_message = utils.TemplateSubst(self.Subst, template)
2258 # Write the rejection email out as the <foo>.reason file
2259 reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
2260 reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
2262 # If we fail here someone is probably trying to exploit the race
2263 # so let's just raise an exception ...
2264 if os.path.exists(reject_filename):
2265 os.unlink(reject_filename)
2267 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
2268 os.write(fd, reject_mail_message)
2271 utils.send_mail(reject_mail_message)
2273 del self.Subst["__REJECTOR_ADDRESS__"]
2274 del self.Subst["__REJECT_MESSAGE__"]
2275 del self.Subst["__CC__"]
2277 ################################################################################
2278 # If any file of an upload has a recent mtime then chances are good
2279 # the file is still being uploaded.
2281 def upload_too_new(self):
2284 # Move back to the original directory to get accurate time stamps
2286 os.chdir(self.pkg.directory)
2287 file_list = self.pkg.files.keys()
2288 file_list.extend(self.pkg.dsc_files.keys())
2289 file_list.append(self.pkg.changes_file)
2292 last_modified = time.time()-os.path.getmtime(f)
2293 if last_modified < int(cnf["Dinstall::SkipTime"]):