5 Queue utility functions for dak
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ###############################################################################
41 from sqlalchemy.sql.expression import desc
42 from sqlalchemy.orm.exc import NoResultFound
46 from dak_exceptions import *
49 from config import Config
50 from holding import Holding
51 from urgencylog import UrgencyLog
53 from summarystats import SummaryStats
54 from utils import parse_changes, check_dsc_files, build_package_list
55 from textutils import fix_maintainer
56 from lintian import parse_lintian_output, generate_reject_messages
57 from contents import UnpackedSource
59 ################################################################################
61 def check_valid(overrides, session):
62 """Check if section and priority for new overrides exist in database.
64 Additionally does sanity checks:
65 - debian-installer packages have to be udeb (or source)
66 - non debian-installer packages cannot be udeb
68 @type overrides: list of dict
69 @param overrides: list of overrides to check. The overrides need
70 to be given in form of a dict with the following keys:
72 - package: package name
76 - type: type of requested override ('dsc', 'deb' or 'udeb')
78 All values are strings.
81 @return: C{True} if all overrides are valid, C{False} if there is any
87 if session.query(Priority).filter_by(priority=o['priority']).first() is None:
89 if session.query(Section).filter_by(section=o['section']).first() is None:
91 if get_mapped_component(o['component'], session) is None:
93 if o['type'] not in ('dsc', 'deb', 'udeb'):
94 raise Exception('Unknown override type {0}'.format(o['type']))
95 if o['type'] == 'udeb' and o['section'] != 'debian-installer':
97 if o['section'] == 'debian-installer' and o['type'] not in ('dsc', 'udeb'):
99 all_valid = all_valid and o['valid']
102 ###############################################################################
104 def prod_maintainer(notes, upload):
106 changes = upload.changes
107 whitelists = [ upload.target_suite.mail_whitelist ]
109 # Here we prepare an editor and get them ready to prod...
110 (fd, temp_filename) = utils.temp_filename()
111 temp_file = os.fdopen(fd, 'w')
112 temp_file.write("\n\n=====\n\n".join([note.comment for note in notes]))
114 editor = os.environ.get("EDITOR","vi")
117 os.system("%s %s" % (editor, temp_filename))
118 temp_fh = utils.open_file(temp_filename)
119 prod_message = "".join(temp_fh.readlines())
121 print "Prod message:"
122 print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
123 prompt = "[P]rod, Edit, Abandon, Quit ?"
125 while prompt.find(answer) == -1:
126 answer = utils.our_raw_input(prompt)
127 m = re_default_answer.search(prompt)
130 answer = answer[:1].upper()
131 os.unlink(temp_filename)
136 # Otherwise, do the proding...
137 user_email_address = utils.whoami() + " <%s>" % (
138 cnf["Dinstall::MyAdminAddress"])
140 changed_by = changes.changedby or changes.maintainer
141 maintainer = changes.maintainer
142 maintainer_to = utils.mail_addresses_for_upload(maintainer, changed_by, changes.fingerprint)
145 '__SOURCE__': upload.changes.source,
146 '__CHANGES_FILENAME__': upload.changes.changesname,
147 '__MAINTAINER_TO__': ", ".join(maintainer_to),
150 Subst["__FROM_ADDRESS__"] = user_email_address
151 Subst["__PROD_MESSAGE__"] = prod_message
152 Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
154 prod_mail_message = utils.TemplateSubst(
155 Subst,cnf["Dir::Templates"]+"/process-new.prod")
158 utils.send_mail(prod_mail_message, whitelists=whitelists)
160 print "Sent prodding message"
162 ################################################################################
164 def edit_note(note, upload, session, trainee=False):
165 # Write the current data to a temporary file
166 (fd, temp_filename) = utils.temp_filename()
167 editor = os.environ.get("EDITOR","vi")
170 os.system("%s %s" % (editor, temp_filename))
171 temp_file = utils.open_file(temp_filename)
172 newnote = temp_file.read().rstrip()
175 print utils.prefix_multi_line_string(newnote," ")
176 prompt = "[D]one, Edit, Abandon, Quit ?"
178 while prompt.find(answer) == -1:
179 answer = utils.our_raw_input(prompt)
180 m = re_default_answer.search(prompt)
183 answer = answer[:1].upper()
184 os.unlink(temp_filename)
190 comment = NewComment()
191 comment.policy_queue = upload.policy_queue
192 comment.package = upload.changes.source
193 comment.version = upload.changes.version
194 comment.comment = newnote
195 comment.author = utils.whoami()
196 comment.trainee = trainee
200 ###############################################################################
202 # FIXME: Should move into the database
203 # suite names DMs can upload to
204 dm_suites = ['unstable', 'experimental', 'squeeze-backports']
206 def get_newest_source(source, session):
207 'returns the newest DBSource object in dm_suites'
208 ## the most recent version of the package uploaded to unstable or
209 ## experimental includes the field "DM-Upload-Allowed: yes" in the source
210 ## section of its control file
211 q = session.query(DBSource).filter_by(source = source). \
212 filter(DBSource.suites.any(Suite.suite_name.in_(dm_suites))). \
213 order_by(desc('source.version'))
216 def get_suite_version_by_source(source, session):
217 'returns a list of tuples (suite_name, version) for source package'
218 q = session.query(Suite.suite_name, DBSource.version). \
219 join(Suite.sources).filter_by(source = source)
222 def get_source_by_package_and_suite(package, suite_name, session):
224 returns a DBSource query filtered by DBBinary.package and this package's
227 return session.query(DBSource). \
228 join(DBSource.binaries).filter_by(package = package). \
229 join(DBBinary.suites).filter_by(suite_name = suite_name)
231 def get_suite_version_by_package(package, arch_string, session):
233 returns a list of tuples (suite_name, version) for binary package and
236 return session.query(Suite.suite_name, DBBinary.version). \
237 join(Suite.binaries).filter_by(package = package). \
238 join(DBBinary.architecture). \
239 filter(Architecture.arch_string.in_([arch_string, 'all'])).all()
241 class Upload(object):
243 Everything that has to do with an upload processed.
251 ###########################################################################
253 def update_subst(self):
254 """ Set up the per-package template substitution mappings """
255 raise Exception('to be removed')
259 # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
260 if not self.pkg.changes.has_key("architecture") or not \
261 isinstance(self.pkg.changes["architecture"], dict):
262 self.pkg.changes["architecture"] = { "Unknown" : "" }
264 # and maintainer2047 may not exist.
265 if not self.pkg.changes.has_key("maintainer2047"):
266 self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
268 self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
269 self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
270 self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
272 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
273 if self.pkg.changes["architecture"].has_key("source") and \
274 self.pkg.changes["changedby822"] != "" and \
275 (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
277 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
278 self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
279 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
281 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
282 self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
283 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
285 # Process policy doesn't set the fingerprint field and I don't want to make it
286 # do it for now as I don't want to have to deal with the case where we accepted
287 # the package into PU-NEW, but the fingerprint has gone away from the keyring in
288 # the meantime so the package will be remarked as rejectable. Urgh.
289 # TODO: Fix this properly
290 if self.pkg.changes.has_key('fingerprint'):
291 session = DBConn().session()
292 fpr = get_fingerprint(self.pkg.changes['fingerprint'], session)
293 if fpr and self.check_if_upload_is_sponsored("%s@debian.org" % fpr.uid.uid, fpr.uid.name):
294 if self.pkg.changes.has_key("sponsoremail"):
295 self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
298 if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
299 self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
301 # Apply any global override of the Maintainer field
302 if cnf.get("Dinstall::OverrideMaintainer"):
303 self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
304 self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
306 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
307 self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
308 self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
309 self.Subst["__SUITE__"] = ", ".join(self.pkg.changes["distribution"])
311 ###########################################################################
313 def check_distributions(self):
314 "Check and map the Distribution field"
318 # Handle suite mappings
319 for m in Cnf.value_list("SuiteMappings"):
322 if mtype == "map" or mtype == "silent-map":
323 (source, dest) = args[1:3]
324 if self.pkg.changes["distribution"].has_key(source):
325 del self.pkg.changes["distribution"][source]
326 self.pkg.changes["distribution"][dest] = 1
327 if mtype != "silent-map":
328 self.notes.append("Mapping %s to %s." % (source, dest))
329 if self.pkg.changes.has_key("distribution-version"):
330 if self.pkg.changes["distribution-version"].has_key(source):
331 self.pkg.changes["distribution-version"][source]=dest
332 elif mtype == "map-unreleased":
333 (source, dest) = args[1:3]
334 if self.pkg.changes["distribution"].has_key(source):
335 for arch in self.pkg.changes["architecture"].keys():
336 if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
337 self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
338 del self.pkg.changes["distribution"][source]
339 self.pkg.changes["distribution"][dest] = 1
341 elif mtype == "ignore":
343 if self.pkg.changes["distribution"].has_key(suite):
344 del self.pkg.changes["distribution"][suite]
345 self.warnings.append("Ignoring %s as a target suite." % (suite))
346 elif mtype == "reject":
348 if self.pkg.changes["distribution"].has_key(suite):
349 self.rejects.append("Uploads to %s are not accepted." % (suite))
350 elif mtype == "propup-version":
351 # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
353 # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
354 if self.pkg.changes["distribution"].has_key(args[1]):
355 self.pkg.changes.setdefault("distribution-version", {})
356 for suite in args[2:]:
357 self.pkg.changes["distribution-version"][suite] = suite
359 # Ensure there is (still) a target distribution
360 if len(self.pkg.changes["distribution"].keys()) < 1:
361 self.rejects.append("No valid distribution remaining.")
363 # Ensure target distributions exist
364 for suite in self.pkg.changes["distribution"].keys():
365 if not get_suite(suite.lower()):
366 self.rejects.append("Unknown distribution `%s'." % (suite))
368 ###########################################################################
370 def per_suite_file_checks(self, f, suite, session):
371 raise Exception('removed')
373 # Handle component mappings
374 for m in cnf.value_list("ComponentMappings"):
375 (source, dest) = m.split()
376 if entry["component"] == source:
377 entry["original component"] = source
378 entry["component"] = dest
380 ###########################################################################
382 # Sanity check the time stamps of files inside debs.
383 # [Files in the near future cause ugly warnings and extreme time
384 # travel can cause errors on extraction]
386 def check_if_upload_is_sponsored(self, uid_email, uid_name):
387 for key in "maintaineremail", "changedbyemail", "maintainername", "changedbyname":
388 if not self.pkg.changes.has_key(key):
390 uid_email = '@'.join(uid_email.split('@')[:2])
391 if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
393 elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
399 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
400 debian_emails = filter(lambda addr: addr.endswith('@debian.org'), sponsor_addresses)
401 if uid_email not in debian_emails:
403 uid_email = debian_emails[0]
404 if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
405 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
406 self.pkg.changes["changedbyemail"] not in sponsor_addresses):
407 self.pkg.changes["sponsoremail"] = uid_email
411 def check_dm_upload(self, fpr, session):
412 # Quoth the GR (http://www.debian.org/vote/2007/vote_003):
413 ## none of the uploaded packages are NEW
414 ## none of the packages are being taken over from other source packages
415 for b in self.pkg.changes["binary"].keys():
416 for suite in self.pkg.changes["distribution"].keys():
417 for s in get_source_by_package_and_suite(b, suite, session):
418 if s.source != self.pkg.changes["source"]:
419 self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite))
421 ###########################################################################
422 # End check_signed_by_key checks
423 ###########################################################################
425 def build_summaries(self):
426 """ Build a summary of changes the upload introduces. """
428 (byhand, new, summary, override_summary) = self.pkg.file_summary()
430 short_summary = summary
432 # This is for direport's benefit...
433 f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
435 summary += "\n\nChanges:\n" + f
437 summary += "\n\nOverride entries for your package:\n" + override_summary + "\n"
439 summary += self.announce(short_summary, 0)
441 return (summary, short_summary)
443 ###########################################################################
445 def announce(self, short_summary, action):
447 Send an announce mail about a new upload.
449 @type short_summary: string
450 @param short_summary: Short summary text to include in the mail
453 @param action: Set to false no real action will be done.
456 @return: Textstring about action taken.
462 # Skip all of this if not sending mail to avoid confusing people
463 if cnf.has_key("Dinstall::Options::No-Mail") and cnf["Dinstall::Options::No-Mail"]:
466 # Only do announcements for source uploads with a recent dpkg-dev installed
467 if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
468 self.pkg.changes["architecture"].has_key("source"):
471 announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
476 # Get a unique list of target lists
477 for dist in self.pkg.changes["distribution"].keys():
478 suite = get_suite(dist)
479 if suite is None: continue
480 for tgt in suite.announce:
483 self.Subst["__SHORT_SUMMARY__"] = short_summary
485 for announce_list in lists_todo.keys():
486 summary += "Announcing to %s\n" % (announce_list)
490 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
491 if cnf.get("Dinstall::TrackingServer") and \
492 self.pkg.changes["architecture"].has_key("source"):
493 trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
494 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
496 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
497 utils.send_mail(mail_message)
499 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
501 if cnf.find_b("Dinstall::CloseBugs") and cnf.has_key("Dinstall::BugServer"):
502 summary = self.close_bugs(summary, action)
504 del self.Subst["__SHORT_SUMMARY__"]
508 ###########################################################################
510 def check_override(self):
512 Checks override entries for validity. Mails "Override disparity" warnings,
513 if that feature is enabled.
515 Abandons the check if
516 - override disparity checks are disabled
517 - mail sending is disabled
522 # Abandon the check if override disparity checks have been disabled
523 if not cnf.find_b("Dinstall::OverrideDisparityCheck"):
526 summary = self.pkg.check_override()
531 overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
534 self.Subst["__SUMMARY__"] = summary
535 mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
536 utils.send_mail(mail_message)
537 del self.Subst["__SUMMARY__"]
539 ################################################################################
540 def get_anyversion(self, sv_list, suite):
543 @param sv_list: list of (suite, version) tuples to check
546 @param suite: suite name
552 anysuite = [suite] + [ vc.reference.suite_name for vc in get_version_checks(suite, "Enhances") ]
553 for (s, v) in sv_list:
554 if s in [ x.lower() for x in anysuite ]:
555 if not anyversion or apt_pkg.version_compare(anyversion, v) <= 0:
560 ################################################################################
562 def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
565 @param sv_list: list of (suite, version) tuples to check
567 @type filename: string
570 @type new_version: string
571 @param new_version: XXX
573 Ensure versions are newer than existing packages in target
574 suites and that cross-suite version checking rules as
575 set out in the conf file are satisfied.
580 # Check versions for each target suite
581 for target_suite in self.pkg.changes["distribution"].keys():
582 # Check we can find the target suite
583 ts = get_suite(target_suite)
585 self.rejects.append("Cannot find target suite %s to perform version checks" % target_suite)
588 must_be_newer_than = [ vc.reference.suite_name for vc in get_version_checks(target_suite, "MustBeNewerThan") ]
589 must_be_older_than = [ vc.reference.suite_name for vc in get_version_checks(target_suite, "MustBeOlderThan") ]
591 # Enforce "must be newer than target suite" even if conffile omits it
592 if target_suite not in must_be_newer_than:
593 must_be_newer_than.append(target_suite)
595 for (suite, existent_version) in sv_list:
596 vercmp = apt_pkg.version_compare(new_version, existent_version)
598 if suite in must_be_newer_than and sourceful and vercmp < 1:
599 self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
601 if suite in must_be_older_than and vercmp > -1:
604 if self.pkg.changes.get('distribution-version', {}).has_key(suite):
605 # we really use the other suite, ignoring the conflicting one ...
606 addsuite = self.pkg.changes["distribution-version"][suite]
608 add_version = self.get_anyversion(sv_list, addsuite)
609 target_version = self.get_anyversion(sv_list, target_suite)
612 # not add_version can only happen if we map to a suite
613 # that doesn't enhance the suite we're propup'ing from.
614 # so "propup-ver x a b c; map a d" is a problem only if
615 # d doesn't enhance a.
617 # i think we could always propagate in this case, rather
618 # than complaining. either way, this isn't a REJECT issue
620 # And - we really should complain to the dorks who configured dak
621 self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
622 self.pkg.changes.setdefault("propdistribution", {})
623 self.pkg.changes["propdistribution"][addsuite] = 1
625 elif not target_version:
626 # not targets_version is true when the package is NEW
627 # we could just stick with the "...old version..." REJECT
629 self.rejects.append("Won't propogate NEW packages.")
630 elif apt_pkg.version_compare(new_version, add_version) < 0:
631 # propogation would be redundant. no need to reject though.
632 self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
634 elif apt_pkg.version_compare(new_version, add_version) > 0 and \
635 apt_pkg.version_compare(add_version, target_version) >= 0:
637 self.warnings.append("Propogating upload to %s" % (addsuite))
638 self.pkg.changes.setdefault("propdistribution", {})
639 self.pkg.changes["propdistribution"][addsuite] = 1
643 self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
645 ################################################################################
647 def accepted_checks(self, overwrite_checks, session):
648 # Recheck anything that relies on the database; since that's not
649 # frozen between accept and our run time when called from p-a.
651 # overwrite_checks is set to False when installing to stable/oldstable
656 for checkfile in self.pkg.files.keys():
657 # The .orig.tar.gz can disappear out from under us is it's a
658 # duplicate of one in the archive.
659 if not self.pkg.files.has_key(checkfile):
662 entry = self.pkg.files[checkfile]
664 # propogate in the case it is in the override tables:
665 for suite in self.pkg.changes.get("propdistribution", {}).keys():
666 if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
669 nopropogate[suite] = 1
671 for suite in propogate.keys():
672 if suite in nopropogate:
674 self.pkg.changes["distribution"][suite] = 1
676 for checkfile in self.pkg.files.keys():
677 # Check the package is still in the override tables
678 for suite in self.pkg.changes["distribution"].keys():
679 if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
680 self.rejects.append("%s is NEW for %s." % (checkfile, suite))