5 Queue utility functions for dak
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ###############################################################################
41 from sqlalchemy.sql.expression import desc
42 from sqlalchemy.orm.exc import NoResultFound
44 from dak_exceptions import *
47 from config import Config
48 from urgencylog import UrgencyLog
50 from summarystats import SummaryStats
51 from utils import parse_changes, check_dsc_files
52 from textutils import fix_maintainer
53 from lintian import parse_lintian_output, generate_reject_messages
54 from contents import UnpackedSource
56 ################################################################################
58 def check_valid(overrides, session):
59 """Check if section and priority for new overrides exist in database.
61 Additionally does sanity checks:
62 - debian-installer packages have to be udeb (or source)
63 - non debian-installer packages cannot be udeb
65 @type overrides: list of dict
66 @param overrides: list of overrides to check. The overrides need
67 to be given in form of a dict with the following keys:
69 - package: package name
73 - type: type of requested override ('dsc', 'deb' or 'udeb')
75 All values are strings.
78 @return: C{True} if all overrides are valid, C{False} if there is any
84 if session.query(Priority).filter_by(priority=o['priority']).first() is None:
86 if session.query(Section).filter_by(section=o['section']).first() is None:
88 if get_mapped_component(o['component'], session) is None:
90 if o['type'] not in ('dsc', 'deb', 'udeb'):
91 raise Exception('Unknown override type {0}'.format(o['type']))
92 if o['type'] == 'udeb' and o['section'] != 'debian-installer':
94 if o['section'] == 'debian-installer' and o['type'] not in ('dsc', 'udeb'):
96 all_valid = all_valid and o['valid']
99 ###############################################################################
101 def prod_maintainer(notes, upload):
103 changes = upload.changes
104 whitelists = [ upload.target_suite.mail_whitelist ]
106 # Here we prepare an editor and get them ready to prod...
107 (fd, temp_filename) = utils.temp_filename()
108 temp_file = os.fdopen(fd, 'w')
109 temp_file.write("\n\n=====\n\n".join([note.comment for note in notes]))
111 editor = os.environ.get("EDITOR","vi")
114 os.system("%s %s" % (editor, temp_filename))
115 temp_fh = utils.open_file(temp_filename)
116 prod_message = "".join(temp_fh.readlines())
118 print "Prod message:"
119 print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
120 prompt = "[P]rod, Edit, Abandon, Quit ?"
122 while prompt.find(answer) == -1:
123 answer = utils.our_raw_input(prompt)
124 m = re_default_answer.search(prompt)
127 answer = answer[:1].upper()
128 os.unlink(temp_filename)
133 # Otherwise, do the proding...
134 user_email_address = utils.whoami() + " <%s>" % (
135 cnf["Dinstall::MyAdminAddress"])
137 changed_by = changes.changedby or changes.maintainer
138 maintainer = changes.maintainer
139 maintainer_to = utils.mail_addresses_for_upload(maintainer, changed_by, changes.fingerprint)
142 '__SOURCE__': upload.changes.source,
143 '__CHANGES_FILENAME__': upload.changes.changesname,
144 '__MAINTAINER_TO__': ", ".join(maintainer_to),
147 Subst["__FROM_ADDRESS__"] = user_email_address
148 Subst["__PROD_MESSAGE__"] = prod_message
149 Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
151 prod_mail_message = utils.TemplateSubst(
152 Subst,cnf["Dir::Templates"]+"/process-new.prod")
155 utils.send_mail(prod_mail_message, whitelists=whitelists)
157 print "Sent prodding message"
159 ################################################################################
161 def edit_note(note, upload, session, trainee=False):
162 # Write the current data to a temporary file
163 (fd, temp_filename) = utils.temp_filename()
164 editor = os.environ.get("EDITOR","vi")
167 os.system("%s %s" % (editor, temp_filename))
168 temp_file = utils.open_file(temp_filename)
169 newnote = temp_file.read().rstrip()
172 print utils.prefix_multi_line_string(newnote," ")
173 prompt = "[D]one, Edit, Abandon, Quit ?"
175 while prompt.find(answer) == -1:
176 answer = utils.our_raw_input(prompt)
177 m = re_default_answer.search(prompt)
180 answer = answer[:1].upper()
181 os.unlink(temp_filename)
187 comment = NewComment()
188 comment.policy_queue = upload.policy_queue
189 comment.package = upload.changes.source
190 comment.version = upload.changes.version
191 comment.comment = newnote
192 comment.author = utils.whoami()
193 comment.trainee = trainee
197 ###############################################################################
199 def get_suite_version_by_source(source, session):
200 'returns a list of tuples (suite_name, version) for source package'
201 q = session.query(Suite.suite_name, DBSource.version). \
202 join(Suite.sources).filter_by(source = source)
205 def get_suite_version_by_package(package, arch_string, session):
207 returns a list of tuples (suite_name, version) for binary package and
210 return session.query(Suite.suite_name, DBBinary.version). \
211 join(Suite.binaries).filter_by(package = package). \
212 join(DBBinary.architecture). \
213 filter(Architecture.arch_string.in_([arch_string, 'all'])).all()
215 class Upload(object):
217 Everything that has to do with an upload processed.
225 ###########################################################################
227 def update_subst(self):
228 """ Set up the per-package template substitution mappings """
229 raise Exception('to be removed')
233 # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
234 if not self.pkg.changes.has_key("architecture") or not \
235 isinstance(self.pkg.changes["architecture"], dict):
236 self.pkg.changes["architecture"] = { "Unknown" : "" }
238 # and maintainer2047 may not exist.
239 if not self.pkg.changes.has_key("maintainer2047"):
240 self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
242 self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
243 self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
244 self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
246 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
247 if self.pkg.changes["architecture"].has_key("source") and \
248 self.pkg.changes["changedby822"] != "" and \
249 (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
251 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
252 self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
253 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
255 self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
256 self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
257 self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
259 # Process policy doesn't set the fingerprint field and I don't want to make it
260 # do it for now as I don't want to have to deal with the case where we accepted
261 # the package into PU-NEW, but the fingerprint has gone away from the keyring in
262 # the meantime so the package will be remarked as rejectable. Urgh.
263 # TODO: Fix this properly
264 if self.pkg.changes.has_key('fingerprint'):
265 session = DBConn().session()
266 fpr = get_fingerprint(self.pkg.changes['fingerprint'], session)
267 if fpr and self.check_if_upload_is_sponsored("%s@debian.org" % fpr.uid.uid, fpr.uid.name):
268 if self.pkg.changes.has_key("sponsoremail"):
269 self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
272 if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
273 self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
275 # Apply any global override of the Maintainer field
276 if cnf.get("Dinstall::OverrideMaintainer"):
277 self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
278 self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
280 self.Subst["__REJECT_MESSAGE__"] = self.package_info()
281 self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
282 self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
283 self.Subst["__SUITE__"] = ", ".join(self.pkg.changes["distribution"])
285 ###########################################################################
287 def check_if_upload_is_sponsored(self, uid_email, uid_name):
288 for key in "maintaineremail", "changedbyemail", "maintainername", "changedbyname":
289 if not self.pkg.changes.has_key(key):
291 uid_email = '@'.join(uid_email.split('@')[:2])
292 if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
294 elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
300 sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
301 debian_emails = filter(lambda addr: addr.endswith('@debian.org'), sponsor_addresses)
302 if uid_email not in debian_emails:
304 uid_email = debian_emails[0]
305 if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
306 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
307 self.pkg.changes["changedbyemail"] not in sponsor_addresses):
308 self.pkg.changes["sponsoremail"] = uid_email
312 ###########################################################################
313 # End check_signed_by_key checks
314 ###########################################################################
316 def announce(self, short_summary, action):
318 Send an announce mail about a new upload.
320 @type short_summary: string
321 @param short_summary: Short summary text to include in the mail
324 @param action: Set to false no real action will be done.
327 @return: Textstring about action taken.
333 # Skip all of this if not sending mail to avoid confusing people
334 if cnf.has_key("Dinstall::Options::No-Mail") and cnf["Dinstall::Options::No-Mail"]:
337 # Only do announcements for source uploads with a recent dpkg-dev installed
338 if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
339 self.pkg.changes["architecture"].has_key("source"):
342 announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
347 # Get a unique list of target lists
348 for dist in self.pkg.changes["distribution"].keys():
349 suite = get_suite(dist)
350 if suite is None: continue
351 for tgt in suite.announce:
354 self.Subst["__SHORT_SUMMARY__"] = short_summary
356 for announce_list in lists_todo.keys():
357 summary += "Announcing to %s\n" % (announce_list)
361 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
362 if cnf.get("Dinstall::TrackingServer") and \
363 self.pkg.changes["architecture"].has_key("source"):
364 trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
365 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
367 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
368 utils.send_mail(mail_message)
370 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
372 if cnf.find_b("Dinstall::CloseBugs") and cnf.has_key("Dinstall::BugServer"):
373 summary = self.close_bugs(summary, action)
375 del self.Subst["__SHORT_SUMMARY__"]
379 ###########################################################################
381 def check_override(self):
383 Checks override entries for validity. Mails "Override disparity" warnings,
384 if that feature is enabled.
386 Abandons the check if
387 - override disparity checks are disabled
388 - mail sending is disabled
393 # Abandon the check if override disparity checks have been disabled
394 if not cnf.find_b("Dinstall::OverrideDisparityCheck"):
397 summary = self.pkg.check_override()
402 overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
405 self.Subst["__SUMMARY__"] = summary
406 mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
407 utils.send_mail(mail_message)
408 del self.Subst["__SUMMARY__"]