]> git.decadent.org.uk Git - dak.git/blob - daklib/queue.py
daklib: kill unused utils.known_hashes
[dak.git] / daklib / queue.py
1 #!/usr/bin/env python
2 # vim:set et sw=4:
3
4 """
5 Queue utility functions for dak
6
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009, 2010  Joerg Jaspert <joerg@debian.org>
10 @license: GNU General Public License version 2 or later
11 """
12
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
17
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21 # GNU General Public License for more details.
22
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26
27 ###############################################################################
28
29 import os
30 import utils
31 from types import *
32
33 from dak_exceptions import *
34 from changes import *
35 from regexes import *
36 from config import Config
37 from dbconn import *
38
39 ################################################################################
40
41 def check_valid(overrides, session):
42     """Check if section and priority for new overrides exist in database.
43
44     Additionally does sanity checks:
45       - debian-installer packages have to be udeb (or source)
46       - non debian-installer packages cannot be udeb
47
48     @type  overrides: list of dict
49     @param overrides: list of overrides to check. The overrides need
50                       to be given in form of a dict with the following keys:
51
52                       - package: package name
53                       - priority
54                       - section
55                       - component
56                       - type: type of requested override ('dsc', 'deb' or 'udeb')
57
58                       All values are strings.
59
60     @rtype:  bool
61     @return: C{True} if all overrides are valid, C{False} if there is any
62              invalid override.
63     """
64     all_valid = True
65     for o in overrides:
66         o['valid'] = True
67         if session.query(Priority).filter_by(priority=o['priority']).first() is None:
68             o['valid'] = False
69         if session.query(Section).filter_by(section=o['section']).first() is None:
70             o['valid'] = False
71         if get_mapped_component(o['component'], session) is None:
72             o['valid'] = False
73         if o['type'] not in ('dsc', 'deb', 'udeb'):
74             raise Exception('Unknown override type {0}'.format(o['type']))
75         if o['type'] == 'udeb' and o['section'] != 'debian-installer':
76             o['valid'] = False
77         if o['section'] == 'debian-installer' and o['type'] not in ('dsc', 'udeb'):
78             o['valid'] = False
79         all_valid = all_valid and o['valid']
80     return all_valid
81
82 ###############################################################################
83
84 def prod_maintainer(notes, upload):
85     cnf = Config()
86     changes = upload.changes
87     whitelists = [ upload.target_suite.mail_whitelist ]
88
89     # Here we prepare an editor and get them ready to prod...
90     (fd, temp_filename) = utils.temp_filename()
91     temp_file = os.fdopen(fd, 'w')
92     temp_file.write("\n\n=====\n\n".join([note.comment for note in notes]))
93     temp_file.close()
94     editor = os.environ.get("EDITOR","vi")
95     answer = 'E'
96     while answer == 'E':
97         os.system("%s %s" % (editor, temp_filename))
98         temp_fh = utils.open_file(temp_filename)
99         prod_message = "".join(temp_fh.readlines())
100         temp_fh.close()
101         print "Prod message:"
102         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
103         prompt = "[P]rod, Edit, Abandon, Quit ?"
104         answer = "XXX"
105         while prompt.find(answer) == -1:
106             answer = utils.our_raw_input(prompt)
107             m = re_default_answer.search(prompt)
108             if answer == "":
109                 answer = m.group(1)
110             answer = answer[:1].upper()
111     os.unlink(temp_filename)
112     if answer == 'A':
113         return
114     elif answer == 'Q':
115         return 0
116     # Otherwise, do the proding...
117     user_email_address = utils.whoami() + " <%s>" % (
118         cnf["Dinstall::MyAdminAddress"])
119
120     changed_by = changes.changedby or changes.maintainer
121     maintainer = changes.maintainer
122     maintainer_to = utils.mail_addresses_for_upload(maintainer, changed_by, changes.fingerprint)
123
124     Subst = {
125         '__SOURCE__': upload.changes.source,
126         '__CHANGES_FILENAME__': upload.changes.changesname,
127         '__MAINTAINER_TO__': ", ".join(maintainer_to),
128         }
129
130     Subst["__FROM_ADDRESS__"] = user_email_address
131     Subst["__PROD_MESSAGE__"] = prod_message
132     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
133
134     prod_mail_message = utils.TemplateSubst(
135         Subst,cnf["Dir::Templates"]+"/process-new.prod")
136
137     # Send the prod mail
138     utils.send_mail(prod_mail_message, whitelists=whitelists)
139
140     print "Sent prodding message"
141
142 ################################################################################
143
144 def edit_note(note, upload, session, trainee=False):
145     # Write the current data to a temporary file
146     (fd, temp_filename) = utils.temp_filename()
147     editor = os.environ.get("EDITOR","vi")
148     answer = 'E'
149     while answer == 'E':
150         os.system("%s %s" % (editor, temp_filename))
151         temp_file = utils.open_file(temp_filename)
152         newnote = temp_file.read().rstrip()
153         temp_file.close()
154         print "New Note:"
155         print utils.prefix_multi_line_string(newnote,"  ")
156         prompt = "[D]one, Edit, Abandon, Quit ?"
157         answer = "XXX"
158         while prompt.find(answer) == -1:
159             answer = utils.our_raw_input(prompt)
160             m = re_default_answer.search(prompt)
161             if answer == "":
162                 answer = m.group(1)
163             answer = answer[:1].upper()
164     os.unlink(temp_filename)
165     if answer == 'A':
166         return
167     elif answer == 'Q':
168         return 0
169
170     comment = NewComment()
171     comment.policy_queue = upload.policy_queue
172     comment.package = upload.changes.source
173     comment.version = upload.changes.version
174     comment.comment = newnote
175     comment.author  = utils.whoami()
176     comment.trainee = trainee
177     session.add(comment)
178     session.commit()
179
180 ###############################################################################
181
182 def get_suite_version_by_source(source, session):
183     'returns a list of tuples (suite_name, version) for source package'
184     q = session.query(Suite.suite_name, DBSource.version). \
185         join(Suite.sources).filter_by(source = source)
186     return q.all()
187
188 def get_suite_version_by_package(package, arch_string, session):
189     '''
190     returns a list of tuples (suite_name, version) for binary package and
191     arch_string
192     '''
193     return session.query(Suite.suite_name, DBBinary.version). \
194         join(Suite.binaries).filter_by(package = package). \
195         join(DBBinary.architecture). \
196         filter(Architecture.arch_string.in_([arch_string, 'all'])).all()
197
198 class Upload(object):
199     """
200     Everything that has to do with an upload processed.
201
202     """
203     def __init__(self):
204         self.logger = None
205         self.pkg = Changes()
206         self.reset()
207
208     ###########################################################################
209
210     def update_subst(self):
211         """ Set up the per-package template substitution mappings """
212         raise Exception('to be removed')
213
214         cnf = Config()
215
216         # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
217         if not self.pkg.changes.has_key("architecture") or not \
218            isinstance(self.pkg.changes["architecture"], dict):
219             self.pkg.changes["architecture"] = { "Unknown" : "" }
220
221         # and maintainer2047 may not exist.
222         if not self.pkg.changes.has_key("maintainer2047"):
223             self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
224
225         self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
226         self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
227         self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
228
229         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
230         if self.pkg.changes["architecture"].has_key("source") and \
231            self.pkg.changes["changedby822"] != "" and \
232            (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
233
234             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
235             self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
236             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
237         else:
238             self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
239             self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
240             self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
241
242         # Process policy doesn't set the fingerprint field and I don't want to make it
243         # do it for now as I don't want to have to deal with the case where we accepted
244         # the package into PU-NEW, but the fingerprint has gone away from the keyring in
245         # the meantime so the package will be remarked as rejectable.  Urgh.
246         # TODO: Fix this properly
247         if self.pkg.changes.has_key('fingerprint'):
248             session = DBConn().session()
249             fpr = get_fingerprint(self.pkg.changes['fingerprint'], session)
250             if fpr and self.check_if_upload_is_sponsored("%s@debian.org" % fpr.uid.uid, fpr.uid.name):
251                 if self.pkg.changes.has_key("sponsoremail"):
252                     self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
253             session.close()
254
255         if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
256             self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
257
258         # Apply any global override of the Maintainer field
259         if cnf.get("Dinstall::OverrideMaintainer"):
260             self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
261             self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
262
263         self.Subst["__REJECT_MESSAGE__"] = self.package_info()
264         self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
265         self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
266         self.Subst["__SUITE__"] = ", ".join(self.pkg.changes["distribution"])
267
268     ###########################################################################
269
270     def check_if_upload_is_sponsored(self, uid_email, uid_name):
271         for key in "maintaineremail", "changedbyemail", "maintainername", "changedbyname":
272             if not self.pkg.changes.has_key(key):
273                 return False
274         uid_email = '@'.join(uid_email.split('@')[:2])
275         if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
276             sponsored = False
277         elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
278             sponsored = False
279             if uid_name == "":
280                 sponsored = True
281         else:
282             sponsored = True
283             sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
284             debian_emails = filter(lambda addr: addr.endswith('@debian.org'), sponsor_addresses)
285             if uid_email not in debian_emails:
286                 if debian_emails:
287                     uid_email = debian_emails[0]
288             if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
289                 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
290                     self.pkg.changes["changedbyemail"] not in sponsor_addresses):
291                         self.pkg.changes["sponsoremail"] = uid_email
292
293         return sponsored
294
295     ###########################################################################
296     # End check_signed_by_key checks
297     ###########################################################################
298
299     def announce(self, short_summary, action):
300         """
301         Send an announce mail about a new upload.
302
303         @type short_summary: string
304         @param short_summary: Short summary text to include in the mail
305
306         @type action: bool
307         @param action: Set to false no real action will be done.
308
309         @rtype: string
310         @return: Textstring about action taken.
311
312         """
313
314         cnf = Config()
315
316         # Skip all of this if not sending mail to avoid confusing people
317         if cnf.has_key("Dinstall::Options::No-Mail") and cnf["Dinstall::Options::No-Mail"]:
318             return ""
319
320         # Only do announcements for source uploads with a recent dpkg-dev installed
321         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
322            self.pkg.changes["architecture"].has_key("source"):
323             return ""
324
325         announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
326
327         lists_todo = {}
328         summary = ""
329
330         # Get a unique list of target lists
331         for dist in self.pkg.changes["distribution"].keys():
332             suite = get_suite(dist)
333             if suite is None: continue
334             for tgt in suite.announce:
335                 lists_todo[tgt] = 1
336
337         self.Subst["__SHORT_SUMMARY__"] = short_summary
338
339         for announce_list in lists_todo.keys():
340             summary += "Announcing to %s\n" % (announce_list)
341
342             if action:
343                 self.update_subst()
344                 self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
345                 if cnf.get("Dinstall::TrackingServer") and \
346                    self.pkg.changes["architecture"].has_key("source"):
347                     trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
348                     self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
349
350                 mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
351                 utils.send_mail(mail_message)
352
353                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
354
355         if cnf.find_b("Dinstall::CloseBugs") and cnf.has_key("Dinstall::BugServer"):
356             summary = self.close_bugs(summary, action)
357
358         del self.Subst["__SHORT_SUMMARY__"]
359
360         return summary
361
362     ###########################################################################
363
364     def check_override(self):
365         """
366         Checks override entries for validity. Mails "Override disparity" warnings,
367         if that feature is enabled.
368
369         Abandons the check if
370           - override disparity checks are disabled
371           - mail sending is disabled
372         """
373
374         cnf = Config()
375
376         # Abandon the check if override disparity checks have been disabled
377         if not cnf.find_b("Dinstall::OverrideDisparityCheck"):
378             return
379
380         summary = self.pkg.check_override()
381
382         if summary == "":
383             return
384
385         overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
386
387         self.update_subst()
388         self.Subst["__SUMMARY__"] = summary
389         mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
390         utils.send_mail(mail_message)
391         del self.Subst["__SUMMARY__"]