2 # vim:set et ts=4 sw=4:
4 """ Handles packages from policy queues
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @copyright: 2009 Mark Hymers <mhy@debian.org>
11 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # <mhy> So how do we handle that at the moment?
30 # <stew> Probably incorrectly.
32 ################################################################################
40 from daklib.dbconn import *
41 from daklib import daklog
42 from daklib import utils
43 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
44 from daklib.config import Config
45 from daklib.archive import ArchiveTransaction
46 from daklib.urgencylog import UrgencyLog
47 from daklib.textutils import fix_maintainer
53 ################################################################################
55 def do_comments(dir, srcqueue, opref, npref, line, fn, transaction):
56 session = transaction.session
57 for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
58 lines = open(os.path.join(dir, comm)).readlines()
59 if len(lines) == 0 or lines[0] != line + "\n": continue
61 # If the ACCEPT includes a _<arch> we only accept that .changes.
62 # Otherwise we accept all .changes that start with the given prefix
63 changes_prefix = comm[len(opref):]
64 if changes_prefix.count('_') < 2:
65 changes_prefix = changes_prefix + '_'
67 changes_prefix = changes_prefix + '.changes'
69 uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=srcqueue) \
70 .join(PolicyQueueUpload.changes).filter(DBChange.changesname.startswith(changes_prefix)) \
71 .order_by(PolicyQueueUpload.source_id)
73 print "Processing changes file: %s" % u.changes.changesname
74 fn(u, srcqueue, "".join(lines[1:]), transaction)
77 newcomm = npref + comm[len(opref):]
78 transaction.fs.move(os.path.join(dir, comm), os.path.join(dir, newcomm))
80 ################################################################################
82 def try_or_reject(function):
83 def wrapper(upload, srcqueue, comments, transaction):
85 function(upload, srcqueue, comments, transaction)
86 except Exception as e:
87 comments = 'An exception was raised while processing the package:\n{0}\nOriginal comments:\n{1}'.format(traceback.format_exc(), comments)
89 transaction.rollback()
90 real_comment_reject(upload, srcqueue, comments, transaction)
91 except Exception as e:
92 comments = 'In addition an exception was raised while trying to reject the upload:\n{0}\nOriginal rejection:\n{1}'.format(traceback.format_exc(), comments)
93 transaction.rollback()
94 real_comment_reject(upload, srcqueue, comments, transaction, notify=False)
95 if not Options['No-Action']:
99 ################################################################################
102 def comment_accept(upload, srcqueue, comments, transaction):
103 for byhand in upload.byhand:
104 path = os.path.join(srcqueue.path, byhand.filename)
105 if os.path.exists(path):
106 raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename))
111 session = transaction.session
112 changesname = upload.changes.changesname
113 allow_tainted = srcqueue.suite.archive.tainted
115 # We need overrides to get the target component
116 overridesuite = upload.target_suite
117 if overridesuite.overridesuite is not None:
118 overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
120 def binary_component_func(db_binary):
121 override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \
122 .join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \
123 .join(Component).one()
124 return override.component
126 def source_component_func(db_source):
127 override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
128 .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
129 .join(Component).one()
130 return override.component
132 all_target_suites = [upload.target_suite]
133 all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
135 for suite in all_target_suites:
136 if upload.source is not None:
137 transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted)
138 for db_binary in upload.binaries:
139 transaction.copy_binary(db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive])
141 # Copy .changes if needed
142 if upload.target_suite.copychanges:
143 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
144 dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
145 fs.copy(src, dst, mode=upload.target_suite.archive.mode)
147 if upload.source is not None and not Options['No-Action']:
148 urgency = upload.changes.urgency
149 if urgency not in cnf.value_list('Urgency::Valid'):
150 urgency = cnf['Urgency::Default']
151 UrgencyLog().log(upload.source.source, upload.source.version, urgency)
154 if not Options['No-Action']:
155 Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
158 subst = subst_for_upload(upload)
159 announce = ", ".join(upload.target_suite.announce or [])
160 tracking = cnf.get('Dinstall::TrackingServer')
161 if tracking and upload.source is not None:
162 announce = '{0}\nBcc: {1}@{2}'.format(announce, upload.changes.source, tracking)
163 subst['__ANNOUNCE_LIST_ADDRESS__'] = announce
164 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
165 utils.send_mail(message)
167 # TODO: code duplication. Similar code is in process-upload.
168 if cnf.find_b('Dinstall::CloseBugs') and upload.changes.closes is not None and upload.source is not None:
169 for bugnum in upload.changes.closes:
170 subst['__BUG_NUMBER__'] = bugnum
171 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
172 utils.send_mail(message)
174 del subst['__BUG_NUMBER__']
176 # TODO: code duplication. Similar code is in process-upload.
177 # Move .changes to done
178 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
179 now = datetime.datetime.now()
180 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
181 dst = os.path.join(donedir, upload.changes.changesname)
182 dst = utils.find_next_free(dst)
183 fs.copy(src, dst, mode=0o644)
185 remove_upload(upload, transaction)
187 ################################################################################
190 def comment_reject(*args):
191 real_comment_reject(*args)
193 def real_comment_reject(upload, srcqueue, comments, transaction, notify=True):
197 session = transaction.session
198 changesname = upload.changes.changesname
199 queuedir = upload.policy_queue.path
200 rejectdir = cnf['Dir::Reject']
202 ### Copy files to reject/
204 poolfiles = [b.poolfile for b in upload.binaries]
205 if upload.source is not None:
206 poolfiles.extend([df.poolfile for df in upload.source.srcfiles])
208 files = [ af.path for af in session.query(ArchiveFile) \
209 .filter_by(archive=upload.policy_queue.suite.archive) \
210 .join(ArchiveFile.file) \
211 .filter(PoolFile.file_id.in_([ f.file_id for f in poolfiles ])) ]
212 for byhand in upload.byhand:
213 path = os.path.join(queuedir, byhand.filename)
214 if os.path.exists(path):
216 files.append(os.path.join(queuedir, changesname))
219 dst = utils.find_next_free(os.path.join(rejectdir, os.path.basename(fn)))
220 fs.copy(fn, dst, link=True)
224 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(changesname)))
229 ### Send mail notification
232 subst = subst_for_upload(upload)
233 subst['__MANUAL_REJECT_MESSAGE__'] = ''
234 subst['__REJECT_MESSAGE__'] = comments
235 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
236 utils.send_mail(message)
239 if not Options["No-Action"]:
240 Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname])
242 remove_upload(upload, transaction)
244 ################################################################################
246 def remove_upload(upload, transaction):
248 session = transaction.session
249 changes = upload.changes
251 # Remove byhand and changes files. Binary and source packages will be
252 # removed from {bin,src}_associations and eventually removed by clean-suites automatically.
253 queuedir = upload.policy_queue.path
254 for byhand in upload.byhand:
255 path = os.path.join(queuedir, byhand.filename)
256 if os.path.exists(path):
258 session.delete(byhand)
259 fs.unlink(os.path.join(queuedir, upload.changes.changesname))
261 session.delete(upload)
262 session.delete(changes)
265 ################################################################################
267 def subst_for_upload(upload):
268 # TODO: similar code in process-upload
271 maintainer_field = upload.changes.changedby or upload.changes.maintainer
272 addresses = utils.mail_addresses_for_upload(upload.changes.maintainer, maintainer_field, upload.changes.fingerprint)
274 changes_path = os.path.join(upload.policy_queue.path, upload.changes.changesname)
275 changes_contents = open(changes_path, 'r').read()
277 bcc = 'X-DAK: dak process-policy'
278 if 'Dinstall::Bcc' in cnf:
279 bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
282 '__DISTRO__': cnf['Dinstall::MyDistribution'],
283 '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
285 '__CHANGES_FILENAME__': upload.changes.changesname,
286 '__SOURCE__': upload.changes.source,
287 '__VERSION__': upload.changes.version,
288 '__ARCHITECTURE__': upload.changes.architecture,
289 '__MAINTAINER__': maintainer_field,
290 '__MAINTAINER_FROM__': fix_maintainer(maintainer_field)[1],
291 '__MAINTAINER_TO__': ", ".join(addresses),
292 '__CC__': 'X-DAK-Rejection: manual or automatic',
293 '__REJECTOR_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
295 '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
296 '__FILE_CONTENTS__': changes_contents,
299 override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
300 if override_maintainer:
301 subst['__MAINTAINER_TO__'] = override_maintainer
305 ################################################################################
307 def remove_unreferenced_binaries(policy_queue, transaction):
308 """Remove binaries that are no longer referenced by an upload
310 @type policy_queue: L{daklib.dbconn.PolicyQueue}
312 @type transaction: L{daklib.archive.ArchiveTransaction}
314 session = transaction.session
315 suite = policy_queue.suite
320 JOIN bin_associations ba ON b.id = ba.bin
321 WHERE ba.suite = :suite_id
322 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload_binaries_map pqubm
323 JOIN policy_queue_upload pqu ON pqubm.policy_queue_upload_id = pqu.id
324 WHERE pqu.policy_queue_id = :policy_queue_id
325 AND pqubm.binary_id = b.id)"""
326 binaries = session.query(DBBinary).from_statement(query) \
327 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id})
329 for binary in binaries:
330 Logger.log(["removed binary from policy queue", policy_queue.queue_name, binary.package, binary.version])
331 transaction.remove_binary(binary, suite)
333 def remove_unreferenced_sources(policy_queue, transaction):
334 """Remove sources that are no longer referenced by an upload or a binary
336 @type policy_queue: L{daklib.dbconn.PolicyQueue}
338 @type transaction: L{daklib.archive.ArchiveTransaction}
340 session = transaction.session
341 suite = policy_queue.suite
346 JOIN src_associations sa ON s.id = sa.source
347 WHERE sa.suite = :suite_id
348 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload pqu
349 WHERE pqu.policy_queue_id = :policy_queue_id
350 AND pqu.source_id = s.id)
351 AND NOT EXISTS (SELECT 1 FROM binaries b
352 JOIN bin_associations ba ON b.id = ba.bin
353 WHERE b.source = s.id
354 AND ba.suite = :suite_id)"""
355 sources = session.query(DBSource).from_statement(query) \
356 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id})
358 for source in sources:
359 Logger.log(["removed source from policy queue", policy_queue.queue_name, source.source, source.version])
360 transaction.remove_source(source, suite)
362 ################################################################################
365 global Options, Logger
368 session = DBConn().session()
370 Arguments = [('h',"help","Process-Policy::Options::Help"),
371 ('n',"no-action","Process-Policy::Options::No-Action")]
373 for i in ["help", "no-action"]:
374 if not cnf.has_key("Process-Policy::Options::%s" % (i)):
375 cnf["Process-Policy::Options::%s" % (i)] = ""
377 queue_name = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
379 if len(queue_name) != 1:
380 print "E: Specify exactly one policy queue"
383 queue_name = queue_name[0]
385 Options = cnf.subtree("Process-Policy::Options")
390 Logger = daklog.Logger("process-policy")
391 if not Options["No-Action"]:
392 urgencylog = UrgencyLog()
394 with ArchiveTransaction() as transaction:
395 session = transaction.session
397 pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
398 except NoResultFound:
399 print "E: Cannot find policy queue %s" % queue_name
402 commentsdir = os.path.join(pq.path, 'COMMENTS')
403 # The comments stuff relies on being in the right directory
406 do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction)
407 do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction)
408 do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction)
410 remove_unreferenced_binaries(pq, transaction)
411 remove_unreferenced_sources(pq, transaction)
413 if not Options['No-Action']:
416 ################################################################################
418 if __name__ == '__main__':