2 # vim:set et ts=4 sw=4:
4 """ Handles packages from policy queues
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @copyright: 2009 Mark Hymers <mhy@debian.org>
11 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # <mhy> So how do we handle that at the moment?
30 # <stew> Probably incorrectly.
32 ################################################################################
41 from daklib.dbconn import *
42 from daklib import daklog
43 from daklib import utils
44 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
45 from daklib.config import Config
46 from daklib.archive import ArchiveTransaction
47 from daklib.urgencylog import UrgencyLog
49 import daklib.announce
55 ################################################################################
57 def do_comments(dir, srcqueue, opref, npref, line, fn, transaction):
58 session = transaction.session
59 for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
60 lines = open(os.path.join(dir, comm)).readlines()
61 if len(lines) == 0 or lines[0] != line + "\n": continue
63 # If the ACCEPT includes a _<arch> we only accept that .changes.
64 # Otherwise we accept all .changes that start with the given prefix
65 changes_prefix = comm[len(opref):]
66 if changes_prefix.count('_') < 2:
67 changes_prefix = changes_prefix + '_'
69 changes_prefix = changes_prefix + '.changes'
71 uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=srcqueue) \
72 .join(PolicyQueueUpload.changes).filter(DBChange.changesname.startswith(changes_prefix)) \
73 .order_by(PolicyQueueUpload.source_id)
75 print "Processing changes file: %s" % u.changes.changesname
76 fn(u, srcqueue, "".join(lines[1:]), transaction)
79 newcomm = npref + comm[len(opref):]
80 transaction.fs.move(os.path.join(dir, comm), os.path.join(dir, newcomm))
82 ################################################################################
84 def try_or_reject(function):
85 def wrapper(upload, srcqueue, comments, transaction):
87 function(upload, srcqueue, comments, transaction)
88 except Exception as e:
89 comments = 'An exception was raised while processing the package:\n{0}\nOriginal comments:\n{1}'.format(traceback.format_exc(), comments)
91 transaction.rollback()
92 real_comment_reject(upload, srcqueue, comments, transaction)
93 except Exception as e:
94 comments = 'In addition an exception was raised while trying to reject the upload:\n{0}\nOriginal rejection:\n{1}'.format(traceback.format_exc(), comments)
95 transaction.rollback()
96 real_comment_reject(upload, srcqueue, comments, transaction, notify=False)
97 if not Options['No-Action']:
101 ################################################################################
104 def comment_accept(upload, srcqueue, comments, transaction):
105 for byhand in upload.byhand:
106 path = os.path.join(srcqueue.path, byhand.filename)
107 if os.path.exists(path):
108 raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename))
113 session = transaction.session
114 changesname = upload.changes.changesname
115 allow_tainted = srcqueue.suite.archive.tainted
117 # We need overrides to get the target component
118 overridesuite = upload.target_suite
119 if overridesuite.overridesuite is not None:
120 overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
122 def binary_component_func(db_binary):
123 override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \
124 .join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \
125 .join(Component).one()
126 return override.component
128 def source_component_func(db_source):
129 override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
130 .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
131 .join(Component).one()
132 return override.component
134 all_target_suites = [upload.target_suite]
135 all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
137 for suite in all_target_suites:
138 if upload.source is not None:
139 transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted)
140 for db_binary in upload.binaries:
141 # build queues may miss the source package if this is a binary-only upload
142 if suite != upload.target_suite:
143 transaction.copy_source(db_binary.source, suite, source_component_func(db_binary.source), allow_tainted=allow_tainted)
144 transaction.copy_binary(db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive])
146 # Copy .changes if needed
147 if upload.target_suite.copychanges:
148 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
149 dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
150 fs.copy(src, dst, mode=upload.target_suite.archive.mode)
152 if upload.source is not None and not Options['No-Action']:
153 urgency = upload.changes.urgency
154 if urgency not in cnf.value_list('Urgency::Valid'):
155 urgency = cnf['Urgency::Default']
156 UrgencyLog().log(upload.source.source, upload.source.version, urgency)
159 if not Options['No-Action']:
160 Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
162 pu = get_processed_upload(upload)
163 daklib.announce.announce_accept(pu)
165 # TODO: code duplication. Similar code is in process-upload.
166 # Move .changes to done
167 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
168 now = datetime.datetime.now()
169 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
170 dst = os.path.join(donedir, upload.changes.changesname)
171 dst = utils.find_next_free(dst)
172 fs.copy(src, dst, mode=0o644)
174 remove_upload(upload, transaction)
176 ################################################################################
179 def comment_reject(*args):
180 real_comment_reject(*args, manual=True)
182 def real_comment_reject(upload, srcqueue, comments, transaction, notify=True, manual=False):
186 session = transaction.session
187 changesname = upload.changes.changesname
188 queuedir = upload.policy_queue.path
189 rejectdir = cnf['Dir::Reject']
191 ### Copy files to reject/
193 poolfiles = [b.poolfile for b in upload.binaries]
194 if upload.source is not None:
195 poolfiles.extend([df.poolfile for df in upload.source.srcfiles])
197 files = [ af.path for af in session.query(ArchiveFile) \
198 .filter_by(archive=upload.policy_queue.suite.archive) \
199 .join(ArchiveFile.file) \
200 .filter(PoolFile.file_id.in_([ f.file_id for f in poolfiles ])) ]
201 for byhand in upload.byhand:
202 path = os.path.join(queuedir, byhand.filename)
203 if os.path.exists(path):
205 files.append(os.path.join(queuedir, changesname))
208 dst = utils.find_next_free(os.path.join(rejectdir, os.path.basename(fn)))
209 fs.copy(fn, dst, link=True)
213 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(changesname)))
218 ### Send mail notification
224 # Try to use From: from comment file if there is one.
225 # This is not very elegant...
226 match = re.match(r"\AFrom: ([^\n]+)\n\n", comments)
228 rejected_by = match.group(1)
229 reason = '\n'.join(comments.splitlines()[2:])
231 pu = get_processed_upload(upload)
232 daklib.announce.announce_reject(pu, reason, rejected_by)
235 if not Options["No-Action"]:
236 Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname])
238 changes = upload.changes
239 remove_upload(upload, transaction)
240 session.delete(changes)
242 ################################################################################
244 def remove_upload(upload, transaction):
246 session = transaction.session
247 changes = upload.changes
249 # Remove byhand and changes files. Binary and source packages will be
250 # removed from {bin,src}_associations and eventually removed by clean-suites automatically.
251 queuedir = upload.policy_queue.path
252 for byhand in upload.byhand:
253 path = os.path.join(queuedir, byhand.filename)
254 if os.path.exists(path):
256 session.delete(byhand)
257 fs.unlink(os.path.join(queuedir, upload.changes.changesname))
259 session.delete(upload)
262 ################################################################################
264 def get_processed_upload(upload):
265 pu = daklib.announce.ProcessedUpload()
267 pu.maintainer = upload.changes.maintainer
268 pu.changed_by = upload.changes.changedby
269 pu.fingerprint = upload.changes.fingerprint
271 pu.suites = [ upload.target_suite ]
272 pu.from_policy_suites = [ upload.target_suite ]
274 changes_path = os.path.join(upload.policy_queue.path, upload.changes.changesname)
275 pu.changes = open(changes_path, 'r').read()
276 pu.changes_filename = upload.changes.changesname
277 pu.sourceful = upload.source is not None
278 pu.source = upload.changes.source
279 pu.version = upload.changes.version
280 pu.architecture = upload.changes.architecture
281 pu.bugs = upload.changes.closes
283 pu.program = "process-policy"
287 ################################################################################
289 def remove_unreferenced_binaries(policy_queue, transaction):
290 """Remove binaries that are no longer referenced by an upload
292 @type policy_queue: L{daklib.dbconn.PolicyQueue}
294 @type transaction: L{daklib.archive.ArchiveTransaction}
296 session = transaction.session
297 suite = policy_queue.suite
302 JOIN bin_associations ba ON b.id = ba.bin
303 WHERE ba.suite = :suite_id
304 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload_binaries_map pqubm
305 JOIN policy_queue_upload pqu ON pqubm.policy_queue_upload_id = pqu.id
306 WHERE pqu.policy_queue_id = :policy_queue_id
307 AND pqubm.binary_id = b.id)"""
308 binaries = session.query(DBBinary).from_statement(query) \
309 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id})
311 for binary in binaries:
312 Logger.log(["removed binary from policy queue", policy_queue.queue_name, binary.package, binary.version])
313 transaction.remove_binary(binary, suite)
315 def remove_unreferenced_sources(policy_queue, transaction):
316 """Remove sources that are no longer referenced by an upload or a binary
318 @type policy_queue: L{daklib.dbconn.PolicyQueue}
320 @type transaction: L{daklib.archive.ArchiveTransaction}
322 session = transaction.session
323 suite = policy_queue.suite
328 JOIN src_associations sa ON s.id = sa.source
329 WHERE sa.suite = :suite_id
330 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload pqu
331 WHERE pqu.policy_queue_id = :policy_queue_id
332 AND pqu.source_id = s.id)
333 AND NOT EXISTS (SELECT 1 FROM binaries b
334 JOIN bin_associations ba ON b.id = ba.bin
335 WHERE b.source = s.id
336 AND ba.suite = :suite_id)"""
337 sources = session.query(DBSource).from_statement(query) \
338 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id})
340 for source in sources:
341 Logger.log(["removed source from policy queue", policy_queue.queue_name, source.source, source.version])
342 transaction.remove_source(source, suite)
344 ################################################################################
347 global Options, Logger
350 session = DBConn().session()
352 Arguments = [('h',"help","Process-Policy::Options::Help"),
353 ('n',"no-action","Process-Policy::Options::No-Action")]
355 for i in ["help", "no-action"]:
356 if not cnf.has_key("Process-Policy::Options::%s" % (i)):
357 cnf["Process-Policy::Options::%s" % (i)] = ""
359 queue_name = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
361 if len(queue_name) != 1:
362 print "E: Specify exactly one policy queue"
365 queue_name = queue_name[0]
367 Options = cnf.subtree("Process-Policy::Options")
372 Logger = daklog.Logger("process-policy")
373 if not Options["No-Action"]:
374 urgencylog = UrgencyLog()
376 with ArchiveTransaction() as transaction:
377 session = transaction.session
379 pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
380 except NoResultFound:
381 print "E: Cannot find policy queue %s" % queue_name
384 commentsdir = os.path.join(pq.path, 'COMMENTS')
385 # The comments stuff relies on being in the right directory
388 do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction)
389 do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction)
390 do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction)
392 remove_unreferenced_binaries(pq, transaction)
393 remove_unreferenced_sources(pq, transaction)
395 if not Options['No-Action']:
398 ################################################################################
400 if __name__ == '__main__':