2 # vim:set et ts=4 sw=4:
4 """ Handles packages from policy queues
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @copyright: 2009 Mark Hymers <mhy@debian.org>
11 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 # <mhy> So how do we handle that at the moment?
30 # <stew> Probably incorrectly.
32 ################################################################################
40 from sqlalchemy.orm.exc import NoResultFound
42 from daklib.dbconn import *
43 from daklib import daklog
44 from daklib import utils
45 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
46 from daklib.config import Config
47 from daklib.archive import ArchiveTransaction, source_component_from_package_list
48 from daklib.urgencylog import UrgencyLog
49 from daklib.packagelist import PackageList
51 import daklib.announce
58 ################################################################################
60 def do_comments(dir, srcqueue, opref, npref, line, fn, transaction):
61 session = transaction.session
63 for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
64 lines = open(os.path.join(dir, comm)).readlines()
65 if len(lines) == 0 or lines[0] != line + "\n": continue
67 # If the ACCEPT includes a _<arch> we only accept that .changes.
68 # Otherwise we accept all .changes that start with the given prefix
69 changes_prefix = comm[len(opref):]
70 if changes_prefix.count('_') < 2:
71 changes_prefix = changes_prefix + '_'
73 changes_prefix = changes_prefix + '.changes'
75 # We need to escape "_" as we use it with the LIKE operator (via the
76 # SQLA startwith) later.
77 changes_prefix = changes_prefix.replace("_", r"\_")
79 uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=srcqueue) \
80 .join(PolicyQueueUpload.changes).filter(DBChange.changesname.startswith(changes_prefix)) \
81 .order_by(PolicyQueueUpload.source_id)
82 reason = "".join(lines[1:])
83 actions.extend((u, reason) for u in uploads)
86 newcomm = npref + comm[len(opref):]
87 newcomm = utils.find_next_free(os.path.join(dir, newcomm))
88 transaction.fs.move(os.path.join(dir, comm), newcomm)
92 for u, reason in actions:
93 print("Processing changes file: {0}".format(u.changes.changesname))
94 fn(u, srcqueue, reason, transaction)
96 ################################################################################
98 def try_or_reject(function):
99 def wrapper(upload, srcqueue, comments, transaction):
101 function(upload, srcqueue, comments, transaction)
102 except Exception as e:
103 comments = 'An exception was raised while processing the package:\n{0}\nOriginal comments:\n{1}'.format(traceback.format_exc(), comments)
105 transaction.rollback()
106 real_comment_reject(upload, srcqueue, comments, transaction)
107 except Exception as e:
108 comments = 'In addition an exception was raised while trying to reject the upload:\n{0}\nOriginal rejection:\n{1}'.format(traceback.format_exc(), comments)
109 transaction.rollback()
110 real_comment_reject(upload, srcqueue, comments, transaction, notify=False)
111 if not Options['No-Action']:
114 transaction.rollback()
117 ################################################################################
120 def comment_accept(upload, srcqueue, comments, transaction):
121 for byhand in upload.byhand:
122 path = os.path.join(srcqueue.path, byhand.filename)
123 if os.path.exists(path):
124 raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename))
129 session = transaction.session
130 changesname = upload.changes.changesname
131 allow_tainted = srcqueue.suite.archive.tainted
133 # We need overrides to get the target component
134 overridesuite = upload.target_suite
135 if overridesuite.overridesuite is not None:
136 overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
138 def binary_component_func(db_binary):
139 section = db_binary.proxy['Section']
140 component_name = 'main'
141 if section.find('/') != -1:
142 component_name = section.split('/', 1)[0]
143 return get_mapped_component(component_name, session=session)
145 def is_debug_binary(db_binary):
146 return daklib.utils.is_in_debug_section(db_binary.proxy)
148 def has_debug_binaries(upload):
149 return any((is_debug_binary(x) for x in upload.binaries))
151 def source_component_func(db_source):
152 package_list = PackageList(db_source.proxy)
153 component = source_component_from_package_list(package_list, upload.target_suite)
154 if component is not None:
155 return get_mapped_component(component.component_name, session=session)
157 # Fallback for packages without Package-List field
158 query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
159 .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
161 return query.one().component
163 all_target_suites = [upload.target_suite]
164 all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
166 for suite in all_target_suites:
167 debug_suite = suite.debug_suite
169 if upload.source is not None:
170 # If we have Source in this upload, let's include it into
172 transaction.copy_source(
175 source_component_func(upload.source),
176 allow_tainted=allow_tainted,
179 if debug_suite is not None and has_debug_binaries(upload):
180 # If we're handing a debug package, we also need to include the
181 # source in the debug suite as well.
182 transaction.copy_source(
185 source_component_func(upload.source),
186 allow_tainted=allow_tainted,
189 for db_binary in upload.binaries:
190 # Now, let's work out where to copy this guy to -- if it's
191 # a debug binary, and the suite has a debug suite, let's go
192 # ahead and target the debug suite rather then the stock
194 copy_to_suite = suite
195 if debug_suite is not None and is_debug_binary(db_binary):
196 copy_to_suite = debug_suite
198 # build queues may miss the source package if this is a
199 # binary-only upload.
200 if suite != upload.target_suite:
201 transaction.copy_source(
204 source_component_func(db_binary.source),
205 allow_tainted=allow_tainted,
208 transaction.copy_binary(
211 binary_component_func(db_binary),
212 allow_tainted=allow_tainted,
213 extra_archives=[upload.target_suite.archive],
216 # Copy .changes if needed
217 if upload.target_suite.copychanges:
218 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
219 dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
220 fs.copy(src, dst, mode=upload.target_suite.archive.mode)
222 # Copy upload to Process-Policy::CopyDir
223 # Used on security.d.o to sync accepted packages to ftp-master, but this
224 # should eventually be replaced by something else.
225 copydir = cnf.get('Process-Policy::CopyDir') or None
226 if copydir is not None:
227 mode = upload.target_suite.archive.mode
228 if upload.source is not None:
229 for f in [ df.poolfile for df in upload.source.srcfiles ]:
230 dst = os.path.join(copydir, f.basename)
231 if not os.path.exists(dst):
232 fs.copy(f.fullpath, dst, mode=mode)
234 for db_binary in upload.binaries:
235 f = db_binary.poolfile
236 dst = os.path.join(copydir, f.basename)
237 if not os.path.exists(dst):
238 fs.copy(f.fullpath, dst, mode=mode)
240 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
241 dst = os.path.join(copydir, upload.changes.changesname)
242 if not os.path.exists(dst):
243 fs.copy(src, dst, mode=mode)
245 if upload.source is not None and not Options['No-Action']:
246 urgency = upload.changes.urgency
247 if urgency not in cnf.value_list('Urgency::Valid'):
248 urgency = cnf['Urgency::Default']
249 UrgencyLog().log(upload.source.source, upload.source.version, urgency)
252 if not Options['No-Action']:
253 Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
255 pu = get_processed_upload(upload)
256 daklib.announce.announce_accept(pu)
258 # TODO: code duplication. Similar code is in process-upload.
259 # Move .changes to done
260 src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
261 now = datetime.datetime.now()
262 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
263 dst = os.path.join(donedir, upload.changes.changesname)
264 dst = utils.find_next_free(dst)
265 fs.copy(src, dst, mode=0o644)
267 remove_upload(upload, transaction)
269 ################################################################################
272 def comment_reject(*args):
273 real_comment_reject(*args, manual=True)
275 def real_comment_reject(upload, srcqueue, comments, transaction, notify=True, manual=False):
279 session = transaction.session
280 changesname = upload.changes.changesname
281 queuedir = upload.policy_queue.path
282 rejectdir = cnf['Dir::Reject']
284 ### Copy files to reject/
286 poolfiles = [b.poolfile for b in upload.binaries]
287 if upload.source is not None:
288 poolfiles.extend([df.poolfile for df in upload.source.srcfiles])
290 files = [ af.path for af in session.query(ArchiveFile) \
291 .filter_by(archive=upload.policy_queue.suite.archive) \
292 .join(ArchiveFile.file) \
293 .filter(PoolFile.file_id.in_([ f.file_id for f in poolfiles ])) ]
294 for byhand in upload.byhand:
295 path = os.path.join(queuedir, byhand.filename)
296 if os.path.exists(path):
298 files.append(os.path.join(queuedir, changesname))
301 dst = utils.find_next_free(os.path.join(rejectdir, os.path.basename(fn)))
302 fs.copy(fn, dst, link=True)
306 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(changesname)))
311 ### Send mail notification
317 # Try to use From: from comment file if there is one.
318 # This is not very elegant...
319 match = re.match(r"\AFrom: ([^\n]+)\n\n", comments)
321 rejected_by = match.group(1)
322 reason = '\n'.join(comments.splitlines()[2:])
324 pu = get_processed_upload(upload)
325 daklib.announce.announce_reject(pu, reason, rejected_by)
328 if not Options["No-Action"]:
329 Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname])
331 changes = upload.changes
332 remove_upload(upload, transaction)
333 session.delete(changes)
335 ################################################################################
337 def remove_upload(upload, transaction):
339 session = transaction.session
340 changes = upload.changes
342 # Remove byhand and changes files. Binary and source packages will be
343 # removed from {bin,src}_associations and eventually removed by clean-suites automatically.
344 queuedir = upload.policy_queue.path
345 for byhand in upload.byhand:
346 path = os.path.join(queuedir, byhand.filename)
347 if os.path.exists(path):
349 session.delete(byhand)
350 fs.unlink(os.path.join(queuedir, upload.changes.changesname))
352 session.delete(upload)
355 ################################################################################
357 def get_processed_upload(upload):
358 pu = daklib.announce.ProcessedUpload()
360 pu.maintainer = upload.changes.maintainer
361 pu.changed_by = upload.changes.changedby
362 pu.fingerprint = upload.changes.fingerprint
364 pu.suites = [ upload.target_suite ]
365 pu.from_policy_suites = [ upload.target_suite ]
367 changes_path = os.path.join(upload.policy_queue.path, upload.changes.changesname)
368 pu.changes = open(changes_path, 'r').read()
369 pu.changes_filename = upload.changes.changesname
370 pu.sourceful = upload.source is not None
371 pu.source = upload.changes.source
372 pu.version = upload.changes.version
373 pu.architecture = upload.changes.architecture
374 pu.bugs = upload.changes.closes
376 pu.program = "process-policy"
380 ################################################################################
382 def remove_unreferenced_binaries(policy_queue, transaction):
383 """Remove binaries that are no longer referenced by an upload
385 @type policy_queue: L{daklib.dbconn.PolicyQueue}
387 @type transaction: L{daklib.archive.ArchiveTransaction}
389 session = transaction.session
390 suite = policy_queue.suite
395 JOIN bin_associations ba ON b.id = ba.bin
396 WHERE ba.suite = :suite_id
397 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload_binaries_map pqubm
398 JOIN policy_queue_upload pqu ON pqubm.policy_queue_upload_id = pqu.id
399 WHERE pqu.policy_queue_id = :policy_queue_id
400 AND pqubm.binary_id = b.id)"""
401 binaries = session.query(DBBinary).from_statement(query) \
402 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id})
404 for binary in binaries:
405 Logger.log(["removed binary from policy queue", policy_queue.queue_name, binary.package, binary.version])
406 transaction.remove_binary(binary, suite)
408 def remove_unreferenced_sources(policy_queue, transaction):
409 """Remove sources that are no longer referenced by an upload or a binary
411 @type policy_queue: L{daklib.dbconn.PolicyQueue}
413 @type transaction: L{daklib.archive.ArchiveTransaction}
415 session = transaction.session
416 suite = policy_queue.suite
421 JOIN src_associations sa ON s.id = sa.source
422 WHERE sa.suite = :suite_id
423 AND NOT EXISTS (SELECT 1 FROM policy_queue_upload pqu
424 WHERE pqu.policy_queue_id = :policy_queue_id
425 AND pqu.source_id = s.id)
426 AND NOT EXISTS (SELECT 1 FROM binaries b
427 JOIN bin_associations ba ON b.id = ba.bin
428 WHERE b.source = s.id
429 AND ba.suite = :suite_id)"""
430 sources = session.query(DBSource).from_statement(query) \
431 .params({'suite_id': policy_queue.suite_id, 'policy_queue_id': policy_queue.policy_queue_id})
433 for source in sources:
434 Logger.log(["removed source from policy queue", policy_queue.queue_name, source.source, source.version])
435 transaction.remove_source(source, suite)
437 ################################################################################
440 global Options, Logger
443 session = DBConn().session()
445 Arguments = [('h',"help","Process-Policy::Options::Help"),
446 ('n',"no-action","Process-Policy::Options::No-Action")]
448 for i in ["help", "no-action"]:
449 if not cnf.has_key("Process-Policy::Options::%s" % (i)):
450 cnf["Process-Policy::Options::%s" % (i)] = ""
452 queue_name = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
454 if len(queue_name) != 1:
455 print "E: Specify exactly one policy queue"
458 queue_name = queue_name[0]
460 Options = cnf.subtree("Process-Policy::Options")
465 Logger = daklog.Logger("process-policy")
466 if not Options["No-Action"]:
467 urgencylog = UrgencyLog()
469 with ArchiveTransaction() as transaction:
470 session = transaction.session
472 pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
473 except NoResultFound:
474 print "E: Cannot find policy queue %s" % queue_name
477 commentsdir = os.path.join(pq.path, 'COMMENTS')
478 # The comments stuff relies on being in the right directory
481 do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction)
482 do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction)
483 do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction)
485 remove_unreferenced_binaries(pq, transaction)
486 remove_unreferenced_sources(pq, transaction)
488 if not Options['No-Action']:
491 ################################################################################
493 if __name__ == '__main__':