4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 # based on process-unchecked and process-accepted
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
36 ## pu: copy CHG to tempdir
37 ## pu: check CHG signature
38 ## pu: parse changes file
40 ## pu: check distribution (mappings, rejects)
41 ## pu: copy FILES to tempdir
42 ## pu: check whether CHG already exists in CopyChanges
43 ## pu: check whether FILES already exist in one of the policy queues
45 ## pu: extract control information
46 ## pu: various checks on control information
47 ## pu|pa: search for source (in CHG, projectb, policy queues)
48 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ## pu|pa: check whether deb already exists in the pool
51 ## pu: various checks on filenames and CHG consistency
52 ## pu: if isdsc: check signature
56 ## //pu: check whether file already exists in the pool
57 ## pu: store what "Component" the package is currently in
58 ## pu: check whether we found everything we were looking for in CHG
60 ## pu: check whether we need and have ONE DSC
62 ## pu: various checks //maybe drop some of the in favor of lintian
63 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ## pu: check whether DSC_FILES is consistent with "Format"
65 ## for src in DSC_FILES:
66 ## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ## pu: create new tempdir
68 ## pu: create symlink mirror of source
70 ## pu: extract changelog information for BTS
71 ## //pu: create missing .orig symlink
72 ## pu: check with lintian
74 ## pu: check checksums and sizes
75 ## for file in DSC_FILES:
76 ## pu: check checksums and sizes
77 ## pu: CHG: check urgency
79 ## pu: extract contents list and check for dubious timestamps
80 ## pu: check that the uploader is actually allowed to upload the package
82 ### if stable_install:
83 ### pa: remove from p-u
85 ### pa: move CHG to morgue
86 ### pa: append data to ChangeLog
88 ### pa: remove .dak file
90 ### pa: add dsc to db:
91 ### for file in DSC_FILES:
92 ### pa: add file to file
93 ### pa: add file to dsc_files
94 ### pa: create source entry
95 ### pa: update source associations
96 ### pa: update src_uploaders
98 ### pa: add deb to db:
99 ### pa: add file to file
100 ### pa: find source entry
101 ### pa: create binaries entry
102 ### pa: update binary associations
103 ### pa: .orig component move
104 ### pa: move files to pool
106 ### pa: move CHG to done/
107 ### pa: change entry in queue_build
108 ## pu: use dispatch table to choose target queue:
110 ## pu: write .dak file
114 ## pu: run autobyhand script
115 ## pu: if stuff left, do byhand or accept
116 ## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ## pu: write .dak file
118 ## pu: check overrides
122 ## pu: write .dak file
123 ## pu: move to ACCEPTED
125 ## pu: create files for BTS
126 ## pu: create entry in queue_build
127 ## pu: check overrides
131 ## Parsing changes (check for duplicates)
135 # New check layout (TODO: Implement)
139 ### version checks (suite)
153 ### src relation check
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
164 from errno import EACCES, EAGAIN
171 from sqlalchemy.orm.exc import NoResultFound
173 from daklib import daklog
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.config import Config
178 import daklib.utils as utils
179 from daklib.textutils import fix_maintainer
180 from daklib.regexes import *
182 import daklib.archive
185 ###############################################################################
190 ###############################################################################
192 def usage (exit_code=0):
193 print """Usage: dak process-upload [OPTION]... [CHANGES]...
194 -a, --automatic automatic run
195 -d, --directory <DIR> process uploads in <DIR>
196 -h, --help show this help and exit.
197 -n, --no-action don't do anything
198 -p, --no-lock don't check lockfile !! for cron.daily only !!
199 -s, --no-mail don't send any mail
200 -V, --version display the version number and exit"""
203 ###############################################################################
205 def try_or_reject(function):
206 """Try to call function or reject the upload if that fails
208 def wrapper(directory, upload, *args, **kwargs):
210 return function(directory, upload, *args, **kwargs)
211 except Exception as e:
213 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
215 return real_reject(directory, upload, reason=reason)
216 except Exception as e:
217 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
219 return real_reject(directory, upload, reason=reason, notify=False)
223 def subst_for_upload(upload):
226 changes = upload.changes
227 control = upload.changes.changes
229 if upload.final_suites is None or len(upload.final_suites) == 0:
230 suite_name = '(unknown)'
233 for suite in upload.final_suites:
234 if suite.policy_queue:
235 suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
237 suite_names.append(suite.suite_name)
238 suite_name = ','.join(suite_names)
240 maintainer_field = control.get('Maintainer', cnf['Dinstall::MyEmailAddress'])
241 changed_by_field = control.get('Changed-By', maintainer_field)
242 maintainer = fix_maintainer(changed_by_field)
243 if upload.changes.source is not None:
244 addresses = utils.mail_addresses_for_upload(maintainer_field, changed_by_field, changes.primary_fingerprint)
246 addresses = utils.mail_addresses_for_upload(maintainer_field, maintainer_field, changes.primary_fingerprint)
248 bcc = 'X-DAK: dak process-upload'
249 if 'Dinstall::Bcc' in cnf:
250 bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
253 '__DISTRO__': cnf['Dinstall::MyDistribution'],
254 '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
256 '__CHANGES_FILENAME__': upload.changes.filename,
258 '__SOURCE__': control.get('Source', '(unknown)'),
259 '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
260 '__VERSION__': control.get('Version', '(unknown)'),
262 '__SUITE__': suite_name,
264 '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
265 '__MAINTAINER_FROM__': maintainer[1],
266 '__MAINTAINER_TO__': ", ".join(addresses),
267 '__MAINTAINER__': changed_by_field,
270 '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
272 '__FILE_CONTENTS__': open(upload.changes.path, 'r').read(),
275 override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
276 if override_maintainer:
277 subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
282 def accept(directory, upload):
285 Logger.log(['ACCEPT', upload.changes.filename])
289 accepted_to_real_suite = False
290 for suite in upload.final_suites:
291 accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
293 sourceful_upload = 'source' in upload.changes.architectures
295 control = upload.changes.changes
296 if sourceful_upload and not Options['No-Action']:
297 urgency = control.get('Urgency')
298 if urgency not in cnf.value_list('Urgency::Valid'):
299 urgency = cnf['Urgency::Default']
300 UrgencyLog().log(control['Source'], control['Version'], urgency)
302 # send mail to maintainer
303 subst = subst_for_upload(upload)
304 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
305 utils.send_mail(message)
307 # send mail to announce lists and tracking server
308 if accepted_to_real_suite and sourceful_upload:
309 subst = subst_for_upload(upload)
311 for suite in upload.final_suites:
312 if suite.policy_queue is not None:
314 announce.update(suite.announce or [])
315 announce_address = ", ".join(announce)
317 tracking = cnf.get('Dinstall::TrackingServer')
318 if tracking and 'source' in upload.changes.architectures:
319 announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
321 subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address
323 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
324 utils.send_mail(message)
326 # Only close bugs for uploads that were not redirected to a policy queue.
327 # process-policy will close bugs for those once they are accepted.
328 subst = subst_for_upload(upload)
329 if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and sourceful_upload:
330 for bugnum in upload.changes.closed_bugs:
331 subst['__BUG_NUMBER__'] = str(bugnum)
333 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
334 utils.send_mail(message)
336 del subst['__BUG_NUMBER__']
338 # Move .changes to done, but only for uploads that were accepted to a
339 # real suite. process-policy will handle this for uploads to queues.
340 if accepted_to_real_suite:
341 src = os.path.join(upload.directory, upload.changes.filename)
343 now = datetime.datetime.now()
344 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
345 dst = os.path.join(donedir, upload.changes.filename)
346 dst = utils.find_next_free(dst)
348 upload.transaction.fs.copy(src, dst, mode=0o644)
350 SummaryStats().accept_count += 1
351 SummaryStats().accept_bytes += upload.changes.bytes
354 def accept_to_new(directory, upload):
357 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
359 upload.install_to_new()
360 # TODO: tag bugs pending, send announcement
362 subst = subst_for_upload(upload)
363 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
364 utils.send_mail(message)
366 SummaryStats().accept_count += 1
367 SummaryStats().accept_bytes += upload.changes.bytes
370 def reject(directory, upload, reason=None, notify=True):
371 real_reject(directory, upload, reason, notify)
373 def real_reject(directory, upload, reason=None, notify=True):
374 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
377 Logger.log(['REJECT', upload.changes.filename])
379 fs = upload.transaction.fs
380 rejectdir = cnf['Dir::Reject']
382 files = [ f.filename for f in upload.changes.files.itervalues() ]
383 files.append(upload.changes.filename)
386 src = os.path.join(upload.directory, fn)
387 dst = utils.find_next_free(os.path.join(rejectdir, fn))
388 if not os.path.exists(src):
392 if upload.reject_reasons is not None:
395 reason = reason + '\n' + '\n'.join(upload.reject_reasons)
398 reason = '(Unknown reason. Please check logs.)'
400 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
407 subst = subst_for_upload(upload)
408 subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
409 subst['__MANUAL_REJECT_MESSAGE__'] = ''
410 subst['__REJECT_MESSAGE__'] = reason
411 subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
413 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
414 utils.send_mail(message)
416 SummaryStats().reject_count += 1
418 ###############################################################################
420 def action(directory, upload):
421 changes = upload.changes
428 okay = upload.check()
430 summary = changes.changes.get('Changes', '')
434 if changes.source is not None:
435 package_info.append("source:{0}".format(changes.source.dsc['Source']))
436 for binary in changes.binaries:
437 package_info.append("binary:{0}".format(binary.control['Package']))
439 (prompt, answer) = ("", "XXX")
440 if Options["No-Action"] or Options["Automatic"]:
447 print "\n".join(package_info)
450 if len(upload.reject_reasons) > 0:
452 print "\n".join(upload.reject_reasons)
455 path = os.path.join(directory, changes.filename)
456 created = os.stat(path).st_mtime
458 too_new = (now - created < int(cnf['Dinstall::SkipTime']))
461 print "SKIP (too new)"
462 prompt = "[S]kip, Quit ?"
464 prompt = "[R]eject, Skip, Quit ?"
465 if Options["Automatic"]:
468 prompt = "[N]ew, Skip, Quit ?"
469 if Options['Automatic']:
472 prompt = "[A]ccept, Skip, Quit ?"
473 if Options['Automatic']:
476 while prompt.find(answer) == -1:
477 answer = utils.our_raw_input(prompt)
478 m = re_default_answer.match(prompt)
481 answer = answer[:1].upper()
484 reject(directory, upload)
486 # upload.try_autobyhand must not be run with No-Action.
487 if Options['No-Action']:
488 accept(directory, upload)
489 elif upload.try_autobyhand():
490 accept(directory, upload)
492 print "W: redirecting to BYHAND as automatic processing failed."
493 accept_to_new(directory, upload)
495 accept_to_new(directory, upload)
501 #raise Exception("FAIL")
502 if not Options['No-Action']:
507 ###############################################################################
509 def unlink_if_exists(path):
513 if e.errno != errno.ENOENT:
516 def process_it(directory, changes, keyrings, session):
519 print "\n{0}\n".format(changes.filename)
520 Logger.log(["Processing changes file", changes.filename])
524 # Some defaults in case we can't fully process the .changes file
525 #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
526 #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
528 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
529 bcc = "X-DAK: dak process-upload"
530 #if cnf.has_key("Dinstall::Bcc"):
531 # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
533 # u.Subst["__BCC__"] = bcc
535 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
536 processed = action(directory, upload)
537 if processed and not Options['No-Action']:
538 unlink_if_exists(os.path.join(directory, changes.filename))
539 for fn in changes.files:
540 unlink_if_exists(os.path.join(directory, fn))
542 ###############################################################################
544 def process_changes(changes_filenames):
545 session = DBConn().session()
546 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
547 keyring_files = [ k.keyring_name for k in keyrings ]
550 for fn in changes_filenames:
552 directory, filename = os.path.split(fn)
553 c = daklib.upload.Changes(directory, filename, keyring_files)
554 changes.append([directory, c])
555 except Exception as e:
556 Logger.log([filename, "Error while loading changes: {0}".format(e)])
558 changes.sort(key=lambda x: x[1])
560 for directory, c in changes:
561 process_it(directory, c, keyring_files, session)
565 ###############################################################################
568 global Options, Logger
571 summarystats = SummaryStats()
573 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
574 ('h',"help","Dinstall::Options::Help"),
575 ('n',"no-action","Dinstall::Options::No-Action"),
576 ('p',"no-lock", "Dinstall::Options::No-Lock"),
577 ('s',"no-mail", "Dinstall::Options::No-Mail"),
578 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
580 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
581 "version", "directory"]:
582 if not cnf.has_key("Dinstall::Options::%s" % (i)):
583 cnf["Dinstall::Options::%s" % (i)] = ""
585 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
586 Options = cnf.subtree("Dinstall::Options")
591 # -n/--dry-run invalidates some other options which would involve things happening
592 if Options["No-Action"]:
593 Options["Automatic"] = ""
595 # Check that we aren't going to clash with the daily cron job
596 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
597 utils.fubar("Archive maintenance in progress. Try again later.")
599 # Obtain lock if not in no-action mode and initialize the log
600 if not Options["No-Action"]:
601 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
603 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
605 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
606 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
610 # Initialise UrgencyLog() - it will deal with the case where we don't
611 # want to log urgencies
612 urgencylog = UrgencyLog()
614 Logger = daklog.Logger("process-upload", Options["No-Action"])
616 # If we have a directory flag, use it to find our files
617 if cnf["Dinstall::Options::Directory"] != "":
618 # Note that we clobber the list of files we were given in this case
619 # so warn if the user has done both
620 if len(changes_files) > 0:
621 utils.warn("Directory provided so ignoring files given on command line")
623 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
624 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
625 elif not len(changes_files) > 0:
626 utils.fubar("No changes files given and no directory specified")
628 Logger.log(["Using changes files from command-line", len(changes_files)])
630 process_changes(changes_files)
632 if summarystats.accept_count:
634 if summarystats.accept_count > 1:
636 print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
637 utils.size_type(int(summarystats.accept_bytes)))
638 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
640 if summarystats.reject_count:
642 if summarystats.reject_count > 1:
644 print "Rejected %d package %s." % (summarystats.reject_count, sets)
645 Logger.log(["rejected", summarystats.reject_count])
647 if not Options["No-Action"]:
652 ###############################################################################
654 if __name__ == '__main__':