4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 # based on process-unchecked and process-accepted
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
36 ## pu: copy CHG to tempdir
37 ## pu: check CHG signature
38 ## pu: parse changes file
40 ## pu: check distribution (mappings, rejects)
41 ## pu: copy FILES to tempdir
42 ## pu: check whether CHG already exists in CopyChanges
43 ## pu: check whether FILES already exist in one of the policy queues
45 ## pu: extract control information
46 ## pu: various checks on control information
47 ## pu|pa: search for source (in CHG, projectb, policy queues)
48 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ## pu|pa: check whether deb already exists in the pool
51 ## pu: various checks on filenames and CHG consistency
52 ## pu: if isdsc: check signature
56 ## //pu: check whether file already exists in the pool
57 ## pu: store what "Component" the package is currently in
58 ## pu: check whether we found everything we were looking for in CHG
60 ## pu: check whether we need and have ONE DSC
62 ## pu: various checks //maybe drop some of the in favor of lintian
63 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ## pu: check whether DSC_FILES is consistent with "Format"
65 ## for src in DSC_FILES:
66 ## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ## pu: create new tempdir
68 ## pu: create symlink mirror of source
70 ## pu: extract changelog information for BTS
71 ## //pu: create missing .orig symlink
72 ## pu: check with lintian
74 ## pu: check checksums and sizes
75 ## for file in DSC_FILES:
76 ## pu: check checksums and sizes
77 ## pu: CHG: check urgency
79 ## pu: extract contents list and check for dubious timestamps
80 ## pu: check that the uploader is actually allowed to upload the package
82 ### if stable_install:
83 ### pa: remove from p-u
85 ### pa: move CHG to morgue
86 ### pa: append data to ChangeLog
88 ### pa: remove .dak file
90 ### pa: add dsc to db:
91 ### for file in DSC_FILES:
92 ### pa: add file to file
93 ### pa: add file to dsc_files
94 ### pa: create source entry
95 ### pa: update source associations
96 ### pa: update src_uploaders
98 ### pa: add deb to db:
99 ### pa: add file to file
100 ### pa: find source entry
101 ### pa: create binaries entry
102 ### pa: update binary associations
103 ### pa: .orig component move
104 ### pa: move files to pool
106 ### pa: move CHG to done/
107 ### pa: change entry in queue_build
108 ## pu: use dispatch table to choose target queue:
110 ## pu: write .dak file
114 ## pu: run autobyhand script
115 ## pu: if stuff left, do byhand or accept
116 ## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ## pu: write .dak file
118 ## pu: check overrides
122 ## pu: write .dak file
123 ## pu: move to ACCEPTED
125 ## pu: create files for BTS
126 ## pu: create entry in queue_build
127 ## pu: check overrides
131 ## Parsing changes (check for duplicates)
135 # New check layout (TODO: Implement)
139 ### version checks (suite)
153 ### src relation check
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
164 from errno import EACCES, EAGAIN
171 from sqlalchemy.orm.exc import NoResultFound
173 from daklib import daklog
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.config import Config
178 import daklib.utils as utils
179 from daklib.textutils import fix_maintainer
180 from daklib.regexes import *
182 import daklib.archive
185 ###############################################################################
190 ###############################################################################
192 def usage (exit_code=0):
193 print """Usage: dak process-upload [OPTION]... [CHANGES]...
194 -a, --automatic automatic run
195 -d, --directory <DIR> process uploads in <DIR>
196 -h, --help show this help and exit.
197 -n, --no-action don't do anything
198 -p, --no-lock don't check lockfile !! for cron.daily only !!
199 -s, --no-mail don't send any mail
200 -V, --version display the version number and exit"""
203 ###############################################################################
205 def try_or_reject(function):
206 """Try to call function or reject the upload if that fails
208 def wrapper(directory, upload, *args, **kwargs):
210 return function(directory, upload, *args, **kwargs)
211 except Exception as e:
213 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
215 return real_reject(directory, upload, reason=reason)
216 except Exception as e:
217 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
219 return real_reject(directory, upload, reason=reason, notify=False)
223 def subst_for_upload(upload):
226 changes = upload.changes
227 control = upload.changes.changes
229 if upload.final_suites is None or len(upload.final_suites) == 0:
230 suite_name = '(unknown)'
233 for suite in upload.final_suites:
234 if suite.policy_queue:
235 suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
237 suite_names.append(suite.suite_name)
238 suite_name = ','.join(suite_names)
240 maintainer_field = control.get('Changed-By', control.get('Maintainer', cnf['Dinstall::MyEmailAddress']))
241 maintainer = fix_maintainer(maintainer_field)
242 addresses = utils.mail_addresses_for_upload(control.get('Maintainer', cnf['Dinstall::MyEmailAddress']), maintainer_field, changes.primary_fingerprint)
244 bcc = 'X-DAK: dak process-upload'
245 if 'Dinstall::Bcc' in cnf:
246 bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
249 '__DISTRO__': cnf['Dinstall::MyDistribution'],
250 '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
252 '__CHANGES_FILENAME__': upload.changes.filename,
254 '__SOURCE__': control.get('Source', '(unknown)'),
255 '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
256 '__VERSION__': control.get('Version', '(unknown)'),
258 '__SUITE__': suite_name,
260 '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
261 '__MAINTAINER_FROM__': maintainer[1],
262 '__MAINTAINER_TO__': ", ".join(addresses),
263 '__MAINTAINER__': maintainer_field,
266 '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
268 # TODO: don't use private member
269 '__FILE_CONTENTS__': upload.changes._signed_file.contents,
274 override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
275 if override_maintainer:
276 subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
281 def accept(directory, upload):
284 Logger.log(['ACCEPT', upload.changes.filename])
288 accepted_to_real_suite = False
289 for suite in upload.final_suites:
290 accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
292 control = upload.changes.changes
293 if 'source' in upload.changes.architectures and not Options['No-Action']:
294 urgency = control.get('Urgency')
295 if urgency not in cnf.value_list('Urgency::Valid'):
296 urgency = cnf['Urgency::Default']
297 UrgencyLog().log(control['Source'], control['Version'], urgency)
299 # send mail to maintainer
300 subst = subst_for_upload(upload)
301 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
302 utils.send_mail(message)
304 # send mail to announce lists and tracking server
305 subst = subst_for_upload(upload)
307 for suite in upload.final_suites:
308 if suite.policy_queue is None:
310 announce.update(suite.announce or [])
311 announce_address = ", ".join(announce)
312 subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address
314 tracking = cnf.get('Dinstall::TrackingServer')
315 if tracking and 'source' in upload.changes.architectures:
316 announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
317 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
318 utils.send_mail(message)
320 del subst['__ANNOUNCE_LIST_ADDRESS__']
322 # Only close bugs for uploads that were not redirected to a policy queue.
323 # process-policy will close bugs for those once they are accepted.
324 subst = subst_for_upload(upload)
325 if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and upload.changes.source is not None:
326 for bugnum in upload.changes.closed_bugs:
327 subst['__BUG_NUMBER__'] = str(bugnum)
329 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
330 utils.send_mail(message)
332 del subst['__BUG_NUMBER__']
334 # Move .changes to done, but only for uploads that were accepted to a
335 # real suite. process-policy will handle this for uploads to queues.
336 if accepted_to_real_suite:
337 src = os.path.join(upload.directory, upload.changes.filename)
339 now = datetime.datetime.now()
340 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
341 dst = os.path.join(donedir, upload.changes.filename)
342 dst = utils.find_next_free(dst)
344 upload.transaction.fs.copy(src, dst, mode=0o644)
346 SummaryStats().accept_count += 1
347 SummaryStats().accept_bytes += upload.changes.bytes
350 def accept_to_new(directory, upload):
353 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
355 upload.install_to_new()
356 # TODO: tag bugs pending, send announcement
358 subst = subst_for_upload(upload)
359 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
360 utils.send_mail(message)
362 SummaryStats().accept_count += 1
363 SummaryStats().accept_bytes += upload.changes.bytes
366 def reject(directory, upload, reason=None, notify=True):
367 real_reject(directory, upload, reason, notify)
369 def real_reject(directory, upload, reason=None, notify=True):
370 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
373 Logger.log(['REJECT', upload.changes.filename])
375 fs = upload.transaction.fs
376 rejectdir = cnf['Dir::Reject']
378 files = [ f.filename for f in upload.changes.files.itervalues() ]
379 files.append(upload.changes.filename)
382 src = os.path.join(upload.directory, fn)
383 dst = utils.find_next_free(os.path.join(rejectdir, fn))
384 if not os.path.exists(src):
388 if upload.reject_reasons is not None:
391 reason = reason + '\n' + '\n'.join(upload.reject_reasons)
394 reason = '(Unknown reason. Please check logs.)'
396 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
403 subst = subst_for_upload(upload)
404 subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
405 subst['__MANUAL_REJECT_MESSAGE__'] = ''
406 subst['__REJECT_MESSAGE__'] = reason
407 subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
409 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
410 utils.send_mail(message)
412 SummaryStats().reject_count += 1
414 ###############################################################################
416 def action(directory, upload):
417 changes = upload.changes
424 okay = upload.check()
426 summary = changes.changes.get('Changes', '')
430 if changes.source is not None:
431 package_info.append("source:{0}".format(changes.source.dsc['Source']))
432 for binary in changes.binaries:
433 package_info.append("binary:{0}".format(binary.control['Package']))
435 (prompt, answer) = ("", "XXX")
436 if Options["No-Action"] or Options["Automatic"]:
443 print "\n".join(package_info)
446 if len(upload.reject_reasons) > 0:
448 print "\n".join(upload.reject_reasons)
451 path = os.path.join(directory, changes.filename)
452 created = os.stat(path).st_mtime
454 too_new = (now - created < int(cnf['Dinstall::SkipTime']))
457 print "SKIP (too new)"
458 prompt = "[S]kip, Quit ?"
460 prompt = "[R]eject, Skip, Quit ?"
461 if Options["Automatic"]:
464 prompt = "[N]ew, Skip, Quit ?"
465 if Options['Automatic']:
468 prompt = "[A]ccept, Skip, Quit ?"
469 if Options['Automatic']:
472 while prompt.find(answer) == -1:
473 answer = utils.our_raw_input(prompt)
474 m = re_default_answer.match(prompt)
477 answer = answer[:1].upper()
480 reject(directory, upload)
482 # upload.try_autobyhand must not be run with No-Action.
483 if Options['No-Action']:
484 accept(directory, upload)
485 elif upload.try_autobyhand():
486 accept(directory, upload)
488 print "W: redirecting to BYHAND as automatic processing failed."
489 accept_to_new(directory, upload)
491 accept_to_new(directory, upload)
497 #raise Exception("FAIL")
498 if not Options['No-Action']:
503 ###############################################################################
505 def unlink_if_exists(path):
509 if e.errno != errno.ENOENT:
512 def process_it(directory, changes, keyrings, session):
515 print "\n{0}\n".format(changes.filename)
516 Logger.log(["Processing changes file", changes.filename])
520 # Some defaults in case we can't fully process the .changes file
521 #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
522 #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
524 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
525 bcc = "X-DAK: dak process-upload"
526 #if cnf.has_key("Dinstall::Bcc"):
527 # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
529 # u.Subst["__BCC__"] = bcc
531 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
532 processed = action(directory, upload)
533 if processed and not Options['No-Action']:
534 unlink_if_exists(os.path.join(directory, changes.filename))
535 for fn in changes.files:
536 unlink_if_exists(os.path.join(directory, fn))
538 ###############################################################################
540 def process_changes(changes_filenames):
541 session = DBConn().session()
542 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
543 keyring_files = [ k.keyring_name for k in keyrings ]
546 for fn in changes_filenames:
548 directory, filename = os.path.split(fn)
549 c = daklib.upload.Changes(directory, filename, keyring_files)
550 changes.append([directory, c])
551 except Exception as e:
552 Logger.log([filename, "Error while loading changes: {0}".format(e)])
554 changes.sort(key=lambda x: x[1])
556 for directory, c in changes:
557 process_it(directory, c, keyring_files, session)
561 ###############################################################################
564 global Options, Logger
567 summarystats = SummaryStats()
569 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
570 ('h',"help","Dinstall::Options::Help"),
571 ('n',"no-action","Dinstall::Options::No-Action"),
572 ('p',"no-lock", "Dinstall::Options::No-Lock"),
573 ('s',"no-mail", "Dinstall::Options::No-Mail"),
574 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
576 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
577 "version", "directory"]:
578 if not cnf.has_key("Dinstall::Options::%s" % (i)):
579 cnf["Dinstall::Options::%s" % (i)] = ""
581 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
582 Options = cnf.subtree("Dinstall::Options")
587 # -n/--dry-run invalidates some other options which would involve things happening
588 if Options["No-Action"]:
589 Options["Automatic"] = ""
591 # Check that we aren't going to clash with the daily cron job
592 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
593 utils.fubar("Archive maintenance in progress. Try again later.")
595 # Obtain lock if not in no-action mode and initialize the log
596 if not Options["No-Action"]:
597 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
599 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
601 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
602 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
606 # Initialise UrgencyLog() - it will deal with the case where we don't
607 # want to log urgencies
608 urgencylog = UrgencyLog()
610 Logger = daklog.Logger("process-upload", Options["No-Action"])
612 # If we have a directory flag, use it to find our files
613 if cnf["Dinstall::Options::Directory"] != "":
614 # Note that we clobber the list of files we were given in this case
615 # so warn if the user has done both
616 if len(changes_files) > 0:
617 utils.warn("Directory provided so ignoring files given on command line")
619 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
620 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
621 elif not len(changes_files) > 0:
622 utils.fubar("No changes files given and no directory specified")
624 Logger.log(["Using changes files from command-line", len(changes_files)])
626 process_changes(changes_files)
628 if summarystats.accept_count:
630 if summarystats.accept_count > 1:
632 print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
633 utils.size_type(int(summarystats.accept_bytes)))
634 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
636 if summarystats.reject_count:
638 if summarystats.reject_count > 1:
640 print "Rejected %d package %s." % (summarystats.reject_count, sets)
641 Logger.log(["rejected", summarystats.reject_count])
643 if not Options["No-Action"]:
648 ###############################################################################
650 if __name__ == '__main__':