4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 # based on process-unchecked and process-accepted
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
36 ## pu: copy CHG to tempdir
37 ## pu: check CHG signature
38 ## pu: parse changes file
40 ## pu: check distribution (mappings, rejects)
41 ## pu: copy FILES to tempdir
42 ## pu: check whether CHG already exists in CopyChanges
43 ## pu: check whether FILES already exist in one of the policy queues
45 ## pu: extract control information
46 ## pu: various checks on control information
47 ## pu|pa: search for source (in CHG, projectb, policy queues)
48 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ## pu|pa: check whether deb already exists in the pool
51 ## pu: various checks on filenames and CHG consistency
52 ## pu: if isdsc: check signature
56 ## //pu: check whether file already exists in the pool
57 ## pu: store what "Component" the package is currently in
58 ## pu: check whether we found everything we were looking for in CHG
60 ## pu: check whether we need and have ONE DSC
62 ## pu: various checks //maybe drop some of the in favor of lintian
63 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ## pu: check whether DSC_FILES is consistent with "Format"
65 ## for src in DSC_FILES:
66 ## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ## pu: create new tempdir
68 ## pu: create symlink mirror of source
70 ## pu: extract changelog information for BTS
71 ## //pu: create missing .orig symlink
72 ## pu: check with lintian
74 ## pu: check checksums and sizes
75 ## for file in DSC_FILES:
76 ## pu: check checksums and sizes
77 ## pu: CHG: check urgency
79 ## pu: extract contents list and check for dubious timestamps
80 ## pu: check that the uploader is actually allowed to upload the package
82 ### if stable_install:
83 ### pa: remove from p-u
85 ### pa: move CHG to morgue
86 ### pa: append data to ChangeLog
88 ### pa: remove .dak file
90 ### pa: add dsc to db:
91 ### for file in DSC_FILES:
92 ### pa: add file to file
93 ### pa: add file to dsc_files
94 ### pa: create source entry
95 ### pa: update source associations
96 ### pa: update src_uploaders
98 ### pa: add deb to db:
99 ### pa: add file to file
100 ### pa: find source entry
101 ### pa: create binaries entry
102 ### pa: update binary associations
103 ### pa: .orig component move
104 ### pa: move files to pool
106 ### pa: move CHG to done/
107 ### pa: change entry in queue_build
108 ## pu: use dispatch table to choose target queue:
110 ## pu: write .dak file
114 ## pu: run autobyhand script
115 ## pu: if stuff left, do byhand or accept
116 ## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ## pu: write .dak file
118 ## pu: check overrides
122 ## pu: write .dak file
123 ## pu: move to ACCEPTED
125 ## pu: create files for BTS
126 ## pu: create entry in queue_build
127 ## pu: check overrides
131 ## Parsing changes (check for duplicates)
135 # New check layout (TODO: Implement)
139 ### version checks (suite)
153 ### src relation check
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
164 from errno import EACCES, EAGAIN
171 from sqlalchemy.orm.exc import NoResultFound
173 from daklib import daklog
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.config import Config
178 import daklib.utils as utils
179 from daklib.textutils import fix_maintainer
180 from daklib.regexes import *
182 import daklib.archive
185 ###############################################################################
190 ###############################################################################
192 def usage (exit_code=0):
193 print """Usage: dak process-upload [OPTION]... [CHANGES]...
194 -a, --automatic automatic run
195 -d, --directory <DIR> process uploads in <DIR>
196 -h, --help show this help and exit.
197 -n, --no-action don't do anything
198 -p, --no-lock don't check lockfile !! for cron.daily only !!
199 -s, --no-mail don't send any mail
200 -V, --version display the version number and exit"""
203 ###############################################################################
205 def try_or_reject(function):
206 """Try to call function or reject the upload if that fails
208 def wrapper(directory, upload, *args, **kwargs):
210 return function(directory, upload, *args, **kwargs)
211 except Exception as e:
213 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
215 return real_reject(directory, upload, reason=reason)
216 except Exception as e:
217 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
219 return real_reject(directory, upload, reason=reason, notify=False)
223 def subst_for_upload(upload):
226 changes = upload.changes
227 control = upload.changes.changes
229 if upload.final_suites is None or len(upload.final_suites) == 0:
230 suite_name = '(unknown)'
233 for suite in upload.final_suites:
234 if suite.policy_queue:
235 suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
237 suite_names.append(suite.suite_name)
238 suite_name = ','.join(suite_names)
240 maintainer_field = control.get('Maintainer', cnf['Dinstall::MyEmailAddress'])
241 changed_by_field = control.get('Changed-By', maintainer_field)
242 maintainer = fix_maintainer(changed_by_field)
243 if upload.changes.source is not None:
244 addresses = utils.mail_addresses_for_upload(maintainer_field, changed_by_field, changes.primary_fingerprint)
246 addresses = utils.mail_addresses_for_upload(maintainer_field, maintainer_field, changes.primary_fingerprint)
248 bcc = 'X-DAK: dak process-upload'
249 if 'Dinstall::Bcc' in cnf:
250 bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
253 '__DISTRO__': cnf['Dinstall::MyDistribution'],
254 '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
256 '__CHANGES_FILENAME__': upload.changes.filename,
258 '__SOURCE__': control.get('Source', '(unknown)'),
259 '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
260 '__VERSION__': control.get('Version', '(unknown)'),
262 '__SUITE__': suite_name,
264 '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
265 '__MAINTAINER_FROM__': maintainer[1],
266 '__MAINTAINER_TO__': ", ".join(addresses),
267 '__MAINTAINER__': changed_by_field,
270 '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
272 '__FILE_CONTENTS__': open(upload.changes.path, 'r').read(),
275 override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
276 if override_maintainer:
277 subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
282 def accept(directory, upload):
285 Logger.log(['ACCEPT', upload.changes.filename])
289 accepted_to_real_suite = False
290 for suite in upload.final_suites:
291 accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
293 control = upload.changes.changes
294 if 'source' in upload.changes.architectures and not Options['No-Action']:
295 urgency = control.get('Urgency')
296 if urgency not in cnf.value_list('Urgency::Valid'):
297 urgency = cnf['Urgency::Default']
298 UrgencyLog().log(control['Source'], control['Version'], urgency)
300 # send mail to maintainer
301 subst = subst_for_upload(upload)
302 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
303 utils.send_mail(message)
305 # send mail to announce lists and tracking server
306 if accepted_to_real_suite:
307 subst = subst_for_upload(upload)
309 for suite in upload.final_suites:
310 if suite.policy_queue is not None:
312 announce.update(suite.announce or [])
313 announce_address = ", ".join(announce)
315 tracking = cnf.get('Dinstall::TrackingServer')
316 if tracking and 'source' in upload.changes.architectures:
317 announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
319 subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address
321 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
322 utils.send_mail(message)
324 # Only close bugs for uploads that were not redirected to a policy queue.
325 # process-policy will close bugs for those once they are accepted.
326 subst = subst_for_upload(upload)
327 if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and upload.changes.source is not None:
328 for bugnum in upload.changes.closed_bugs:
329 subst['__BUG_NUMBER__'] = str(bugnum)
331 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
332 utils.send_mail(message)
334 del subst['__BUG_NUMBER__']
336 # Move .changes to done, but only for uploads that were accepted to a
337 # real suite. process-policy will handle this for uploads to queues.
338 if accepted_to_real_suite:
339 src = os.path.join(upload.directory, upload.changes.filename)
341 now = datetime.datetime.now()
342 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
343 dst = os.path.join(donedir, upload.changes.filename)
344 dst = utils.find_next_free(dst)
346 upload.transaction.fs.copy(src, dst, mode=0o644)
348 SummaryStats().accept_count += 1
349 SummaryStats().accept_bytes += upload.changes.bytes
352 def accept_to_new(directory, upload):
355 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
357 upload.install_to_new()
358 # TODO: tag bugs pending, send announcement
360 subst = subst_for_upload(upload)
361 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
362 utils.send_mail(message)
364 SummaryStats().accept_count += 1
365 SummaryStats().accept_bytes += upload.changes.bytes
368 def reject(directory, upload, reason=None, notify=True):
369 real_reject(directory, upload, reason, notify)
371 def real_reject(directory, upload, reason=None, notify=True):
372 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
375 Logger.log(['REJECT', upload.changes.filename])
377 fs = upload.transaction.fs
378 rejectdir = cnf['Dir::Reject']
380 files = [ f.filename for f in upload.changes.files.itervalues() ]
381 files.append(upload.changes.filename)
384 src = os.path.join(upload.directory, fn)
385 dst = utils.find_next_free(os.path.join(rejectdir, fn))
386 if not os.path.exists(src):
390 if upload.reject_reasons is not None:
393 reason = reason + '\n' + '\n'.join(upload.reject_reasons)
396 reason = '(Unknown reason. Please check logs.)'
398 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
405 subst = subst_for_upload(upload)
406 subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
407 subst['__MANUAL_REJECT_MESSAGE__'] = ''
408 subst['__REJECT_MESSAGE__'] = reason
409 subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
411 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
412 utils.send_mail(message)
414 SummaryStats().reject_count += 1
416 ###############################################################################
418 def action(directory, upload):
419 changes = upload.changes
426 okay = upload.check()
428 summary = changes.changes.get('Changes', '')
432 if changes.source is not None:
433 package_info.append("source:{0}".format(changes.source.dsc['Source']))
434 for binary in changes.binaries:
435 package_info.append("binary:{0}".format(binary.control['Package']))
437 (prompt, answer) = ("", "XXX")
438 if Options["No-Action"] or Options["Automatic"]:
445 print "\n".join(package_info)
448 if len(upload.reject_reasons) > 0:
450 print "\n".join(upload.reject_reasons)
453 path = os.path.join(directory, changes.filename)
454 created = os.stat(path).st_mtime
456 too_new = (now - created < int(cnf['Dinstall::SkipTime']))
459 print "SKIP (too new)"
460 prompt = "[S]kip, Quit ?"
462 prompt = "[R]eject, Skip, Quit ?"
463 if Options["Automatic"]:
466 prompt = "[N]ew, Skip, Quit ?"
467 if Options['Automatic']:
470 prompt = "[A]ccept, Skip, Quit ?"
471 if Options['Automatic']:
474 while prompt.find(answer) == -1:
475 answer = utils.our_raw_input(prompt)
476 m = re_default_answer.match(prompt)
479 answer = answer[:1].upper()
482 reject(directory, upload)
484 # upload.try_autobyhand must not be run with No-Action.
485 if Options['No-Action']:
486 accept(directory, upload)
487 elif upload.try_autobyhand():
488 accept(directory, upload)
490 print "W: redirecting to BYHAND as automatic processing failed."
491 accept_to_new(directory, upload)
493 accept_to_new(directory, upload)
499 #raise Exception("FAIL")
500 if not Options['No-Action']:
505 ###############################################################################
507 def unlink_if_exists(path):
511 if e.errno != errno.ENOENT:
514 def process_it(directory, changes, keyrings, session):
517 print "\n{0}\n".format(changes.filename)
518 Logger.log(["Processing changes file", changes.filename])
522 # Some defaults in case we can't fully process the .changes file
523 #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
524 #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
526 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
527 bcc = "X-DAK: dak process-upload"
528 #if cnf.has_key("Dinstall::Bcc"):
529 # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
531 # u.Subst["__BCC__"] = bcc
533 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
534 processed = action(directory, upload)
535 if processed and not Options['No-Action']:
536 unlink_if_exists(os.path.join(directory, changes.filename))
537 for fn in changes.files:
538 unlink_if_exists(os.path.join(directory, fn))
540 ###############################################################################
542 def process_changes(changes_filenames):
543 session = DBConn().session()
544 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
545 keyring_files = [ k.keyring_name for k in keyrings ]
548 for fn in changes_filenames:
550 directory, filename = os.path.split(fn)
551 c = daklib.upload.Changes(directory, filename, keyring_files)
552 changes.append([directory, c])
553 except Exception as e:
554 Logger.log([filename, "Error while loading changes: {0}".format(e)])
556 changes.sort(key=lambda x: x[1])
558 for directory, c in changes:
559 process_it(directory, c, keyring_files, session)
563 ###############################################################################
566 global Options, Logger
569 summarystats = SummaryStats()
571 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
572 ('h',"help","Dinstall::Options::Help"),
573 ('n',"no-action","Dinstall::Options::No-Action"),
574 ('p',"no-lock", "Dinstall::Options::No-Lock"),
575 ('s',"no-mail", "Dinstall::Options::No-Mail"),
576 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
578 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
579 "version", "directory"]:
580 if not cnf.has_key("Dinstall::Options::%s" % (i)):
581 cnf["Dinstall::Options::%s" % (i)] = ""
583 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
584 Options = cnf.subtree("Dinstall::Options")
589 # -n/--dry-run invalidates some other options which would involve things happening
590 if Options["No-Action"]:
591 Options["Automatic"] = ""
593 # Check that we aren't going to clash with the daily cron job
594 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
595 utils.fubar("Archive maintenance in progress. Try again later.")
597 # Obtain lock if not in no-action mode and initialize the log
598 if not Options["No-Action"]:
599 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
601 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
603 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
604 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
608 # Initialise UrgencyLog() - it will deal with the case where we don't
609 # want to log urgencies
610 urgencylog = UrgencyLog()
612 Logger = daklog.Logger("process-upload", Options["No-Action"])
614 # If we have a directory flag, use it to find our files
615 if cnf["Dinstall::Options::Directory"] != "":
616 # Note that we clobber the list of files we were given in this case
617 # so warn if the user has done both
618 if len(changes_files) > 0:
619 utils.warn("Directory provided so ignoring files given on command line")
621 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
622 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
623 elif not len(changes_files) > 0:
624 utils.fubar("No changes files given and no directory specified")
626 Logger.log(["Using changes files from command-line", len(changes_files)])
628 process_changes(changes_files)
630 if summarystats.accept_count:
632 if summarystats.accept_count > 1:
634 print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
635 utils.size_type(int(summarystats.accept_bytes)))
636 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
638 if summarystats.reject_count:
640 if summarystats.reject_count > 1:
642 print "Rejected %d package %s." % (summarystats.reject_count, sets)
643 Logger.log(["rejected", summarystats.reject_count])
645 if not Options["No-Action"]:
650 ###############################################################################
652 if __name__ == '__main__':