4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 # based on process-unchecked and process-accepted
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
36 ## pu: copy CHG to tempdir
37 ## pu: check CHG signature
38 ## pu: parse changes file
40 ## pu: check distribution (mappings, rejects)
41 ## pu: copy FILES to tempdir
42 ## pu: check whether CHG already exists in CopyChanges
43 ## pu: check whether FILES already exist in one of the policy queues
45 ## pu: extract control information
46 ## pu: various checks on control information
47 ## pu|pa: search for source (in CHG, projectb, policy queues)
48 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ## pu|pa: check whether deb already exists in the pool
51 ## pu: various checks on filenames and CHG consistency
52 ## pu: if isdsc: check signature
56 ## //pu: check whether file already exists in the pool
57 ## pu: store what "Component" the package is currently in
58 ## pu: check whether we found everything we were looking for in CHG
60 ## pu: check whether we need and have ONE DSC
62 ## pu: various checks //maybe drop some of the in favor of lintian
63 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ## pu: check whether DSC_FILES is consistent with "Format"
65 ## for src in DSC_FILES:
66 ## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ## pu: create new tempdir
68 ## pu: create symlink mirror of source
70 ## pu: extract changelog information for BTS
71 ## //pu: create missing .orig symlink
72 ## pu: check with lintian
74 ## pu: check checksums and sizes
75 ## for file in DSC_FILES:
76 ## pu: check checksums and sizes
77 ## pu: CHG: check urgency
79 ## pu: extract contents list and check for dubious timestamps
80 ## pu: check that the uploader is actually allowed to upload the package
82 ### if stable_install:
83 ### pa: remove from p-u
85 ### pa: move CHG to morgue
86 ### pa: append data to ChangeLog
88 ### pa: remove .dak file
90 ### pa: add dsc to db:
91 ### for file in DSC_FILES:
92 ### pa: add file to file
93 ### pa: add file to dsc_files
94 ### pa: create source entry
95 ### pa: update source associations
96 ### pa: update src_uploaders
98 ### pa: add deb to db:
99 ### pa: add file to file
100 ### pa: find source entry
101 ### pa: create binaries entry
102 ### pa: update binary associations
103 ### pa: .orig component move
104 ### pa: move files to pool
106 ### pa: move CHG to done/
107 ### pa: change entry in queue_build
108 ## pu: use dispatch table to choose target queue:
110 ## pu: write .dak file
114 ## pu: run autobyhand script
115 ## pu: if stuff left, do byhand or accept
116 ## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ## pu: write .dak file
118 ## pu: check overrides
122 ## pu: write .dak file
123 ## pu: move to ACCEPTED
125 ## pu: create files for BTS
126 ## pu: create entry in queue_build
127 ## pu: check overrides
131 ## Parsing changes (check for duplicates)
135 # New check layout (TODO: Implement)
139 ### version checks (suite)
153 ### src relation check
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
164 from errno import EACCES, EAGAIN
171 from sqlalchemy.orm.exc import NoResultFound
173 from daklib import daklog
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.config import Config
178 import daklib.utils as utils
179 from daklib.textutils import fix_maintainer
180 from daklib.regexes import *
182 import daklib.archive
185 ###############################################################################
190 ###############################################################################
192 def usage (exit_code=0):
193 print """Usage: dak process-upload [OPTION]... [CHANGES]...
194 -a, --automatic automatic run
195 -d, --directory <DIR> process uploads in <DIR>
196 -h, --help show this help and exit.
197 -n, --no-action don't do anything
198 -p, --no-lock don't check lockfile !! for cron.daily only !!
199 -s, --no-mail don't send any mail
200 -V, --version display the version number and exit"""
203 ###############################################################################
205 def try_or_reject(function):
206 """Try to call function or reject the upload if that fails
208 def wrapper(directory, upload, *args, **kwargs):
210 return function(directory, upload, *args, **kwargs)
211 except Exception as e:
213 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
215 return real_reject(directory, upload, reason=reason)
216 except Exception as e:
217 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
219 return real_reject(directory, upload, reason=reason, notify=False)
223 def subst_for_upload(upload):
226 changes = upload.changes
227 control = upload.changes.changes
229 if upload.final_suites is None or len(upload.final_suites) == 0:
230 suite_name = '(unknown)'
233 for suite in upload.final_suites:
234 if suite.policy_queue:
235 suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
237 suite_names.append(suite.suite_name)
238 suite_name = ','.join(suite_names)
240 maintainer_field = control.get('Changed-By', control.get('Maintainer', cnf['Dinstall::MyEmailAddress']))
241 maintainer = fix_maintainer(maintainer_field)
242 addresses = utils.mail_addresses_for_upload(control.get('Maintainer', cnf['Dinstall::MyEmailAddress']), maintainer_field, changes.primary_fingerprint)
244 bcc = 'X-DAK: dak process-upload'
245 if 'Dinstall::Bcc' in cnf:
246 bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
249 '__DISTRO__': cnf['Dinstall::MyDistribution'],
250 '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
252 '__CHANGES_FILENAME__': upload.changes.filename,
254 '__SOURCE__': control.get('Source', '(unknown)'),
255 '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
256 '__VERSION__': control.get('Version', '(unknown)'),
258 '__SUITE__': suite_name,
260 '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
261 '__MAINTAINER_FROM__': maintainer[1],
262 '__MAINTAINER_TO__': ", ".join(addresses),
263 '__MAINTAINER__': maintainer_field,
266 '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
268 # TODO: don't use private member
269 '__FILE_CONTENTS__': upload.changes._signed_file.contents,
274 override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
275 if override_maintainer:
276 subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
281 def accept(directory, upload):
284 Logger.log(['ACCEPT', upload.changes.filename])
288 accepted_to_real_suite = False
289 for suite in upload.final_suites:
290 accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
292 control = upload.changes.changes
293 if 'source' in upload.changes.architectures and not Options['No-Action']:
294 urgency = control.get('Urgency')
295 if urgency not in cnf.value_list('Urgency::Valid'):
296 urgency = cnf['Urgency::Default']
297 UrgencyLog().log(control['Source'], control['Version'], urgency)
299 # send mail to maintainer
300 subst = subst_for_upload(upload)
301 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
302 utils.send_mail(message)
304 # send mail to announce lists and tracking server
305 subst = subst_for_upload(upload)
307 for suite in upload.final_suites:
308 if suite.policy_queue is None:
310 announce.update(suite.announce or [])
311 announce_address = ", ".join(announce)
312 tracking = cnf.get('Dinstall::TrackingServer')
313 if tracking and 'source' in upload.changes.architectures:
314 announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
315 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
316 utils.send_mail(message)
318 # Only close bugs for uploads that were not redirected to a policy queue.
319 # process-policy will close bugs for those once they are accepted.
320 subst = subst_for_upload(upload)
321 if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and upload.changes.source is not None:
322 for bugnum in upload.changes.closed_bugs:
323 subst['__BUG_NUMBER__'] = str(bugnum)
325 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
326 utils.send_mail(message)
328 del subst['__BUG_NUMBER__']
330 # Move .changes to done, but only for uploads that were accepted to a
331 # real suite. process-policy will handle this for uploads to queues.
332 if accepted_to_real_suite:
333 src = os.path.join(upload.directory, upload.changes.filename)
335 now = datetime.datetime.now()
336 donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
337 dst = os.path.join(donedir, upload.changes.filename)
338 dst = utils.find_next_free(dst)
340 upload.transaction.fs.copy(src, dst, mode=0o644)
342 SummaryStats().accept_count += 1
343 SummaryStats().accept_bytes += upload.changes.bytes
346 def accept_to_new(directory, upload):
349 Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
351 upload.install_to_new()
352 # TODO: tag bugs pending, send announcement
354 subst = subst_for_upload(upload)
355 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
356 utils.send_mail(message)
358 SummaryStats().accept_count += 1
359 SummaryStats().accept_bytes += upload.changes.bytes
362 def reject(directory, upload, reason=None, notify=True):
363 real_reject(directory, upload, reason, notify)
365 def real_reject(directory, upload, reason=None, notify=True):
366 # XXX: rejection itself should go to daklib.archive.ArchiveUpload
369 Logger.log(['REJECT', upload.changes.filename])
371 fs = upload.transaction.fs
372 rejectdir = cnf['Dir::Reject']
374 files = [ f.filename for f in upload.changes.files.itervalues() ]
375 files.append(upload.changes.filename)
378 src = os.path.join(upload.directory, fn)
379 dst = utils.find_next_free(os.path.join(rejectdir, fn))
380 if not os.path.exists(src):
384 if upload.reject_reasons is not None:
387 reason = reason + '\n' + '\n'.join(upload.reject_reasons)
390 reason = '(Unknown reason. Please check logs.)'
392 dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
399 subst = subst_for_upload(upload)
400 subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
401 subst['__MANUAL_REJECT_MESSAGE__'] = ''
402 subst['__REJECT_MESSAGE__'] = reason
403 subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
405 message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
406 utils.send_mail(message)
408 SummaryStats().reject_count += 1
410 ###############################################################################
412 def action(directory, upload):
413 changes = upload.changes
420 okay = upload.check()
422 summary = changes.changes.get('Changes', '')
426 if changes.source is not None:
427 package_info.append("source:{0}".format(changes.source.dsc['Source']))
428 for binary in changes.binaries:
429 package_info.append("binary:{0}".format(binary.control['Package']))
431 (prompt, answer) = ("", "XXX")
432 if Options["No-Action"] or Options["Automatic"]:
439 print "\n".join(package_info)
442 if len(upload.reject_reasons) > 0:
444 print "\n".join(upload.reject_reasons)
447 path = os.path.join(directory, changes.filename)
448 created = os.stat(path).st_mtime
450 too_new = (now - created < int(cnf['Dinstall::SkipTime']))
453 print "SKIP (too new)"
454 prompt = "[S]kip, Quit ?"
456 prompt = "[R]eject, Skip, Quit ?"
457 if Options["Automatic"]:
460 prompt = "[N]ew, Skip, Quit ?"
461 if Options['Automatic']:
464 prompt = "[A]ccept, Skip, Quit ?"
465 if Options['Automatic']:
468 while prompt.find(answer) == -1:
469 answer = utils.our_raw_input(prompt)
470 m = re_default_answer.match(prompt)
473 answer = answer[:1].upper()
476 reject(directory, upload)
478 # upload.try_autobyhand must not be run with No-Action.
479 if Options['No-Action']:
480 accept(directory, upload)
481 elif upload.try_autobyhand():
482 accept(directory, upload)
484 print "W: redirecting to BYHAND as automatic processing failed."
485 accept_to_new(directory, upload)
487 accept_to_new(directory, upload)
493 #raise Exception("FAIL")
494 if not Options['No-Action']:
499 ###############################################################################
501 def unlink_if_exists(path):
505 if e.errno != errno.ENOENT:
508 def process_it(directory, changes, keyrings, session):
511 print "\n{0}\n".format(changes.filename)
512 Logger.log(["Processing changes file", changes.filename])
516 # Some defaults in case we can't fully process the .changes file
517 #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
518 #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
520 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
521 bcc = "X-DAK: dak process-upload"
522 #if cnf.has_key("Dinstall::Bcc"):
523 # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
525 # u.Subst["__BCC__"] = bcc
527 with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
528 processed = action(directory, upload)
529 if processed and not Options['No-Action']:
530 unlink_if_exists(os.path.join(directory, changes.filename))
531 for fn in changes.files:
532 unlink_if_exists(os.path.join(directory, fn))
534 ###############################################################################
536 def process_changes(changes_filenames):
537 session = DBConn().session()
538 keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
539 keyring_files = [ k.keyring_name for k in keyrings ]
542 for fn in changes_filenames:
544 directory, filename = os.path.split(fn)
545 c = daklib.upload.Changes(directory, filename, keyring_files)
546 changes.append([directory, c])
547 except Exception as e:
548 Logger.log([filename, "Error while loading changes: {0}".format(e)])
550 changes.sort(key=lambda x: x[1])
552 for directory, c in changes:
553 process_it(directory, c, keyring_files, session)
557 ###############################################################################
560 global Options, Logger
563 summarystats = SummaryStats()
565 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
566 ('h',"help","Dinstall::Options::Help"),
567 ('n',"no-action","Dinstall::Options::No-Action"),
568 ('p',"no-lock", "Dinstall::Options::No-Lock"),
569 ('s',"no-mail", "Dinstall::Options::No-Mail"),
570 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
572 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
573 "version", "directory"]:
574 if not cnf.has_key("Dinstall::Options::%s" % (i)):
575 cnf["Dinstall::Options::%s" % (i)] = ""
577 changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
578 Options = cnf.subtree("Dinstall::Options")
583 # -n/--dry-run invalidates some other options which would involve things happening
584 if Options["No-Action"]:
585 Options["Automatic"] = ""
587 # Check that we aren't going to clash with the daily cron job
588 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
589 utils.fubar("Archive maintenance in progress. Try again later.")
591 # Obtain lock if not in no-action mode and initialize the log
592 if not Options["No-Action"]:
593 lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
595 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
597 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
598 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
602 # Initialise UrgencyLog() - it will deal with the case where we don't
603 # want to log urgencies
604 urgencylog = UrgencyLog()
606 Logger = daklog.Logger("process-upload", Options["No-Action"])
608 # If we have a directory flag, use it to find our files
609 if cnf["Dinstall::Options::Directory"] != "":
610 # Note that we clobber the list of files we were given in this case
611 # so warn if the user has done both
612 if len(changes_files) > 0:
613 utils.warn("Directory provided so ignoring files given on command line")
615 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
616 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
617 elif not len(changes_files) > 0:
618 utils.fubar("No changes files given and no directory specified")
620 Logger.log(["Using changes files from command-line", len(changes_files)])
622 process_changes(changes_files)
624 if summarystats.accept_count:
626 if summarystats.accept_count > 1:
628 print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
629 utils.size_type(int(summarystats.accept_bytes)))
630 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
632 if summarystats.reject_count:
634 if summarystats.reject_count > 1:
636 print "Rejected %d package %s." % (summarystats.reject_count, sets)
637 Logger.log(["rejected", summarystats.reject_count])
639 if not Options["No-Action"]:
644 ###############################################################################
646 if __name__ == '__main__':