4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 # Originally based on dinstall by Guy Maor <maor@debian.org>
28 ################################################################################
30 # Computer games don't affect kids. I mean if Pacman affected our generation as
31 # kids, we'd all run around in a darkened room munching pills and listening to
35 ################################################################################
44 from daklib.dbconn import *
45 from daklib import daklog
46 from daklib.queue import *
47 from daklib import utils
48 from daklib.textutils import fix_maintainer
49 from daklib.dak_exceptions import *
50 from daklib.regexes import re_default_answer
51 from daklib.summarystats import SummaryStats
52 from daklib.holding import Holding
53 from daklib.config import Config
57 ################################################################################
60 ################################################################################
66 ###############################################################################
74 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
75 ('h',"help","Dinstall::Options::Help"),
76 ('n',"no-action","Dinstall::Options::No-Action"),
77 ('p',"no-lock", "Dinstall::Options::No-Lock"),
78 ('s',"no-mail", "Dinstall::Options::No-Mail"),
79 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
81 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
82 "override-distribution", "version", "directory"]:
83 cnf["Dinstall::Options::%s" % (i)] = ""
85 changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
86 Options = cnf.SubTree("Dinstall::Options")
91 # If we have a directory flag, use it to find our files
92 if cnf["Dinstall::Options::Directory"] != "":
93 # Note that we clobber the list of files we were given in this case
94 # so warn if the user has done both
95 if len(changes_files) > 0:
96 utils.warn("Directory provided so ignoring files given on command line")
98 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
102 ################################################################################
104 def usage (exit_code=0):
105 print """Usage: dak process-unchecked [OPTION]... [CHANGES]...
106 -a, --automatic automatic run
107 -h, --help show this help and exit.
108 -n, --no-action don't do anything
109 -p, --no-lock don't check lockfile !! for cron.daily only !!
110 -s, --no-mail don't send any mail
111 -V, --version display the version number and exit"""
114 ################################################################################
119 # changes["distribution"] may not exist in corner cases
120 # (e.g. unreadable changes files)
121 if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType):
122 u.pkg.changes["distribution"] = {}
124 (summary, short_summary) = u.build_summaries()
126 # q-unapproved hax0ring
128 "New": { "is": is_new, "process": acknowledge_new },
129 "Autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand },
130 "Byhand" : { "is": is_byhand, "process": do_byhand },
131 "OldStableUpdate" : { "is": is_oldstableupdate,
132 "process": do_oldstableupdate },
133 "StableUpdate" : { "is": is_stableupdate, "process": do_stableupdate },
134 "Unembargo" : { "is": is_unembargo, "process": queue_unembargo },
135 "Embargo" : { "is": is_embargo, "process": queue_embargo },
138 queues = [ "New", "Autobyhand", "Byhand" ]
139 if cnf.FindB("Dinstall::SecurityQueueHandling"):
140 queues += [ "Unembargo", "Embargo" ]
142 queues += [ "OldStableUpdate", "StableUpdate" ]
144 (prompt, answer) = ("", "XXX")
145 if Options["No-Action"] or Options["Automatic"]:
150 pi = u.package_info()
152 if len(u.rejects) > 0:
153 if u.upload_too_new():
154 print "SKIP (too new)\n" + pi,
155 prompt = "[S]kip, Quit ?"
157 print "REJECT\n" + pi
158 prompt = "[R]eject, Skip, Quit ?"
159 if Options["Automatic"]:
164 if queue_info[q]["is"](u):
168 print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary)
169 queuekey = qu[0].upper()
170 if queuekey in "RQSA":
172 prompt = "[D]ivert, Skip, Quit ?"
174 prompt = "[%s]%s, Skip, Quit ?" % (queuekey, qu[1:].lower())
175 if Options["Automatic"]:
178 print "ACCEPT\n" + pi + summary,
179 prompt = "[A]ccept, Skip, Quit ?"
180 if Options["Automatic"]:
183 while prompt.find(answer) == -1:
184 answer = utils.our_raw_input(prompt)
185 m = re_default_answer.match(prompt)
188 answer = answer[:1].upper()
191 os.chdir(u.pkg.directory)
194 u.pkg.add_known_changes( "Accepted" )
195 u.accept(summary, short_summary)
198 elif answer == queuekey:
199 u.pkg.add_known_changes( qu )
200 queue_info[qu]["process"](u, summary, short_summary)
205 ################################################################################
207 def package_to_suite(u, suite):
208 if not u.pkg.changes["distribution"].has_key(suite):
213 if not u.pkg.changes["architecture"].has_key("source"):
214 s = DBConn().session()
215 q = s.query(SrcAssociation.sa_id)
216 q = q.join(Suite).filter_by(suite_name=suite)
217 q = q.join(DBSource).filter_by(source=u.pkg.changes['source'])
218 q = q.filter_by(version=u.pkg.changes['version']).limit(1)
220 # NB: Careful, this logic isn't what you would think it is
221 # Source is already in {old-,}proposed-updates so no need to hold
222 # Instead, we don't move to the holding area, we just do an ACCEPT
230 def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, announce=None):
232 dir = cnf["Dir::Queue::%s" % queue]
234 print "Moving to %s holding area" % queue.upper()
235 Logger.log(["Moving to %s" % queue, u.pkg.changes_file])
237 u.pkg.write_dot_dak(dir)
238 u.move_to_dir(dir, perms=perms)
240 get_or_set_queue(queue.lower()).autobuild_upload(u.pkg, dir)
242 # Check for override disparities
245 # Send accept mail, announce to lists and close bugs
246 if announce and not cnf["Dinstall::Options::No-Mail"]:
247 template = os.path.join(cnf["Dir::Templates"], announce)
249 u.Subst["__SUITE__"] = ""
250 mail_message = utils.TemplateSubst(u.Subst, template)
251 utils.send_mail(mail_message)
252 u.announce(short_summary, True)
254 ################################################################################
257 session = DBConn().session()
260 q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes)
266 os.chdir(cnf["Dir::Queue::Disembargo"])
272 if u.pkg.directory == disdir:
273 if u.pkg.changes["architecture"].has_key("source"):
274 if not Options["No-Action"]:
275 session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes)
284 def queue_unembargo(u, summary, short_summary):
285 return package_to_queue(u, summary, short_summary, "Unembargoed",
286 perms=0660, build=True, announce='process-unchecked.accepted')
288 ################################################################################
291 # if embargoed queues are enabled always embargo
294 def queue_embargo(u, summary, short_summary):
295 return package_to_queue(u, summary, short_summary, "Unembargoed",
296 perms=0660, build=True, announce='process-unchecked.accepted')
298 ################################################################################
300 def is_stableupdate(u):
301 return package_to_suite(u, 'proposed-updates')
303 def do_stableupdate(u, summary, short_summary):
304 return package_to_queue(u, summary, short_summary, "ProposedUpdates",
305 perms=0664, build=False, announce=None)
307 ################################################################################
309 def is_oldstableupdate(u):
310 return package_to_suite(u, 'oldstable-proposed-updates')
312 def do_oldstableupdate(u, summary, short_summary):
313 return package_to_queue(u, summary, short_summary, "OldProposedUpdates",
314 perms=0664, build=False, announce=None)
316 ################################################################################
318 def is_autobyhand(u):
323 for f in u.pkg.files.keys():
324 if u.pkg.files[f].has_key("byhand"):
327 # filename is of form "PKG_VER_ARCH.EXT" where PKG, VER and ARCH
328 # don't contain underscores, and ARCH doesn't contain dots.
329 # further VER matches the .changes Version:, and ARCH should be in
330 # the .changes Architecture: list.
335 (pckg, ver, archext) = f.split("_", 2)
336 if archext.count(".") < 1 or u.pkg.changes["version"] != ver:
340 ABH = cnf.SubTree("AutomaticByHandPackages")
341 if not ABH.has_key(pckg) or \
342 ABH["%s::Source" % (pckg)] != u.pkg.changes["source"]:
343 print "not match %s %s" % (pckg, u.pkg.changes["source"])
347 (arch, ext) = archext.split(".", 1)
348 if arch not in u.pkg.changes["architecture"]:
352 u.pkg.files[f]["byhand-arch"] = arch
353 u.pkg.files[f]["byhand-script"] = ABH["%s::Script" % (pckg)]
355 return any_auto and all_auto
357 def do_autobyhand(u, summary, short_summary):
358 print "Attempting AUTOBYHAND."
360 for f, entry in u.pkg.files.items():
363 if not entry.has_key("byhand"):
366 if not entry.has_key("byhand-script"):
370 os.system("ls -l %s" % byhandfile)
372 result = os.system("%s %s %s %s %s" % (
373 entry["byhand-script"],
375 u.pkg.changes["version"],
376 entry["byhand-arch"],
377 os.path.abspath(u.pkg.changes_file)))
380 os.unlink(byhandfile)
383 print "Error processing %s, left as byhand." % (f)
387 do_byhand(u, summary, short_summary)
389 u.accept(summary, short_summary)
391 # XXX: We seem to be missing a u.remove() here
392 # This might explain why we get byhand leftovers in unchecked - mhy
394 ################################################################################
397 for f in u.pkg.files.keys():
398 if u.pkg.files[f].has_key("byhand"):
402 def do_byhand(u, summary, short_summary):
403 return package_to_queue(u, summary, short_summary, "Byhand",
404 perms=0660, build=False, announce=None)
406 ################################################################################
409 for f in u.pkg.files.keys():
410 if u.pkg.files[f].has_key("new"):
414 def acknowledge_new(u, summary, short_summary):
417 print "Moving to NEW holding area."
418 Logger.log(["Moving to new", u.pkg.changes_file])
420 u.pkg.write_dot_dak(cnf["Dir::Queue::New"])
421 u.move_to_dir(cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
423 if not Options["No-Mail"]:
424 print "Sending new ack."
425 template = os.path.join(cnf["Dir::Templates"], 'process-unchecked.new')
427 u.Subst["__SUMMARY__"] = summary
428 new_ack_message = utils.TemplateSubst(u.Subst, template)
429 utils.send_mail(new_ack_message)
431 ################################################################################
433 # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in
434 # Incoming. -1 will reference the .orig.tar.gz, but -2 will not.
435 # Upload.check_dsc_against_db() can find the .orig.tar.gz but it will
436 # not have processed it during it's checks of -2. If -1 has been
437 # deleted or otherwise not checked by 'dak process-unchecked', the
438 # .orig.tar.gz will not have been checked at all. To get round this,
439 # we force the .orig.tar.gz into the .changes structure and reprocess
442 def process_it(changes_file):
450 u.pkg.changes_file = changes_file
451 u.pkg.directory = os.getcwd()
453 origchanges = os.path.join(u.pkg.directory, u.pkg.changes_file)
455 # Some defaults in case we can't fully process the .changes file
456 u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
457 u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
459 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
460 bcc = "X-DAK: dak process-unchecked"
461 if cnf.has_key("Dinstall::Bcc"):
462 u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
464 u.Subst["__BCC__"] = bcc
466 # Remember where we are so we can come back after cd-ing into the
467 # holding directory. TODO: Fix this stupid hack
468 u.prevdir = os.getcwd()
470 # TODO: Figure out something better for this (or whether it's even
471 # necessary - it seems to have been for use when we were
472 # still doing the is_unchecked check; reprocess = 2)
476 # If this is the Real Thing(tm), copy things into a private
477 # holding directory first to avoid replacable file races.
478 if not Options["No-Action"]:
479 os.chdir(cnf["Dir::Queue::Holding"])
481 # Absolutize the filename to avoid the requirement of being in the
482 # same directory as the .changes file.
483 holding.copy_to_holding(origchanges)
485 # Relativize the filename so we use the copy in holding
486 # rather than the original...
487 changespath = os.path.basename(u.pkg.changes_file)
489 (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changespath)
491 if u.pkg.changes["fingerprint"]:
492 valid_changes_p = u.load_changes(changespath)
494 valid_changes_p = False
495 u.rejects.extend(rejects)
499 u.check_distributions()
500 u.check_files(not Options["No-Action"])
501 valid_dsc_p = u.check_dsc(not Options["No-Action"])
502 if valid_dsc_p and not Options["No-Action"]:
508 u.check_signed_by_key()
512 except (SystemExit, KeyboardInterrupt):
517 traceback.print_exc(file=sys.stderr)
519 # Restore previous WD
522 ###############################################################################
525 global Options, Logger
528 changes_files = init()
530 # -n/--dry-run invalidates some other options which would involve things happening
531 if Options["No-Action"]:
532 Options["Automatic"] = ""
534 # Initialize our Holding singleton
537 # Ensure all the arguments we were given are .changes files
538 for f in changes_files:
539 if not f.endswith(".changes"):
540 utils.warn("Ignoring '%s' because it's not a .changes file." % (f))
541 changes_files.remove(f)
543 if changes_files == []:
544 if cnf["Dinstall::Options::Directory"] == "":
545 utils.fubar("Need at least one .changes file as an argument.")
549 # Check that we aren't going to clash with the daily cron job
550 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
551 utils.fubar("Archive maintenance in progress. Try again later.")
553 # Obtain lock if not in no-action mode and initialize the log
554 if not Options["No-Action"]:
555 lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
557 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
559 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
560 utils.fubar("Couldn't obtain lock; assuming another 'dak process-unchecked' is already running.")
563 Logger = daklog.Logger(cnf, "process-unchecked")
565 # Sort the .changes files so that we process sourceful ones first
566 changes_files.sort(utils.changes_compare)
568 # Process the changes files
569 for changes_file in changes_files:
570 print "\n" + changes_file
572 process_it (changes_file)
574 if not Options["No-Action"]:
577 accept_count = SummaryStats().accept_count
578 accept_bytes = SummaryStats().accept_bytes
584 print "Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes)))
585 Logger.log(["total",accept_count,accept_bytes])
587 if not Options["No-Action"]:
590 ################################################################################
592 if __name__ == '__main__':