4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009 Mark Hymers <mhy@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 # based on process-unchecked and process-accepted
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
36 ## pu: copy CHG to tempdir
37 ## pu: check CHG signature
38 ## pu: parse changes file
40 ## pu: check distribution (mappings, rejects)
41 ## pu: copy FILES to tempdir
42 ## pu: check whether CHG already exists in CopyChanges
43 ## pu: check whether FILES already exist in one of the policy queues
45 ## pu: extract control information
46 ## pu: various checks on control information
47 ## pu|pa: search for source (in CHG, projectb, policy queues)
48 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ## pu|pa: check whether deb already exists in the pool
51 ## pu: various checks on filenames and CHG consistency
52 ## pu: if isdsc: check signature
56 ## //pu: check whether file already exists in the pool
57 ## pu: store what "Component" the package is currently in
58 ## pu: check whether we found everything we were looking for in CHG
60 ## pu: check whether we need and have ONE DSC
62 ## pu: various checks //maybe drop some of the in favor of lintian
63 ## pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ## pu: check whether DSC_FILES is consistent with "Format"
65 ## for src in DSC_FILES:
66 ## pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ## pu: create new tempdir
68 ## pu: create symlink mirror of source
70 ## pu: extract changelog information for BTS
71 ## //pu: create missing .orig symlink
72 ## pu: check with lintian
74 ## pu: check checksums and sizes
75 ## for file in DSC_FILES:
76 ## pu: check checksums and sizes
77 ## pu: CHG: check urgency
79 ## pu: extract contents list and check for dubious timestamps
80 ## pu: check that the uploader is actually allowed to upload the package
82 ### if stable_install:
83 ### pa: remove from p-u
85 ### pa: move CHG to morgue
86 ### pa: append data to ChangeLog
88 ### pa: remove .dak file
90 ### pa: add dsc to db:
91 ### for file in DSC_FILES:
92 ### pa: add file to file
93 ### pa: add file to dsc_files
94 ### pa: create source entry
95 ### pa: update source associations
96 ### pa: update src_uploaders
98 ### pa: add deb to db:
99 ### pa: add file to file
100 ### pa: find source entry
101 ### pa: create binaries entry
102 ### pa: update binary associations
103 ### pa: .orig component move
104 ### pa: move files to pool
106 ### pa: move CHG to done/
107 ### pa: change entry in queue_build
108 ## pu: use dispatch table to choose target queue:
110 ## pu: write .dak file
114 ## pu: run autobyhand script
115 ## pu: if stuff left, do byhand or accept
116 ## elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ## pu: write .dak file
118 ## pu: check overrides
122 ## pu: write .dak file
123 ## pu: move to ACCEPTED
125 ## pu: create files for BTS
126 ## pu: create entry in queue_build
127 ## pu: check overrides
131 ## Parsing changes (check for duplicates)
135 # New check layout (TODO: Implement)
139 ### version checks (suite)
153 ### src relation check
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
162 from errno import EACCES, EAGAIN
168 from sqlalchemy.orm.exc import NoResultFound
170 from daklib import daklog
171 from daklib.queue import *
172 from daklib.queue_install import *
173 from daklib import utils
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.holding import Holding
178 from daklib.config import Config
179 from daklib.regexes import re_match_expired
181 ###############################################################################
186 ###############################################################################
188 def usage (exit_code=0):
189 print """Usage: dak process-upload [OPTION]... [CHANGES]...
190 -a, --automatic automatic run
191 -h, --help show this help and exit.
192 -n, --no-action don't do anything
193 -p, --no-lock don't check lockfile !! for cron.daily only !!
194 -s, --no-mail don't send any mail
195 -V, --version display the version number and exit"""
198 ###############################################################################
201 if not Options["No-Action"]:
202 # Clean out the queue files
203 session = DBConn().session()
204 session.execute("DELETE FROM changes_pending_files WHERE id NOT IN (SELECT file_id FROM changes_pending_files_map )")
209 def action(u, session):
213 # changes["distribution"] may not exist in corner cases
214 # (e.g. unreadable changes files)
215 if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], dict):
216 u.pkg.changes["distribution"] = {}
218 (summary, short_summary) = u.build_summaries()
220 (prompt, answer) = ("", "XXX")
221 if Options["No-Action"] or Options["Automatic"]:
226 pi = u.package_info()
229 chg = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one()
230 except NoResultFound, e:
233 if len(u.rejects) > 0:
234 if u.upload_too_new():
235 print "SKIP (too new)\n" + pi,
236 prompt = "[S]kip, Quit ?"
238 print "REJECT\n" + pi
239 prompt = "[R]eject, Skip, Quit ?"
240 if Options["Automatic"]:
243 # Are we headed for NEW / BYHAND / AUTOBYHAND?
244 # Note that policy queues are no longer handled here
245 qu = determine_target(u)
247 print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary)
248 queuekey = qu[0].upper()
249 if queuekey in "RQSA":
251 prompt = "[D]ivert, Skip, Quit ?"
253 prompt = "[%s]%s, Skip, Quit ?" % (queuekey, qu[1:].lower())
254 if Options["Automatic"]:
257 # Does suite have a policy_queue configured
259 for s in u.pkg.changes["distribution"].keys():
260 suite = get_suite(s, session)
261 if suite.policy_queue:
262 if not chg or chg.approved_for_id != suite.policy_queue.policy_queue_id:
263 # This routine will check whether the upload is a binary
264 # upload when the source is already in the target suite. If
265 # so, we skip the policy queue, otherwise we go there.
266 divert = package_to_suite(u, suite.suite_name, session=session)
268 print "%s for %s\n%s%s" % ( suite.policy_queue.queue_name.upper(),
269 ", ".join(u.pkg.changes["distribution"].keys()),
272 prompt = "[P]olicy, Skip, Quit ?"
273 policyqueue = suite.policy_queue
274 if Options["Automatic"]:
279 print "ACCEPT\n" + pi + summary,
280 prompt = "[A]ccept, Skip, Quit ?"
281 if Options["Automatic"]:
284 while prompt.find(answer) == -1:
285 answer = utils.our_raw_input(prompt)
286 m = re_default_answer.match(prompt)
289 answer = answer[:1].upper()
292 os.chdir(u.pkg.directory)
296 chg = u.pkg.add_known_changes(holding.holding_dir, session=session)
298 u.accept(summary, short_summary, session)
300 chg.clean_from_queue()
305 chg = u.pkg.add_known_changes(holding.holding_dir, session=session)
306 package_to_queue(u, summary, short_summary, policyqueue, chg, session)
309 elif answer == queuekey:
311 chg = u.pkg.add_known_changes(holding.holding_dir, session=session)
312 QueueInfo[qu]["process"](u, summary, short_summary, chg, session)
321 ###############################################################################
325 if not Options["No-Action"]:
328 def process_it(changes_file, session):
331 Logger.log(["Processing changes file", changes_file])
337 # TODO: Actually implement using pending* tables so that we don't lose track
341 u.pkg.changes_file = changes_file
342 u.pkg.directory = os.getcwd()
344 origchanges = os.path.abspath(u.pkg.changes_file)
346 # Some defaults in case we can't fully process the .changes file
347 u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
348 u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
350 # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
351 bcc = "X-DAK: dak process-upload"
352 if cnf.has_key("Dinstall::Bcc"):
353 u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
355 u.Subst["__BCC__"] = bcc
357 # Remember where we are so we can come back after cd-ing into the
358 # holding directory. TODO: Fix this stupid hack
359 u.prevdir = os.getcwd()
362 # If this is the Real Thing(tm), copy things into a private
363 # holding directory first to avoid replacable file races.
364 if not Options["No-Action"]:
365 os.chdir(cnf["Dir::Queue::Holding"])
367 # Absolutize the filename to avoid the requirement of being in the
368 # same directory as the .changes file.
369 holding.copy_to_holding(origchanges)
371 # Relativize the filename so we use the copy in holding
372 # rather than the original...
373 changespath = os.path.basename(u.pkg.changes_file)
375 changespath = origchanges
377 (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changespath)
379 if u.pkg.changes["fingerprint"]:
380 valid_changes_p = u.load_changes(changespath)
382 for reason in rejects:
383 if re_match_expired.match(reason):
384 # Hrm, key expired. Lets see if we can still parse the .changes before
385 # we reject. Then we would be able to mail the maintainer, instead of
386 # just silently dropping the upload.
387 u.load_changes(changespath)
388 valid_changes_p = False
389 u.rejects.extend(rejects)
392 u.check_distributions()
393 u.check_files(not Options["No-Action"])
394 valid_dsc_p = u.check_dsc(not Options["No-Action"])
395 if valid_dsc_p and not Options["No-Action"]:
398 if valid_dsc_p and not Options["No-Action"] and not len(u.rejects):
402 u.check_signed_by_key()
406 except (SystemExit, KeyboardInterrupt):
412 traceback.print_exc(file=sys.stderr)
415 # Restore previous WD
418 ###############################################################################
421 global Options, Logger
424 summarystats = SummaryStats()
429 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
430 ('h',"help","Dinstall::Options::Help"),
431 ('n',"no-action","Dinstall::Options::No-Action"),
432 ('p',"no-lock", "Dinstall::Options::No-Lock"),
433 ('s',"no-mail", "Dinstall::Options::No-Mail"),
434 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
436 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
437 "version", "directory"]:
438 if not cnf.has_key("Dinstall::Options::%s" % (i)):
439 cnf["Dinstall::Options::%s" % (i)] = ""
441 changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
442 Options = cnf.SubTree("Dinstall::Options")
447 # -n/--dry-run invalidates some other options which would involve things happening
448 if Options["No-Action"]:
449 Options["Automatic"] = ""
451 # Check that we aren't going to clash with the daily cron job
452 if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
453 utils.fubar("Archive maintenance in progress. Try again later.")
455 # Obtain lock if not in no-action mode and initialize the log
456 if not Options["No-Action"]:
457 lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
459 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
461 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
462 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
465 if cnf.get("Dir::UrgencyLog"):
466 # Initialise UrgencyLog()
470 Logger = daklog.Logger(cnf, "process-upload", Options["No-Action"])
472 # If we have a directory flag, use it to find our files
473 if cnf["Dinstall::Options::Directory"] != "":
474 # Note that we clobber the list of files we were given in this case
475 # so warn if the user has done both
476 if len(changes_files) > 0:
477 utils.warn("Directory provided so ignoring files given on command line")
479 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
480 Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
481 elif not len(changes_files) > 0:
482 utils.fubar("No changes files given and no directory specified")
484 Logger.log(["Using changes files from command-line", len(changes_files)])
486 # Sort the .changes files so that we process sourceful ones first
487 changes_files.sort(utils.changes_compare)
489 # Process the changes files
490 for changes_file in changes_files:
491 print "\n" + changes_file
492 session = DBConn().session()
493 process_it(changes_file, session)
496 if summarystats.accept_count:
498 if summarystats.accept_count > 1:
500 print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
501 utils.size_type(int(summarystats.accept_bytes)))
502 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
506 if not Options["No-Action"]:
512 ###############################################################################
514 if __name__ == '__main__':