4 Installs Debian packages from queue/accepted into the pool
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ###############################################################################
28 # Cartman: "I'm trying to make the best of a bad situation, I don't
29 # need to hear crap from a bunch of hippy freaks living in
30 # denial. Screw you guys, I'm going home."
32 # Kyle: "But Cartman, we're trying to..."
34 # Cartman: "uhh.. screw you guys... home."
36 ###############################################################################
42 from datetime import datetime
44 import apt_pkg, commands
46 from daklib import daklog
47 from daklib import queue
48 from daklib import utils
49 from daklib.dbconn import *
50 from daklib.binary import copy_temporary_contents
51 from daklib.dak_exceptions import *
52 from daklib.regexes import re_default_answer, re_issource, re_fdnic
53 from daklib.urgencylog import UrgencyLog
54 from daklib.summarystats import SummaryStats
56 ###############################################################################
61 ###############################################################################
66 # Initialize config and connection to db
70 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
71 ('h',"help","Dinstall::Options::Help"),
72 ('n',"no-action","Dinstall::Options::No-Action"),
73 ('p',"no-lock", "Dinstall::Options::No-Lock"),
74 ('s',"no-mail", "Dinstall::Options::No-Mail"),
75 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
77 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
78 "version", "directory"]:
79 if not cnf.has_key("Dinstall::Options::%s" % (i)):
80 cnf["Dinstall::Options::%s" % (i)] = ""
82 changes_files = apt_pkg.ParseCommandLine(cnf, Arguments, sys.argv)
83 Options = cnf.SubTree("Dinstall::Options")
88 # If we have a directory flag, use it to find our files
89 if cnf["Dinstall::Options::Directory"] != "":
90 # Note that we clobber the list of files we were given in this case
91 # so warn if the user has done both
92 if len(changes_files) > 0:
93 utils.warn("Directory provided so ignoring files given on command line")
95 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
99 ###############################################################################
101 def usage (exit_code=0):
102 print """Usage: dak process-accepted [OPTION]... [CHANGES]...
103 -a, --automatic automatic run
104 -h, --help show this help and exit.
105 -n, --no-action don't do anything
106 -p, --no-lock don't check lockfile !! for cron.daily only !!
107 -s, --no-mail don't send any mail
108 -V, --version display the version number and exit"""
111 ###############################################################################
113 def action (u, stable_queue=None, log_urgency=True):
114 (summary, short_summary) = u.build_summaries()
115 pi = u.package_info()
117 (prompt, answer) = ("", "XXX")
118 if Options["No-Action"] or Options["Automatic"]:
121 if len(u.rejects) > 0:
122 print "REJECT\n" + pi
123 prompt = "[R]eject, Skip, Quit ?"
124 if Options["Automatic"]:
127 print "INSTALL to " + ", ".join(u.pkg.changes["distribution"].keys())
129 prompt = "[I]nstall, Skip, Quit ?"
130 if Options["Automatic"]:
133 while prompt.find(answer) == -1:
134 answer = utils.our_raw_input(prompt)
135 m = re_default_answer.match(prompt)
138 answer = answer[:1].upper()
142 Logger.log(["unaccepted", u.pkg.changes_file])
145 stable_install(u, summary, short_summary, stable_queue, log_urgency)
147 install(u, log_urgency)
152 ###############################################################################
153 def add_poolfile(filename, datadict, location_id, session):
154 poolfile = PoolFile()
155 poolfile.filename = filename
156 poolfile.filesize = datadict["size"]
157 poolfile.md5sum = datadict["md5sum"]
158 poolfile.sha1sum = datadict["sha1sum"]
159 poolfile.sha256sum = datadict["sha256sum"]
160 poolfile.location_id = location_id
162 session.add(poolfile)
163 # Flush to get a file id (NB: This is not a commit)
168 def add_dsc_to_db(u, filename, session):
169 entry = u.pkg.files[filename]
172 source.source = u.pkg.dsc["source"]
173 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
174 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
175 source.changedby_id = get_or_set_maintainer(u.pkg.dsc["changed-by"], session).maintainer_id
176 source.fingerprint_id = get_or_set_fingerprint(u.pkg.dsc["fingerprint"], session).fingerprint_id
177 source.install_date = datetime.now().date()
179 dsc_component = entry["component"]
180 dsc_location_id = entry["location id"]
182 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
184 # Set up a new poolfile if necessary
185 if not entry.has_key("files id") or not entry["files id"]:
186 filename = entry["pool name"] + filename
187 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
188 entry["files id"] = poolfile.file_id
190 source.poolfile_id = entry["files id"]
194 for suite_name in u.pkg.changes["distribution"].keys():
195 sa = SrcAssociation()
196 sa.source_id = source.source_id
197 sa.suite_id = get_suite(suite_name).suite_id
202 # Add the source files to the DB (files and dsc_files)
204 dscfile.source_id = source.source_id
205 dscfile.poolfile_id = entry["files id"]
208 for dsc_file, dentry in u.pkg.dsc_files.keys():
210 df.source_id = source.source_id
212 # If the .orig.tar.gz is already in the pool, it's
213 # files id is stored in dsc_files by check_dsc().
214 files_id = dentry.get("files id", None)
217 filename = dentry["pool name"] + dsc_file
219 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
220 # FIXME: needs to check for -1/-2 and or handle exception
221 if found and obj is not None:
222 files_id = obj.file_id
224 # If still not found, add it
226 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
227 files_id = poolfile.file_id
229 df.poolfile_id = files_id
234 # Add the src_uploaders to the DB
235 uploader_ids = [maintainer_id]
236 if u.pkg.dsc.has_key("uploaders"):
237 for up in u.pkg.dsc["uploaders"].split(","):
239 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
242 for up in uploader_ids:
243 if added_ids.has_key(up):
244 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
250 su.maintainer_id = up
251 su.source_id = source_id
256 return dsc_component, dsc_location_id
258 def add_deb_to_db(u, filename, session):
260 Contrary to what you might expect, this routine deals with both
261 debs and udebs. That info is in 'dbtype', whilst 'type' is
262 'deb' for both of them
265 entry = u.pkg.files[filename]
268 bin.package = entry["package"]
269 bin.version = entry["version"]
270 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
271 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
272 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
273 bin.binarytype = entry["dbtype"]
276 filename = entry["pool name"] + filename
277 if not entry.get("location id", None):
278 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
280 if not entry.get("files id", None):
281 poolfile = add_poolfile(filename, entry, entry["location id"], session)
282 entry["files id"] = poolfile.file_id
284 bin.poolfile_id = entry["files id"]
287 bin_sources = get_sources_from_name(entry["source package"], entry["source version"])
288 if len(bin_sources) != 1:
289 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
290 (bin.package, bin.version, bin.architecture.arch_string,
291 filename, bin.binarytype, u.pkg.changes["fingerprint"])
293 bin.source_id = bin_sources[0].source_id
295 # Add and flush object so it has an ID
299 # Add BinAssociations
300 for suite_name in u.pkg.changes["distribution"].keys():
301 ba = BinAssociation()
302 ba.binary_id = bin.binary_id
303 ba.suite_id = get_suite(suite_name).suite_id
309 contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, filename, reject=None)
311 print "REJECT\n" + "\n".join(contents.rejects)
313 raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
316 def install(u, log_urgency=True):
318 summarystats = SummaryStats()
322 Logger.log(["installing changes",pkg.changes_file])
324 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
325 session = DBConn().session()
327 # Ensure that we have all the hashes we need below.
329 if len(u.rejects) > 0:
330 # There were errors. Print them and SKIP the changes.
331 for msg in u.rejects:
335 # Add the .dsc file to the DB first
336 for newfile in u.pkg.files.keys():
337 if entry["type"] == "dsc":
338 dsc_component, dsc_location_id = add_dsc_to_db(u, newfile, session)
340 # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
341 for newfile in u.pkg.files.keys():
342 if entry["type"] == "deb":
343 add_deb_to_db(u, newfile, session)
345 # If this is a sourceful diff only upload that is moving
346 # cross-component we need to copy the .orig.tar.gz into the new
347 # component too for the same reasons as above.
349 if u.pkg.changes["architecture"].has_key("source") and u.pkg.orig_tar_id and \
350 u.pkg.orig_tar_location != dsc_location_id:
352 oldf = get_poolfile_by_id(u.pkg.orig_tar_id, session)
353 old_filename = os.path.join(oldf.location.path, oldf.filename)
354 old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
355 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
357 new_filename = os.path.join(utils.poolify(u.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
359 # TODO: Care about size/md5sum collisions etc
360 (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
363 utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
364 newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
366 # TODO: Check that there's only 1 here
367 source = get_sources_from_name(u.pkg.changes["source"], u.pkg.changes["version"])[0]
368 dscf = get_dscfiles(source_id = source.source_id, poolfile_id=u.pkg.orig_tar_id, session=session)[0]
369 dscf.poolfile_id = newf.file_id
373 # Install the files into the pool
374 for newfile, entry in u.pkg.files.items():
375 destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
376 utils.move(newfile, destination)
377 Logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
378 summarystats.accept_bytes += float(entry["size"])
380 # Copy the .changes file across for suite which need it.
383 for suite_name in changes["distribution"].keys():
384 if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
385 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
386 # and the .dak file...
387 if cnf.has_key("Suite::%s::CopyDotDak" % (suite_name)):
388 copy_dot_dak[cnf["Suite::%s::CopyDotDak" % (suite_name)]] = ""
390 for dest in copy_changes.keys():
391 utils.copy(u.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
393 for dest in copy_dot_dak.keys():
394 utils.copy(u.pkg.changes_file[:-8]+".dak", dest)
396 # We're done - commit the database changes
399 # Move the .changes into the 'done' directory
400 utils.move(u.pkg.changes_file,
401 os.path.join(cnf["Dir::Queue::Done"], os.path.basename(u.pkg.changes_file)))
403 # Remove the .dak file
404 os.unlink(u.pkg.changes_file[:-8] + ".dak")
406 if u.pkg.changes["architecture"].has_key("source") and log_urgency:
407 UrgencyLog().log(u.pkg.dsc["source"], u.pkg.dsc["version"], u.pkg.changes["urgency"])
409 # Our SQL session will automatically start a new transaction after
412 # Undo the work done in queue.py(accept) to help auto-building
414 now_date = datetime.now()
416 for suite_name in u.pkg.changes["distribution"].keys():
417 if suite_name not in cnf.ValueList("Dinstall::QueueBuildSuites"):
420 suite = get_suite(suite_name, session)
421 dest_dir = cnf["Dir::QueueBuild"]
423 if cnf.FindB("Dinstall::SecurityQueueBuild"):
424 dest_dir = os.path.join(dest_dir, suite_name)
426 for newfile, entry in u.pkg.files.items():
427 dest = os.path.join(dest_dir, newfile)
429 qb = get_queue_build(dest, suite.suite_id, session)
431 # Remove it from the list of packages for later processing by apt-ftparchive
433 qb.last_used = now_date
437 if not cnf.FindB("Dinstall::SecurityQueueBuild"):
438 # Update the symlink to point to the new location in the pool
439 pool_location = utils.poolify(u.pkg.changes["source"], entry["component"])
440 src = os.path.join(cnf["Dir::Pool"], pool_location, os.path.basename(newfile))
441 if os.path.islink(dest):
443 os.symlink(src, dest)
445 # Update last_used on any non-upload .orig.tar.gz symlink
446 if u.pkg.orig_tar_id:
447 # Determine the .orig.tar.gz file name
448 for dsc_file in u.pkg.dsc_files.keys():
449 if dsc_file.endswith(".orig.tar.gz"):
450 u.pkg.orig_tar_gz = os.path.join(dest_dir, dsc_file)
452 # Remove it from the list of packages for later processing by apt-ftparchive
453 qb = get_queue_build(u.pkg.orig_tar_gz, suite.suite_id, session)
456 qb.last_used = now_date
462 summarystats.accept_count += 1
464 ################################################################################
467 def stable_install(u, summary, short_summary, fromsuite_name="proposed-updates"):
468 summarystats = SummaryStats()
470 fromsuite_name = fromsuite_name.lower()
471 tosuite_name = "Stable"
472 if fromsuite_name == "oldstable-proposed-updates":
473 tosuite_name = "OldStable"
475 print "Installing from %s to %s." % (fromsuite_name, tosuite_name)
477 fromsuite = get_suite(fromsuite_name)
478 tosuite = get_suite(tosuite_name)
480 # Begin a transaction; if we bomb out anywhere between here and
481 # the COMMIT WORK below, the DB won't be changed.
482 session = DBConn().session()
484 # Add the source to stable (and remove it from proposed-updates)
485 for newfile, entry in u.pkg.files.items():
486 if entry["type"] == "dsc":
487 package = u.pkg.dsc["source"]
488 # NB: not files[file]["version"], that has no epoch
489 version = u.pkg.dsc["version"]
491 source = get_sources_from_name(package, version, session)
493 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version))
496 # Remove from old suite
497 old = session.query(SrcAssociation).filter_by(source_id = source.source_id)
498 old = old.filter_by(suite_id = fromsuite.suite_id)
502 new = SrcAssociation()
503 new.source_id = source.source_id
504 new.suite_id = tosuite.suite_id
507 # Add the binaries to stable (and remove it/them from proposed-updates)
508 for newfile, entry in u.pkg.files.items():
509 if entry["type"] == "deb":
510 package = entry["package"]
511 version = entry["version"]
512 architecture = entry["architecture"]
514 binary = get_binaries_from_name(package, version, [architecture, 'all'])
517 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture))
520 # Remove from old suite
521 old = session.query(BinAssociation).filter_by(binary_id = binary.binary_id)
522 old = old.filter_by(suite_id = fromsuite.suite_id)
526 new = BinAssociation()
527 new.binary_id = binary.binary_id
528 new.suite_id = tosuite.suite_id
533 utils.move(u.pkg.changes_file,
534 os.path.join(cnf["Dir::Morgue"], 'process-accepted', os.path.basename(u.pkg.changes_file)))
536 ## Update the Stable ChangeLog file
537 # TODO: URGH - Use a proper tmp file
538 new_changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + ".ChangeLog"
539 changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + "ChangeLog"
540 if os.path.exists(new_changelog_filename):
541 os.unlink(new_changelog_filename)
543 new_changelog = utils.open_file(new_changelog_filename, 'w')
544 for newfile, entry in u.pkg.files.items():
545 if entry["type"] == "deb":
546 new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.suite_name,
548 entry["architecture"],
550 elif re_issource.match(newfile):
551 new_changelog.write("%s/%s/source/%s\n" % (tosuite.suite_name,
555 new_changelog.write("%s\n" % (newfile))
557 chop_changes = re_fdnic.sub("\n", u.pkg.changes["changes"])
558 new_changelog.write(chop_changes + '\n\n')
560 if os.access(changelog_filename, os.R_OK) != 0:
561 changelog = utils.open_file(changelog_filename)
562 new_changelog.write(changelog.read())
564 new_changelog.close()
566 if os.access(changelog_filename, os.R_OK) != 0:
567 os.unlink(changelog_filename)
568 utils.move(new_changelog_filename, changelog_filename)
570 summarystats.accept_count += 1
572 if not Options["No-Mail"] and u.pkg.changes["architecture"].has_key("source"):
573 u.Subst["__SUITE__"] = " into %s" % (tosuite)
574 u.Subst["__SUMMARY__"] = summary
575 u.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
577 if cnf.has_key("Dinstall::Bcc"):
578 u.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
580 template = os.path.join(cnf["Dir::Templates"], 'process-accepted.install')
582 mail_message = utils.TemplateSubst(u.Subst, template)
583 utils.send_mail(mail_message)
584 u.announce(short_summary, True)
586 # Finally remove the .dak file
587 dot_dak_file = os.path.join(cnf["Suite::%s::CopyDotDak" % (fromsuite.suite_name)],
588 os.path.basename(u.pkg.changes_file[:-8]+".dak"))
589 os.unlink(dot_dak_file)
591 ################################################################################
593 def process_it(changes_file, stable_queue=None, log_urgency=True):
597 overwrite_checks = True
599 # Absolutize the filename to avoid the requirement of being in the
600 # same directory as the .changes file.
601 cfile = os.path.abspath(changes_file)
603 # And since handling of installs to stable munges with the CWD
604 # save and restore it.
605 u.prevdir = os.getcwd()
609 cfile = os.path.basename(old)
610 os.chdir(cnf["Suite::%s::CopyDotDak" % (stable_queue)])
611 # overwrite_checks should not be performed if installing to stable
612 overwrite_checks = False
614 u.load_dot_dak(cfile)
618 u.pkg.changes_file = old
620 u.accepted_checks(overwrite_checks)
621 action(u, stable_queue, log_urgency)
626 ###############################################################################
632 summarystats = SummaryStats()
633 changes_files = init()
637 # -n/--dry-run invalidates some other options which would involve things happening
638 if Options["No-Action"]:
639 Options["Automatic"] = ""
641 # Check that we aren't going to clash with the daily cron job
643 if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (cnf["Dir::Root"])) and not Options["No-Lock"]:
644 utils.fubar("Archive maintenance in progress. Try again later.")
646 # If running from within proposed-updates; assume an install to stable
648 if os.getenv('PWD').find('oldstable-proposed-updates') != -1:
649 stable_queue = "Oldstable-Proposed-Updates"
650 elif os.getenv('PWD').find('proposed-updates') != -1:
651 stable_queue = "Proposed-Updates"
653 # Obtain lock if not in no-action mode and initialize the log
654 if not Options["No-Action"]:
655 lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
657 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
659 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
660 utils.fubar("Couldn't obtain lock; assuming another 'dak process-accepted' is already running.")
663 Logger = daklog.Logger(cnf, "process-accepted")
664 if not stable_queue and cnf.get("Dir::UrgencyLog"):
665 # Initialise UrgencyLog()
669 # Sort the .changes files so that we process sourceful ones first
670 changes_files.sort(utils.changes_compare)
672 # Process the changes files
673 for changes_file in changes_files:
674 print "\n" + changes_file
675 process_it(changes_file, stable_queue, log_urgency)
677 if summarystats.accept_count:
679 if summarystats.accept_count > 1:
681 sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets,
682 utils.size_type(int(summarystats.accept_bytes))))
683 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
685 if not Options["No-Action"]:
690 ###############################################################################
692 if __name__ == '__main__':