4 Installs Debian packages from queue/accepted into the pool
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ###############################################################################
28 # Cartman: "I'm trying to make the best of a bad situation, I don't
29 # need to hear crap from a bunch of hippy freaks living in
30 # denial. Screw you guys, I'm going home."
32 # Kyle: "But Cartman, we're trying to..."
34 # Cartman: "uhh.. screw you guys... home."
36 ###############################################################################
42 from datetime import datetime
44 import apt_pkg, commands
46 from daklib import daklog
47 from daklib import queue
48 from daklib import utils
49 from daklib.dbconn import *
50 from daklib.binary import copy_temporary_contents
51 from daklib.dak_exceptions import *
52 from daklib.regexes import re_default_answer, re_issource, re_fdnic
53 from daklib.urgencylog import UrgencyLog
54 from daklib.summarystats import SummaryStats
55 from daklib.config import Config
57 ###############################################################################
62 ###############################################################################
67 # Initialize config and connection to db
71 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
72 ('h',"help","Dinstall::Options::Help"),
73 ('n',"no-action","Dinstall::Options::No-Action"),
74 ('p',"no-lock", "Dinstall::Options::No-Lock"),
75 ('s',"no-mail", "Dinstall::Options::No-Mail"),
76 ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
78 for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
79 "version", "directory"]:
80 if not cnf.has_key("Dinstall::Options::%s" % (i)):
81 cnf["Dinstall::Options::%s" % (i)] = ""
83 changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
84 Options = cnf.SubTree("Dinstall::Options")
89 # If we have a directory flag, use it to find our files
90 if cnf["Dinstall::Options::Directory"] != "":
91 # Note that we clobber the list of files we were given in this case
92 # so warn if the user has done both
93 if len(changes_files) > 0:
94 utils.warn("Directory provided so ignoring files given on command line")
96 changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
100 ###############################################################################
102 def usage (exit_code=0):
103 print """Usage: dak process-accepted [OPTION]... [CHANGES]...
104 -a, --automatic automatic run
105 -h, --help show this help and exit.
106 -n, --no-action don't do anything
107 -p, --no-lock don't check lockfile !! for cron.daily only !!
108 -s, --no-mail don't send any mail
109 -V, --version display the version number and exit"""
112 ###############################################################################
114 def action (u, stable_queue=None, log_urgency=True):
115 (summary, short_summary) = u.build_summaries()
116 pi = u.package_info()
118 (prompt, answer) = ("", "XXX")
119 if Options["No-Action"] or Options["Automatic"]:
122 if len(u.rejects) > 0:
123 print "REJECT\n" + pi
124 prompt = "[R]eject, Skip, Quit ?"
125 if Options["Automatic"]:
128 print "INSTALL to " + ", ".join(u.pkg.changes["distribution"].keys())
130 prompt = "[I]nstall, Skip, Quit ?"
131 if Options["Automatic"]:
134 while prompt.find(answer) == -1:
135 answer = utils.our_raw_input(prompt)
136 m = re_default_answer.match(prompt)
139 answer = answer[:1].upper()
143 Logger.log(["unaccepted", u.pkg.changes_file])
146 stable_install(u, summary, short_summary, stable_queue, log_urgency)
148 install(u, log_urgency)
153 ###############################################################################
154 def add_poolfile(filename, datadict, location_id, session):
155 poolfile = PoolFile()
156 poolfile.filename = filename
157 poolfile.filesize = datadict["size"]
158 poolfile.md5sum = datadict["md5sum"]
159 poolfile.sha1sum = datadict["sha1sum"]
160 poolfile.sha256sum = datadict["sha256sum"]
161 poolfile.location_id = location_id
163 session.add(poolfile)
164 # Flush to get a file id (NB: This is not a commit)
169 def add_dsc_to_db(u, filename, session):
170 entry = u.pkg.files[filename]
173 source.source = u.pkg.dsc["source"]
174 source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
175 source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
176 source.changedby_id = get_or_set_maintainer(u.pkg.dsc["changed-by"], session).maintainer_id
177 source.fingerprint_id = get_or_set_fingerprint(u.pkg.dsc["fingerprint"], session).fingerprint_id
178 source.install_date = datetime.now().date()
180 dsc_component = entry["component"]
181 dsc_location_id = entry["location id"]
183 source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
185 # Set up a new poolfile if necessary
186 if not entry.has_key("files id") or not entry["files id"]:
187 filename = entry["pool name"] + filename
188 poolfile = add_poolfile(filename, entry, dsc_location_id, session)
189 entry["files id"] = poolfile.file_id
191 source.poolfile_id = entry["files id"]
195 for suite_name in u.pkg.changes["distribution"].keys():
196 sa = SrcAssociation()
197 sa.source_id = source.source_id
198 sa.suite_id = get_suite(suite_name).suite_id
203 # Add the source files to the DB (files and dsc_files)
205 dscfile.source_id = source.source_id
206 dscfile.poolfile_id = entry["files id"]
209 for dsc_file, dentry in u.pkg.dsc_files.keys():
211 df.source_id = source.source_id
213 # If the .orig.tar.gz is already in the pool, it's
214 # files id is stored in dsc_files by check_dsc().
215 files_id = dentry.get("files id", None)
218 filename = dentry["pool name"] + dsc_file
220 (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
221 # FIXME: needs to check for -1/-2 and or handle exception
222 if found and obj is not None:
223 files_id = obj.file_id
225 # If still not found, add it
227 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
228 files_id = poolfile.file_id
230 df.poolfile_id = files_id
235 # Add the src_uploaders to the DB
236 uploader_ids = [maintainer_id]
237 if u.pkg.dsc.has_key("uploaders"):
238 for up in u.pkg.dsc["uploaders"].split(","):
240 uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
243 for up in uploader_ids:
244 if added_ids.has_key(up):
245 utils.warn("Already saw uploader %s for source %s" % (up, source.source))
251 su.maintainer_id = up
252 su.source_id = source_id
257 return dsc_component, dsc_location_id
259 def add_deb_to_db(u, filename, session):
261 Contrary to what you might expect, this routine deals with both
262 debs and udebs. That info is in 'dbtype', whilst 'type' is
263 'deb' for both of them
266 entry = u.pkg.files[filename]
269 bin.package = entry["package"]
270 bin.version = entry["version"]
271 bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
272 bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
273 bin.arch_id = get_architecture(entry["architecture"], session).arch_id
274 bin.binarytype = entry["dbtype"]
277 filename = entry["pool name"] + filename
278 if not entry.get("location id", None):
279 entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
281 if not entry.get("files id", None):
282 poolfile = add_poolfile(filename, entry, entry["location id"], session)
283 entry["files id"] = poolfile.file_id
285 bin.poolfile_id = entry["files id"]
288 bin_sources = get_sources_from_name(entry["source package"], entry["source version"])
289 if len(bin_sources) != 1:
290 raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
291 (bin.package, bin.version, bin.architecture.arch_string,
292 filename, bin.binarytype, u.pkg.changes["fingerprint"])
294 bin.source_id = bin_sources[0].source_id
296 # Add and flush object so it has an ID
300 # Add BinAssociations
301 for suite_name in u.pkg.changes["distribution"].keys():
302 ba = BinAssociation()
303 ba.binary_id = bin.binary_id
304 ba.suite_id = get_suite(suite_name).suite_id
310 contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, filename, reject=None)
312 print "REJECT\n" + "\n".join(contents.rejects)
314 raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
317 def install(u, log_urgency=True):
319 summarystats = SummaryStats()
323 Logger.log(["installing changes",pkg.changes_file])
325 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
326 session = DBConn().session()
328 # Ensure that we have all the hashes we need below.
330 if len(u.rejects) > 0:
331 # There were errors. Print them and SKIP the changes.
332 for msg in u.rejects:
336 # Add the .dsc file to the DB first
337 for newfile in u.pkg.files.keys():
338 if entry["type"] == "dsc":
339 dsc_component, dsc_location_id = add_dsc_to_db(u, newfile, session)
341 # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
342 for newfile in u.pkg.files.keys():
343 if entry["type"] == "deb":
344 add_deb_to_db(u, newfile, session)
346 # If this is a sourceful diff only upload that is moving
347 # cross-component we need to copy the .orig.tar.gz into the new
348 # component too for the same reasons as above.
350 if u.pkg.changes["architecture"].has_key("source") and u.pkg.orig_tar_id and \
351 u.pkg.orig_tar_location != dsc_location_id:
353 oldf = get_poolfile_by_id(u.pkg.orig_tar_id, session)
354 old_filename = os.path.join(oldf.location.path, oldf.filename)
355 old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
356 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
358 new_filename = os.path.join(utils.poolify(u.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
360 # TODO: Care about size/md5sum collisions etc
361 (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
364 utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
365 newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
367 # TODO: Check that there's only 1 here
368 source = get_sources_from_name(u.pkg.changes["source"], u.pkg.changes["version"])[0]
369 dscf = get_dscfiles(source_id = source.source_id, poolfile_id=u.pkg.orig_tar_id, session=session)[0]
370 dscf.poolfile_id = newf.file_id
374 # Install the files into the pool
375 for newfile, entry in u.pkg.files.items():
376 destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
377 utils.move(newfile, destination)
378 Logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
379 summarystats.accept_bytes += float(entry["size"])
381 # Copy the .changes file across for suite which need it.
384 for suite_name in changes["distribution"].keys():
385 if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
386 copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
387 # and the .dak file...
388 if cnf.has_key("Suite::%s::CopyDotDak" % (suite_name)):
389 copy_dot_dak[cnf["Suite::%s::CopyDotDak" % (suite_name)]] = ""
391 for dest in copy_changes.keys():
392 utils.copy(u.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
394 for dest in copy_dot_dak.keys():
395 utils.copy(u.pkg.changes_file[:-8]+".dak", dest)
397 # We're done - commit the database changes
400 # Move the .changes into the 'done' directory
401 utils.move(u.pkg.changes_file,
402 os.path.join(cnf["Dir::Queue::Done"], os.path.basename(u.pkg.changes_file)))
404 # Remove the .dak file
405 os.unlink(u.pkg.changes_file[:-8] + ".dak")
407 if u.pkg.changes["architecture"].has_key("source") and log_urgency:
408 UrgencyLog().log(u.pkg.dsc["source"], u.pkg.dsc["version"], u.pkg.changes["urgency"])
410 # Our SQL session will automatically start a new transaction after
413 # Undo the work done in queue.py(accept) to help auto-building
415 now_date = datetime.now()
417 for suite_name in u.pkg.changes["distribution"].keys():
418 if suite_name not in cnf.ValueList("Dinstall::QueueBuildSuites"):
421 suite = get_suite(suite_name, session)
422 dest_dir = cnf["Dir::QueueBuild"]
424 if cnf.FindB("Dinstall::SecurityQueueBuild"):
425 dest_dir = os.path.join(dest_dir, suite_name)
427 for newfile, entry in u.pkg.files.items():
428 dest = os.path.join(dest_dir, newfile)
430 qb = get_queue_build(dest, suite.suite_id, session)
432 # Remove it from the list of packages for later processing by apt-ftparchive
434 qb.last_used = now_date
438 if not cnf.FindB("Dinstall::SecurityQueueBuild"):
439 # Update the symlink to point to the new location in the pool
440 pool_location = utils.poolify(u.pkg.changes["source"], entry["component"])
441 src = os.path.join(cnf["Dir::Pool"], pool_location, os.path.basename(newfile))
442 if os.path.islink(dest):
444 os.symlink(src, dest)
446 # Update last_used on any non-upload .orig.tar.gz symlink
447 if u.pkg.orig_tar_id:
448 # Determine the .orig.tar.gz file name
449 for dsc_file in u.pkg.dsc_files.keys():
450 if dsc_file.endswith(".orig.tar.gz"):
451 u.pkg.orig_tar_gz = os.path.join(dest_dir, dsc_file)
453 # Remove it from the list of packages for later processing by apt-ftparchive
454 qb = get_queue_build(u.pkg.orig_tar_gz, suite.suite_id, session)
457 qb.last_used = now_date
463 summarystats.accept_count += 1
465 ################################################################################
468 def stable_install(u, summary, short_summary, fromsuite_name="proposed-updates"):
469 summarystats = SummaryStats()
471 fromsuite_name = fromsuite_name.lower()
472 tosuite_name = "Stable"
473 if fromsuite_name == "oldstable-proposed-updates":
474 tosuite_name = "OldStable"
476 print "Installing from %s to %s." % (fromsuite_name, tosuite_name)
478 fromsuite = get_suite(fromsuite_name)
479 tosuite = get_suite(tosuite_name)
481 # Begin a transaction; if we bomb out anywhere between here and
482 # the COMMIT WORK below, the DB won't be changed.
483 session = DBConn().session()
485 # Add the source to stable (and remove it from proposed-updates)
486 for newfile, entry in u.pkg.files.items():
487 if entry["type"] == "dsc":
488 package = u.pkg.dsc["source"]
489 # NB: not files[file]["version"], that has no epoch
490 version = u.pkg.dsc["version"]
492 source = get_sources_from_name(package, version, session)
494 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version))
497 # Remove from old suite
498 old = session.query(SrcAssociation).filter_by(source_id = source.source_id)
499 old = old.filter_by(suite_id = fromsuite.suite_id)
503 new = SrcAssociation()
504 new.source_id = source.source_id
505 new.suite_id = tosuite.suite_id
508 # Add the binaries to stable (and remove it/them from proposed-updates)
509 for newfile, entry in u.pkg.files.items():
510 if entry["type"] == "deb":
511 package = entry["package"]
512 version = entry["version"]
513 architecture = entry["architecture"]
515 binary = get_binaries_from_name(package, version, [architecture, 'all'])
518 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture))
521 # Remove from old suite
522 old = session.query(BinAssociation).filter_by(binary_id = binary.binary_id)
523 old = old.filter_by(suite_id = fromsuite.suite_id)
527 new = BinAssociation()
528 new.binary_id = binary.binary_id
529 new.suite_id = tosuite.suite_id
534 utils.move(u.pkg.changes_file,
535 os.path.join(cnf["Dir::Morgue"], 'process-accepted', os.path.basename(u.pkg.changes_file)))
537 ## Update the Stable ChangeLog file
538 # TODO: URGH - Use a proper tmp file
539 new_changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + ".ChangeLog"
540 changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + "ChangeLog"
541 if os.path.exists(new_changelog_filename):
542 os.unlink(new_changelog_filename)
544 new_changelog = utils.open_file(new_changelog_filename, 'w')
545 for newfile, entry in u.pkg.files.items():
546 if entry["type"] == "deb":
547 new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.suite_name,
549 entry["architecture"],
551 elif re_issource.match(newfile):
552 new_changelog.write("%s/%s/source/%s\n" % (tosuite.suite_name,
556 new_changelog.write("%s\n" % (newfile))
558 chop_changes = re_fdnic.sub("\n", u.pkg.changes["changes"])
559 new_changelog.write(chop_changes + '\n\n')
561 if os.access(changelog_filename, os.R_OK) != 0:
562 changelog = utils.open_file(changelog_filename)
563 new_changelog.write(changelog.read())
565 new_changelog.close()
567 if os.access(changelog_filename, os.R_OK) != 0:
568 os.unlink(changelog_filename)
569 utils.move(new_changelog_filename, changelog_filename)
571 summarystats.accept_count += 1
573 if not Options["No-Mail"] and u.pkg.changes["architecture"].has_key("source"):
574 u.Subst["__SUITE__"] = " into %s" % (tosuite)
575 u.Subst["__SUMMARY__"] = summary
576 u.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
578 if cnf.has_key("Dinstall::Bcc"):
579 u.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
581 template = os.path.join(cnf["Dir::Templates"], 'process-accepted.install')
583 mail_message = utils.TemplateSubst(u.Subst, template)
584 utils.send_mail(mail_message)
585 u.announce(short_summary, True)
587 # Finally remove the .dak file
588 dot_dak_file = os.path.join(cnf["Suite::%s::CopyDotDak" % (fromsuite.suite_name)],
589 os.path.basename(u.pkg.changes_file[:-8]+".dak"))
590 os.unlink(dot_dak_file)
592 ################################################################################
594 def process_it(changes_file, stable_queue=None, log_urgency=True):
598 overwrite_checks = True
600 # Absolutize the filename to avoid the requirement of being in the
601 # same directory as the .changes file.
602 cfile = os.path.abspath(changes_file)
604 # And since handling of installs to stable munges with the CWD
605 # save and restore it.
606 u.prevdir = os.getcwd()
610 cfile = os.path.basename(old)
611 os.chdir(cnf["Suite::%s::CopyDotDak" % (stable_queue)])
612 # overwrite_checks should not be performed if installing to stable
613 overwrite_checks = False
615 u.load_dot_dak(cfile)
619 u.pkg.changes_file = old
621 u.accepted_checks(overwrite_checks)
622 action(u, stable_queue, log_urgency)
627 ###############################################################################
633 summarystats = SummaryStats()
634 changes_files = init()
638 # -n/--dry-run invalidates some other options which would involve things happening
639 if Options["No-Action"]:
640 Options["Automatic"] = ""
642 # Check that we aren't going to clash with the daily cron job
644 if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (cnf["Dir::Root"])) and not Options["No-Lock"]:
645 utils.fubar("Archive maintenance in progress. Try again later.")
647 # If running from within proposed-updates; assume an install to stable
649 if os.getenv('PWD').find('oldstable-proposed-updates') != -1:
650 stable_queue = "Oldstable-Proposed-Updates"
651 elif os.getenv('PWD').find('proposed-updates') != -1:
652 stable_queue = "Proposed-Updates"
654 # Obtain lock if not in no-action mode and initialize the log
655 if not Options["No-Action"]:
656 lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
658 fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
660 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
661 utils.fubar("Couldn't obtain lock; assuming another 'dak process-accepted' is already running.")
664 Logger = daklog.Logger(cnf, "process-accepted")
665 if not stable_queue and cnf.get("Dir::UrgencyLog"):
666 # Initialise UrgencyLog()
670 # Sort the .changes files so that we process sourceful ones first
671 changes_files.sort(utils.changes_compare)
673 # Process the changes files
674 for changes_file in changes_files:
675 print "\n" + changes_file
676 process_it(changes_file, stable_queue, log_urgency)
678 if summarystats.accept_count:
680 if summarystats.accept_count > 1:
682 sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets,
683 utils.size_type(int(summarystats.accept_bytes))))
684 Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
686 if not Options["No-Action"]:
691 ###############################################################################
693 if __name__ == '__main__':