X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fgenerate_releases.py;h=6c4fed0dcf64c2b84bfb6983cb14556fd858225d;hb=3b8862ae0e21fae9fc552d4df160f45684976d7d;hp=e67bd91f6d2b3e0ed0078dc93d72d1e0c51002a9;hpb=886522194878afe478e4dd61397493f35b5b0e5d;p=dak.git diff --git a/dak/generate_releases.py b/dak/generate_releases.py index e67bd91f..6c4fed0d 100755 --- a/dak/generate_releases.py +++ b/dak/generate_releases.py @@ -40,7 +40,6 @@ import bz2 import apt_pkg from tempfile import mkstemp, mkdtemp import commands -from multiprocessing import Pool, TimeoutError from sqlalchemy.orm import object_session from daklib import utils, daklog @@ -48,10 +47,10 @@ from daklib.regexes import re_gensubrelease, re_includeinrelease from daklib.dak_exceptions import * from daklib.dbconn import * from daklib.config import Config +from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS ################################################################################ Logger = None #: Our logging object -results = [] #: Results of the subprocesses ################################################################################ @@ -74,12 +73,7 @@ SUITE can be a space seperated list, e.g. ######################################################################## -def get_result(arg): - global results - if arg: - results.append(arg) - -def sign_release_dir(dirname): +def sign_release_dir(suite, dirname): cnf = Config() if cnf.has_key("Dinstall::SigningKeyring"): @@ -88,7 +82,6 @@ def sign_release_dir(dirname): keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"] arguments = "--no-options --batch --no-tty --armour" - signkeyids = cnf.signingkeyids.split() relname = os.path.join(dirname, 'Release') @@ -100,17 +93,20 @@ def sign_release_dir(dirname): if os.path.exists(inlinedest): os.unlink(inlinedest) - for keyid in signkeyids: - if keyid != "": - defkeyid = "--default-key %s" % keyid - else: - defkeyid = "" + # We can only use one key for inline signing so use the first one in + # the array for consistency + firstkey = True + + for keyid in suite.signingkeys: + defkeyid = "--default-key %s" % keyid os.system("gpg %s %s %s --detach-sign <%s >>%s" % (keyring, defkeyid, arguments, relname, dest)) - os.system("gpg %s %s %s --clearsign <%s >>%s" % - (keyring, defkeyid, arguments, relname, inlinedest)) + if firstkey: + os.system("gpg %s %s %s --clearsign <%s >>%s" % + (keyring, defkeyid, arguments, relname, inlinedest)) + firstkey = False class ReleaseWriter(object): def __init__(self, suite): @@ -138,9 +134,9 @@ class ReleaseWriter(object): ('Codename', 'codename') ) # A "Sub" Release file has slightly different fields - subattribs = ( ('Origin', 'origin'), + subattribs = ( ('Archive', 'suite_name'), + ('Origin', 'origin'), ('Label', 'label'), - ('Archive', 'suite_name'), ('Version', 'version') ) # Boolean stuff. If we find it true in database, write out "yes" into the release file @@ -152,7 +148,7 @@ class ReleaseWriter(object): suite_suffix = "%s" % (cnf.Find("Dinstall::SuiteSuffix")) outfile = os.path.join(cnf["Dir::Root"], 'dists', "%s/%s" % (suite.suite_name, suite_suffix), "Release") - out = open(outfile, "w") + out = open(outfile + ".new", "w") for key, dbfield in attribs: if getattr(suite, dbfield) is not None: @@ -182,12 +178,12 @@ class ReleaseWriter(object): out.write("Description: %s\n" % suite.description) for comp in components: - for dirpath, dirnames, filenames in os.walk("%sdists/%s/%s" % (cnf["Dir::Root"], suite.suite_name, comp), topdown=True): + for dirpath, dirnames, filenames in os.walk("%sdists/%s/%s%s" % (cnf["Dir::Root"], suite.suite_name, suite_suffix, comp), topdown=True): if not re_gensubrelease.match(dirpath): continue subfile = os.path.join(dirpath, "Release") - subrel = open(subfile, "w") + subrel = open(subfile + '.new', "w") for key, dbfield in subattribs: if getattr(suite, dbfield) is not None: @@ -198,8 +194,18 @@ class ReleaseWriter(object): subrel.write("%s: yes\n" % (key)) subrel.write("Component: %s%s\n" % (suite_suffix, comp)) + + # Urgh, but until we have all the suite/component/arch stuff in the DB, + # this'll have to do + arch = os.path.split(dirpath)[-1] + if arch.startswith('binary-'): + arch = arch[7:] + + subrel.write("Architecture: %s\n" % (arch)) subrel.close() + os.rename(subfile + '.new', subfile) + # Now that we have done the groundwork, we want to get off and add the files with # their checksums to the main Release file oldcwd = os.getcwd() @@ -268,8 +274,9 @@ class ReleaseWriter(object): out.write(" %s %8d %s\n" % (fileinfo[filename][h], fileinfo[filename]['len'], filename)) out.close() + os.rename(outfile + '.new', outfile) - sign_release_dir(os.path.dirname(outfile)) + sign_release_dir(suite, os.path.dirname(outfile)) os.chdir(oldcwd) @@ -277,7 +284,7 @@ class ReleaseWriter(object): def main (): - global Logger, results + global Logger cnf = Config() @@ -312,10 +319,8 @@ def main (): suites = session.query(Suite).filter(Suite.untouchable == False).all() broken=[] - # For each given suite, run one process - results = [] - pool = Pool() + pool = DakProcessPool() for s in suites: # Setup a multiprocessing Pool. As many workers as we have CPU cores. @@ -331,12 +336,10 @@ def main (): pool.close() pool.join() - retcode = 0 + retcode = p.overall_status() - if len(results) > 0: - Logger.log(['Release file generation broken: %s' % (results)]) - print "Release file generation broken:\n", '\n'.join(results) - retcode = 1 + if retcode > 0: + Logger.log(['Release file generation broken: %s' % (p.results)]) Logger.close() @@ -348,13 +351,12 @@ def generate_helper(suite_id): ''' session = DBConn().session() suite = Suite.get(suite_id, session) - try: - rw = ReleaseWriter(suite) - rw.generate_release_files() - except Exception, e: - return str(e) - return + # We allow the process handler to catch and deal with any exceptions + rw = ReleaseWriter(suite) + rw.generate_release_files() + + return (PROC_STATUS_SUCCESS, 'Release file written for %s' % suite.suite_name) #######################################################################################