From: Michael Casadevall Date: Tue, 30 Dec 2008 21:10:35 +0000 (-0500) Subject: Merge branch 'master' into content_generation X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=30c2db7d2f47ff65ec9126f03c1e1e893db1c22c;hp=5e1dd4d24cf24e29213ae7d461f3bc8f4f040fb4;p=dak.git Merge branch 'master' into content_generation Signed-off-by: Michael Casadevall --- diff --git a/ChangeLog b/ChangeLog index d550ecda..946cf32a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2008-12-30 Joerg Jaspert + + * config/debian/cron.hourly: Generate the 822 format for accepted, + new, byhand and proposed-updates + +2008-12-30 Michael Casadevall + + * dak/queue_report.py - Added directories option to queue report + 2008-12-28 Frank Lichtenheld * dak/override.py (main): Handle source-only packages better diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index 3ff3f7d5..e543434a 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -11,7 +11,8 @@ date -u > $ftpdir/project/trace/ftp-master.debian.org echo "Using dak v1" >> $ftpdir/project/trace/ftp-master.debian.org echo "Running on host: $(hostname -f)" >> $ftpdir/project/trace/ftp-master.debian.org dak import-users-from-passwd -dak queue-report -n -8 > $webdir/new.html +dak queue-report -n > $webdir/new.html +dak queue-report -8 -d accepted,new,byhand,proposedupdates dak show-deferred > ${webdir}/deferred.html cd $queuedir/new ; dak show-new *.changes > /dev/null $base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc diff --git a/dak/generate_contents.py b/dak/generate_contents.py index 9696bb86..54b70bde 100755 --- a/dak/generate_contents.py +++ b/dak/generate_contents.py @@ -27,7 +27,7 @@ ################################################################################ import sys, os, popen2, tempfile, stat, time, pg -import apt_pkg +import gzip, apt_pkg from daklib import database, utils from daklib.dak_exceptions import * @@ -51,17 +51,6 @@ Generate Contents files ################################################################################ -def handle_dup_files(file_list): - # Sort the list, and then handle finding dups in the filenames key - - # Walk the list, seeing if the current entry and the next one are the same - # and if so, join them together - - - return file_list - -################################################################################ - def generate_contents(suites): global projectB, Cnf # Ok, the contents information is in the database @@ -69,38 +58,76 @@ def generate_contents(suites): # We need to work and get the contents, and print it out on a per # architectual basis + # Read in the contents file header + header = False + if Cnf.has_key("Generate-Contents::Header"): + h = open(Cnf["Generate-Contents::Header"], "r") + header = h.read() + h.close() + # Get our suites, and the architectures for s in suites: suite_id = database.get_suite_id(s) - q = projectB.query("SELECT architecture FROM suite_architectures WHERE suite = '%d'" % suite_id) + q = projectB.query("SELECT s.architecture, a.arch_string FROM suite_architectures s JOIN architecture a ON (s.architecture=a.id) WHERE suite = '%d'" % suite_id) arch_list = [ ] for r in q.getresult(): - arch_list.append(r[0]) + if r[1] != "source" and r[1] != "all": + arch_list.append((r[0], r[1])) arch_all_id = database.get_architecture_id("all") - # Got the arch all packages, now we need to get the arch dependent packages - # attach the arch all, stick them together, and write out the result + # Time for the query from hell. Essentially, we need to get the assiocations, the filenames, the paths, + # and all that fun stuff from the database. for arch_id in arch_list: - print "SELECT b.package, c.file, s.section FROM contents c JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE (b.architecture = '%d' OR b.architecture = '%d') AND ba.suite = '%d'" % (arch_id, arch_all_id, suite_id) - q = projectB.query("SELECT b.package, c.file, s.section FROM contents c JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE (b.architecture = '%d' OR b.architecture = '%d') AND ba.suite = '%d'" % (arch_id, arch_all_id, suite_id)) - # We need to copy the arch_all packages table into arch packages + q = projectB.query("""SELECT p.path||'/'||n.file, comma_separated_list(s.section||'/'||b.package) FROM content_associations c JOIN content_file_paths p ON (c.filepath=p.id) JOIN content_file_names n ON (c.filename=n.id) JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE (b.architecture = '%d' OR b.architecture = '%d') AND ba.suite = '%d' AND b.type = 'deb' GROUP BY (p.path||'/'||n.file)""" % (arch_id[0], arch_all_id, suite_id)) + + f = gzip.open(Cnf["Dir::Root"] + "dists/%s/Contents-%s.gz" % (s, arch_id[1]), "w") + + if header: + f.write(header) + + for contents in q.getresult(): + f.write(contents[0] + "\t\t\t" + contents[-1] + "\n") + + f.close() - # This is for the corner case of arch dependent packages colliding - # with arch all packages only on some architectures. - # Ugly, I know ... + # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free) + # This is HORRIBLY debian specific :-/ + # First off, udeb - arch_packages = [] - for r in q.getresult(): - arch_packages.append((r[1], (r[2] + '/' + r[0]))) + section_id = database.get_section_id('debian-installer') # all udebs should be here) - arch_packages = handle_dup_files(arch_packages) + if section_id != -1: + q = projectB.query("""SELECT p.path||'/'||n.file, comma_separated_list(s.section||'/'||b.package) FROM content_associations c JOIN content_file_paths p ON (c.filepath=p.id) JOIN content_file_names n ON (c.filename=n.id) JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE s.id = '%d' AND ba.suite = '%d' AND b.type = 'udeb' GROUP BY (p.path||'/'||n.file)""" % (section_id, suite_id)) - #for contents in arch_packages: - #print contents[0] + '\t\t\t\t' + contents[1] + f = gzip.open(Cnf["Dir::Root"] + "dists/%s/Contents-udeb.gz" % (s), "w") + + if header: + f.write(header) + + for contents in q.getresult(): + f.write(contents[0] + "\t\t\t" + contents[-1] + "\n") + + f.close() + + # Once more, with non-free + section_id = database.get_section_id('non-free/debian-installer') # all udebs should be here) + + if section_id != -1: + q = projectB.query("""SELECT p.path||'/'||n.file, comma_separated_list(s.section||'/'||b.package) FROM content_associations c JOIN content_file_paths p ON (c.filepath=p.id) JOIN content_file_names n ON (c.filename=n.id) JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE s.id = '%d' AND ba.suite = '%d' AND b.type = 'udeb' GROUP BY (p.path||'/'||n.file)""" % (section_id, suite_id)) + + f = gzip.open(Cnf["Dir::Root"] + "dists/%s/Contents-udeb-nf.gz" % (s), "w") + + if header: + f.write(header) + + for contents in q.getresult(): + f.write(contents[0] + "\t\t\t" + contents[-1] + "\n") + + f.close() ################################################################################ @@ -113,6 +140,7 @@ def main (): Arguments = [('h',"help","Generate-Contents::Options::Help"), ('s',"suite","Generate-Contents::Options::Suite","HasArg"), ] + for i in [ "help", "suite" ]: if not Cnf.has_key("Generate-Contents::Options::%s" % (i)): Cnf["Generate-Contents::Options::%s" % (i)] = "" diff --git a/dak/process_accepted.py b/dak/process_accepted.py index b6f8a311..4dd5b69d 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -407,8 +407,10 @@ def install (): projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id)) # insert contents into the database + q = projectB.query("SELECT currval('binaries_id_seq')") + bin_id = int(q.getresult()[0][0]) for file in contents: - projectB.query("INSERT INTO contents (binary_pkg, file) VALUES (currval('binaries_id_seq'), '%s')" % file) + database.insert_content_path(bin_id, file) # If the .orig.tar.gz is in a legacy directory we need to poolify # it, so that apt-get source (and anything else that goes by the diff --git a/dak/queue_report.py b/dak/queue_report.py index 31b11d8d..9e73d1c3 100755 --- a/dak/queue_report.py +++ b/dak/queue_report.py @@ -58,6 +58,7 @@ Prints a report of packages in queue directories (usually new and byhand). -s, --sort=key sort output according to key, see below. -a, --age=key if using sort by age, how should time be treated? If not given a default of hours will be used. + -d, --directories=key A comma seperated list of queues to be scanned Sorting Keys: ao=age, oldest first. an=age, newest first. na=name, ascending nd=name, descending @@ -515,7 +516,8 @@ def main(): ('n',"new","Queue-Report::Options::New"), ('8','822',"Queue-Report::Options::822"), ('s',"sort","Queue-Report::Options::Sort", "HasArg"), - ('a',"age","Queue-Report::Options::Age", "HasArg")] + ('a',"age","Queue-Report::Options::Age", "HasArg"), + ('d',"directories","Queue-Report::Options::Directories", "HasArg")] for i in [ "help" ]: if not Cnf.has_key("Queue-Report::Options::%s" % (i)): Cnf["Queue-Report::Options::%s" % (i)] = "" @@ -531,8 +533,14 @@ def main(): if Cnf.has_key("Queue-Report::Options::New"): header() - directories = Cnf.ValueList("Queue-Report::Directories") - if not directories: + directories = [ ] + + if Cnf.has_key("Queue-Report::Options::Directories"): + for i in Cnf["Queue-Report::Options::Directories"].split(","): + directories.append(i) + elif Cnf.has_key("Queue-Report::Directories"): + directories = Cnf.ValueList("Queue-Report::Directories") + else: directories = [ "byhand", "new" ] f = None diff --git a/daklib/database.py b/daklib/database.py index 5c7bd838..b9bb29dd 100755 --- a/daklib/database.py +++ b/daklib/database.py @@ -19,7 +19,7 @@ ################################################################################ -import sys, time, types +import os, sys, time, types ################################################################################ @@ -42,6 +42,8 @@ fingerprint_id_cache = {} queue_id_cache = {} uid_id_cache = {} suite_version_cache = {} +content_path_id_cache = {} +content_file_id_cache = {} ################################################################################ @@ -247,6 +249,26 @@ def get_suite_version(source, suite): return version +def get_latest_binary_version_id(binary, suite): + global suite_version_cache + cache_key = "%s_%s" % (binary, suite) + + + if suite_version_cache.has_key(cache_key): + return suite_version_cache[cache_key] + + #print "SELECT b.id, b.version FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) WHERE b.package = '%s AND ba.suite = '%d'" % (binary, int(suite)) + q = projectB.query("SELECT b.id, b.version FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) WHERE b.package = '%s AND ba.suite = '%d'" % (binary, int(suite))) + + highest_bid, highest_version = None, None + + for bi in q.getresult(): + if highest_version == None or apt_pkg.VersionCompare(bi[1], highest_version) == 1: + highest_bid = bi[0] + highest_version = bi[1] + + return highest_bid + ################################################################################ def get_or_set_maintainer_id (maintainer): @@ -397,3 +419,54 @@ def get_suites(pkgname, src=False): sql = "select suite_name from binaries, bin_associations,suite where binaries.id=bin_associations.bin and package='%s' and bin_associations.suite = suite.id"%pkgname q = projectB.query(sql) return map(lambda x: x[0], q.getresult()) + +################################################################################ + +def get_or_set_contents_file_id(file): + global content_file_id_cache + + if not content_file_id_cache.has_key(file): + sql_select = "SELECT id FROM content_file_names WHERE file = '%s'" % file + q = projectB.query(sql_select) + if not q.getresult(): + # since this can be called within a transaction, we can't use currval + q = projectB.query("SELECT nextval('content_file_names_id_seq')") + file_id = int(q.getresult()[0][0]) + projectB.query("INSERT INTO content_file_names VALUES ('%d', '%s')" % (file_id, file)) + content_file_id_cache[file] = file_id + else: + content_file_id_cache[file] = int(q.getresult()[0][0]) + return content_file_id_cache[file] + +################################################################################ + +def get_or_set_contents_path_id(path): + global content_path_id_cache + + if not content_path_id_cache.has_key(path): + sql_select = "SELECT id FROM content_file_paths WHERE path = '%s'" % path + q = projectB.query(sql_select) + if not q.getresult(): + # since this can be called within a transaction, we can't use currval + q = projectB.query("SELECT nextval('content_file_names_id_seq')") + path_id = int(q.getresult()[0][0]) + projectB.query("INSERT INTO content_file_paths VALUES ('%d', '%s')" % ( path_id, path)) + content_path_id_cache[path] = path_id + else: + content_path_id_cache[path] = int(q.getresult()[0][0]) + + return content_path_id_cache[path] + +################################################################################ + +def insert_content_path(bin_id, fullpath): + # split the path into basename, and pathname + (path, file) = os.path.split(fullpath) + + # Get the necessary IDs ... + file_id = get_or_set_contents_file_id(file) + path_id = get_or_set_contents_path_id(path) + + # Put them into content_assiocations + projectB.query("INSERT INTO content_associations VALUES (DEFAULT, '%d', '%d', '%d')" % (bin_id, path_id, file_id)) + return