From 04a4fe70fce62ece3b61fce34d5e6effdb4ab8b0 Mon Sep 17 00:00:00 2001 From: Mike O'Connor Date: Wed, 28 Oct 2009 16:16:58 +0000 Subject: [PATCH] merge from master with sqla --- dak/contents.py | 487 ++++++++++++++++++++++++------------------ dak/dakdb/update17.py | 310 +++++++++++++++++++++++++++ dak/update_db.py | 9 +- daklib/dbconn.py | 6 +- daklib/utils.py | 7 +- 5 files changed, 601 insertions(+), 218 deletions(-) create mode 100644 dak/dakdb/update17.py diff --git a/dak/contents.py b/dak/contents.py index 1b3f3e1f..073eb501 100755 --- a/dak/contents.py +++ b/dak/contents.py @@ -40,14 +40,18 @@ import logging import math import gzip import threading +import traceback import Queue import apt_pkg +import datetime #just for debugging, can be removed from daklib import utils from daklib.binary import Binary from daklib.config import Config from daklib.dbconn import DBConn ################################################################################ +log=None + def usage (exit_code=0): print """Usage: dak contents [options] command [arguments] @@ -83,7 +87,7 @@ OPTIONS options_prefix = "Contents" options_prefix = "%s::Options" % options_prefix -log = logging.getLogger() +#log = logging.getLogger() ################################################################################ @@ -103,54 +107,23 @@ debs_q = """PREPARE debs_q(int, int) as WHERE suite = $1 AND arch = $2""" -# ask if we already have contents associated with this binary -olddeb_q = """PREPARE olddeb_q(int) as - SELECT 1 FROM content_associations - WHERE binary_pkg = $1 - LIMIT 1""" - # find me all of the contents for a given .deb contents_q = """PREPARE contents_q(int,int) as - SELECT (p.path||'/'||n.file) AS fn, - s.section, - b.package, - b.architecture - FROM content_associations c join content_file_paths p ON (c.filepath=p.id) - JOIN content_file_names n ON (c.filename=n.id) - JOIN binaries b ON (b.id=c.binary_pkg) - JOIN override o ON (o.package=b.package) - JOIN section s ON (s.id=o.section) - WHERE o.suite = $1 AND o.type = $2 - AND b.type='deb' - ORDER BY fn""" - + SELECT file, section, package + FROM deb_contents + WHERE suite = $1 + AND (arch = $2 or arch=2)""" +# ORDER BY file""" + # find me all of the contents for a given .udeb -udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as - SELECT (p.path||'/'||n.file) AS fn, - s.section, - b.package, - b.architecture - FROM content_associations c join content_file_paths p ON (c.filepath=p.id) - JOIN content_file_names n ON (c.filename=n.id) - JOIN binaries b ON (b.id=c.binary_pkg) - JOIN override o ON (o.package=b.package) - JOIN section s ON (s.id=o.section) - WHERE o.suite = $1 AND o.type = $2 - AND s.id = $3 - AND b.type='udeb' - ORDER BY fn""" - -# FROM content_file_paths p join content_associations c ON (c.filepath=p.id) -# JOIN content_file_names n ON (c.filename=n.id) -# JOIN binaries b ON (b.id=c.binary_pkg) -# JOIN override o ON (o.package=b.package) -# JOIN section s ON (s.id=o.section) -# WHERE o.suite = $1 AND o.type = $2 -# AND s.id = $3 -# AND b.id in (SELECT ba.bin from bin_associations ba join binaries b on b.id=ba.bin where (b.architecture=$3 or b.architecture=$4)and ba.suite=$1 and b.type='udeb') -# GROUP BY fn -# ORDER BY fn;""" - +udeb_contents_q = """PREPARE udeb_contents_q(int,int,text, int) as + SELECT file, section, package, arch + FROM udeb_contents + WHERE suite = $1 + AND otype = $2 + AND section = $3 + and arch = $4 + ORDER BY file""" # clear out all of the temporarily stored content associations @@ -159,121 +132,213 @@ udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as # so there should no longer be anything in the queue remove_pending_contents_cruft_q = """DELETE FROM pending_content_associations""" -# delete any filenames we are storing which have no binary associated with them -remove_filename_cruft_q = """DELETE FROM content_file_names - WHERE id IN (SELECT cfn.id FROM content_file_names cfn - LEFT JOIN content_associations ca - ON ca.filename=cfn.id - WHERE ca.id IS NULL)""" - -# delete any paths we are storing which have no binary associated with them -remove_filepath_cruft_q = """DELETE FROM content_file_paths - WHERE id IN (SELECT cfn.id FROM content_file_paths cfn - LEFT JOIN content_associations ca - ON ca.filepath=cfn.id - WHERE ca.id IS NULL)""" - class EndOfContents(object): - """ - A sentry object for the end of the filename stream - """ pass -class GzippedContentWriter(object): +class OneAtATime(object): """ - An object which will write contents out to a Contents-$arch.gz - file on a separate thread """ + def __init__(self): + self.next_in_line = None + self.next_lock = threading.Condition() + + def enqueue(self, next): + self.next_lock.acquire() + while self.next_in_line: + self.next_lock.wait() + + assert( not self.next_in_line ) + self.next_in_line = next + self.next_lock.notify() + self.next_lock.release() + + def dequeue(self): + self.next_lock.acquire() + while not self.next_in_line: + self.next_lock.wait() + result = self.next_in_line + self.next_in_line = None + self.next_lock.notify() + self.next_lock.release() + return result + + +class ContentsWorkThread(threading.Thread): + """ + """ + def __init__(self, upstream, downstream): + threading.Thread.__init__(self) + self.upstream = upstream + self.downstream = downstream - header = None # a class object holding the header section of contents file + def run(self): + while True: + try: + contents_file = self.upstream.dequeue() + if isinstance(contents_file,EndOfContents): + if self.downstream: + self.downstream.enqueue(contents_file) + break + + s = datetime.datetime.now() + print("%s start: %s" % (self,contents_file) ) + self._run(contents_file) + print("%s finished: %s in %d seconds" % (self, contents_file, (datetime.datetime.now()-s).seconds )) + if self.downstream: + self.downstream.enqueue(contents_file) + except: + traceback.print_exc() + +class QueryThread(ContentsWorkThread): + def __init__(self, upstream, downstream): + ContentsWorkThread.__init__(self, upstream, downstream) + + def __str__(self): + return "QueryThread" + __repr__ = __str__ + + def _run(self, contents_file): + contents_file.query() + +class IngestThread(ContentsWorkThread): + def __init__(self, upstream, downstream): + ContentsWorkThread.__init__(self, upstream, downstream) + + def __str__(self): + return "IngestThread" + __repr__ = __str__ + + def _run(self, contents_file): + contents_file.ingest() + +class SortThread(ContentsWorkThread): + def __init__(self, upstream, downstream): + ContentsWorkThread.__init__(self, upstream, downstream) + + def __str__(self): + return "SortThread" + __repr__ = __str__ + + def _run(self, contents_file): + contents_file.sorted_keys = sorted(contents_file.filenames.keys()) + +class OutputThread(ContentsWorkThread): + def __init__(self, upstream, downstream): + ContentsWorkThread.__init__(self, upstream, downstream) + + def __str__(self): + return "OutputThread" + __repr__ = __str__ + + def _run(self, contents_file): + contents_file.open_file() + for fname in contents_file.sorted_keys: + contents_file.filehandle.write("%s\t%s\n" % (fname,contents_file.filenames[fname])) + contents_file.sorted_keys = None + contents_file.filenames.clear() + +class GzipThread(ContentsWorkThread): + def __init__(self, upstream, downstream): + ContentsWorkThread.__init__(self, upstream, downstream) + + def __str__(self): + return "GzipThread" + __repr__ = __str__ + + def _run(self, contents_file): + os.system("gzip -f %s" % contents_file.filename) + +class ContentFile(object): + def __init__(self, + filename, + suite_str, + suite_id, + arch_str, + arch_id): + + self.filename = filename + self.filenames = {} + self.sorted_keys = None + self.suite_str = suite_str + self.suite_id = suite_id + self.arch_str = arch_str + self.arch_id = arch_id + self.cursor = None + self.filehandle = None + + def __str__(self): + return self.filename + __repr__ = __str__ + + + def cleanup(self): + self.filenames = None + self.sortedkeys = None + self.filehandle.close() + self.cursor.close() + + def query(self): + self.cursor = DBConn().cursor(); + + self.cursor.execute("""SELECT file, section || '/' || package + FROM deb_contents + WHERE ( arch=2 or arch = %d) AND suite = %d + """ % (self.arch_id, self.suite_id)) + + def ingest(self): + while True: + r = self.cursor.fetchone() + if not r: + break + filename, package = r + if self.filenames.has_key(filename): + self.filenames[filename] += ",%s" % (package) + else: + self.filenames[filename] = "%s" % (package) + self.cursor.close() - def __init__(self, filename): - """ - @type filename: string - @param filename: the name of the file to write to - """ - self.queue = Queue.Queue() - self.current_file = None - self.first_package = True - self.output = self.open_file(filename) - self.thread = threading.Thread(target=self.write_thread, - name='Contents writer') - self.thread.start() - - def open_file(self, filename): + def open_file(self): """ opens a gzip stream to the contents file """ - filepath = Config()["Contents::Root"] + filename - filedir = os.path.dirname(filepath) +# filepath = Config()["Contents::Root"] + self.filename + self.filename = "/home/stew/contents/" + self.filename + filedir = os.path.dirname(self.filename) if not os.path.isdir(filedir): os.makedirs(filedir) - return gzip.open(filepath, "w") - - def write(self, filename, section, package): - """ - enqueue content to be written to the file on a separate thread - """ - self.queue.put((filename,section,package)) - - def write_thread(self): - """ - the target of a Thread which will do the actual writing - """ - while True: - next = self.queue.get() - if isinstance(next, EndOfContents): - self.output.write('\n') - self.output.close() - break - - (filename,section,package)=next - if next != self.current_file: - # this is the first file, so write the header first - if not self.current_file: - self.output.write(self._getHeader()) - - self.output.write('\n%s\t' % filename) - self.first_package = True +# self.filehandle = gzip.open(self.filename, "w") + self.filehandle = open(self.filename, "w") + self._write_header() - self.current_file=filename + def _write_header(self): + self._get_header(); + self.filehandle.write(ContentFile.header) - if not self.first_package: - self.output.write(',') - else: - self.first_package=False - self.output.write('%s/%s' % (section,package)) - - def finish(self): - """ - enqueue the sentry object so that writers will know to terminate - """ - self.queue.put(EndOfContents()) + header=None @classmethod - def _getHeader(self): + def _get_header(self): """ Internal method to return the header for Contents.gz files This is boilerplate which explains the contents of the file and how it can be used. """ - if not GzippedContentWriter.header: + if not ContentFile.header: if Config().has_key("Contents::Header"): try: h = open(os.path.join( Config()["Dir::Templates"], Config()["Contents::Header"] ), "r") - GzippedContentWriter.header = h.read() + ContentFile.header = h.read() h.close() except: log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"], traceback.format_exc() )) - GzippedContentWriter.header = None + ContentFile.header = None else: - GzippedContentWriter.header = None - - return GzippedContentWriter.header + ContentFile.header = None + return ContentFile.header class Contents(object): """ @@ -312,7 +377,6 @@ class Contents(object): cursor = DBConn().cursor(); DBConn().prepare("debs_q",debs_q) - DBConn().prepare("olddeb_q",olddeb_q) DBConn().prepare("arches_q",arches_q) suites = self._suites() @@ -331,76 +395,92 @@ class Contents(object): break count += 1 cursor1 = DBConn().cursor(); - cursor1.execute( "EXECUTE olddeb_q(%d)" % (deb[0] ) ) + cursor1.execute( "SELECT 1 FROM deb_contents WHERE binary_id = %d LIMIT 1" % (deb[0] ) ) old = cursor1.fetchone() if old: - log.debug( "already imported: %s" % (deb[1]) ) + log.log( "already imported: %s" % (deb[1]) ) else: - log.debug( "scanning: %s" % (deb[1]) ) +# log.debug( "scanning: %s" % (deb[1]) ) + log.log( "scanning: %s" % (deb[1]) ) debfile = os.path.join( pooldir, deb[1] ) if os.path.exists( debfile ): - Binary(debfile, self.reject).scan_package(deb[0],True) + Binary(debfile, self.reject).scan_package(deb[0], True) else: log.error("missing .deb: %s" % deb[1]) + def generate(self): """ - Generate Contents-$arch.gz files for every available arch in each given suite. + Generate contents files for both deb and udeb """ - cursor = DBConn().cursor() - DBConn().prepare("arches_q", arches_q) - DBConn().prepare("contents_q", contents_q) - DBConn().prepare("udeb_contents_q", udeb_contents_q) - - debtype_id=DBConn().get_override_type_id("deb") - udebtype_id=DBConn().get_override_type_id("udeb") + self.deb_generate() +# self.udeb_generate() - arch_all_id = DBConn().get_architecture_id("all") + def deb_generate(self): + """ + Generate Contents-$arch.gz files for every available arch in each given suite. + """ + cursor = DBConn().cursor() + debtype_id = DBConn().get_override_type_id("deb") suites = self._suites() - + inputtoquery = OneAtATime() + querytoingest = OneAtATime() + ingesttosort = OneAtATime() + sorttooutput = OneAtATime() + outputtogzip = OneAtATime() + + qt = QueryThread(inputtoquery,querytoingest) + it = IngestThread(querytoingest,ingesttosort) +# these actually make things worse +# it2 = IngestThread(querytoingest,ingesttosort) +# it3 = IngestThread(querytoingest,ingesttosort) +# it4 = IngestThread(querytoingest,ingesttosort) + st = SortThread(ingesttosort,sorttooutput) + ot = OutputThread(sorttooutput,outputtogzip) + gt = GzipThread(outputtogzip, None) + + qt.start() + it.start() +# it2.start() +# it3.start() +# it2.start() + st.start() + ot.start() + gt.start() + # Get our suites, and the architectures for suite in [i.lower() for i in suites]: suite_id = DBConn().get_suite_id(suite) arch_list = self._arches(cursor, suite_id) - file_writers = {} - - try: - for arch_id in arch_list: - file_writers[arch_id[0]] = GzippedContentWriter("dists/%s/Contents-%s.gz" % (suite, arch_id[1])) - - cursor.execute("EXECUTE contents_q(%d,%d);" % (suite_id, debtype_id)) + for (arch_id,arch_str) in arch_list: + print( "suite: %s, arch: %s time: %s" %(suite_id, arch_id, datetime.datetime.now().isoformat()) ) - while True: - r = cursor.fetchone() - if not r: - break - - filename, section, package, arch = r - - if not file_writers.has_key( arch ): - continue +# filename = "dists/%s/Contents-%s.gz" % (suite, arch_str) + filename = "dists/%s/Contents-%s" % (suite, arch_str) + cf = ContentFile(filename, suite, suite_id, arch_str, arch_id) + inputtoquery.enqueue( cf ) - if arch == arch_all_id: - ## its arch all, so all contents files get it - for writer in file_writers.values(): - writer.write(filename, section, package) + inputtoquery.enqueue( EndOfContents() ) + gt.join() - else: - file_writers[arch].write(filename, section, package) + def udeb_generate(self): + """ + Generate Contents-$arch.gz files for every available arch in each given suite. + """ + cursor = DBConn().cursor() - finally: - # close all the files - for writer in file_writers.values(): - writer.finish() + DBConn().prepare("udeb_contents_q", udeb_contents_q) + udebtype_id=DBConn().get_override_type_id("udeb") + suites = self._suites() +# for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"), +# ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]: - # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free) - # This is HORRIBLY debian specific :-/ - for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"), - ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]: + for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s"), + ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s")]: section_id = DBConn().get_section_id(section) # all udebs should be here) if section_id != -1: @@ -410,35 +490,22 @@ class Contents(object): suite_id = DBConn().get_suite_id(suite) arch_list = self._arches(cursor, suite_id) - file_writers = {} + for arch_id in arch_list: - try: - for arch_id in arch_list: - file_writers[arch_id[0]] = GzippedContentWriter(fn_pattern % (suite, arch_id[1])) + writer = GzippedContentWriter(fn_pattern % (suite, arch_id[1])) + try: - cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id)) + cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id, arch_id)) - while True: - r = cursor.fetchone() - if not r: - break + while True: + r = cursor.fetchone() + if not r: + break - filename, section, package, arch = r - - if not file_writers.has_key( arch ): - continue - - if arch == arch_all_id: - ## its arch all, so all contents files get it - for writer in file_writers.values(): - writer.write(filename, section, package) - - else: - file_writers[arch].write(filename, section, package) - finally: - # close all the files - for writer in file_writers.values(): - writer.finish() + filename, section, package, arch = r + writer.write(filename, section, package) + finally: + writer.close() @@ -451,7 +518,8 @@ class Contents(object): if Config().has_key( "%s::%s" %(options_prefix,"Suite")): suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")]) else: - suites = Config().SubTree("Suite").List() + suites = [ 'unstable', 'testing' ] +# suites = Config().SubTree("Suite").List() return suites @@ -476,7 +544,8 @@ class Contents(object): def main(): cnf = Config() - +# log = logging.Logger(cnf, "contents") + arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")), ('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"), ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")), @@ -496,17 +565,17 @@ def main(): if cnf.has_key("%s::%s" % (options_prefix,"Help")): usage() - level=logging.INFO - if cnf.has_key("%s::%s" % (options_prefix,"Quiet")): - level=logging.ERROR +# level=logging.INFO +# if cnf.has_key("%s::%s" % (options_prefix,"Quiet")): +# level=logging.ERROR - elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")): - level=logging.DEBUG +# elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")): +# level=logging.DEBUG - logging.basicConfig( level=level, - format='%(asctime)s %(levelname)s %(message)s', - stream = sys.stderr ) +# logging.basicConfig( level=level, +# format='%(asctime)s %(levelname)s %(message)s', +# stream = sys.stderr ) commands[args[0]](Contents()) diff --git a/dak/dakdb/update17.py b/dak/dakdb/update17.py new file mode 100644 index 00000000..a3e134af --- /dev/null +++ b/dak/dakdb/update17.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Adding a trainee field to the process-new notes + +@contact: Debian FTP Master +@copyright: 2009 Mike O'Connor +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + + +################################################################################ + +import psycopg2 +import time +from daklib.dak_exceptions import DBUpdateError + +################################################################################ + +def suites(): + """ + return a list of suites to operate on + """ + if Config().has_key( "%s::%s" %(options_prefix,"Suite")): + suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")]) + else: + suites = [ 'unstable', 'testing' ] +# suites = Config().SubTree("Suite").List() + + return suites + +def arches(cursor, suite): + """ + return a list of archs to operate on + """ + arch_list = [] + cursor.execute("EXECUTE arches_q(%d)" % (suite)) + while True: + r = cursor.fetchone() + if not r: + break + + if r[1] != "source" and r[1] != "all": + arch_list.append((r[0], r[1])) + + return arch_list + +def do_update(self): + """ + Adding contents table as first step to maybe, finally getting rid + of apt-ftparchive + """ + + print __doc__ + + try: + c = self.db.cursor() + c.execute("""CREATE TABLE deb_contents ( + file text, + section text, + package text, + binary_id integer, + arch integer, + suite integer, + component integer)""" ) + + c.execute("""CREATE TABLE udeb_contents ( + file text, + section text, + package text, + binary_id integer, + suite integer, + component integer )""" ) + + c.execute("""ALTER TABLE ONLY deb_contents + ADD CONSTRAINT deb_contents_arch_fkey + FOREIGN KEY (arch) REFERENCES architecture(id) + ON DELETE CASCADE;""") + + c.execute("""ALTER TABLE ONLY udeb_contents + ADD CONSTRAINT udeb_contents_arch_fkey + FOREIGN KEY (arch) REFERENCES architecture(id) + ON DELETE CASCADE;""") + + c.execute("""ALTER TABLE ONLY deb_contents + ADD CONSTRAINT deb_contents_suite_fkey + FOREIGN KEY (suite) REFERENCES suite(id) + ON DELETE CASCADE;""") + + c.execute("""ALTER TABLE ONLY udeb_contents + ADD CONSTRAINT udeb_contents_suite_fkey + FOREIGN KEY (suite) REFERENCES suite(id) + ON DELETE CASCADE;""") + + c.execute("""ALTER TABLE ONLY deb_contents + ADD CONSTRAINT deb_contents_binary_fkey + FOREIGN KEY (binary_id) REFERENCES binaries(id) + ON DELETE CASCADE;""") + + c.execute("""ALTER TABLE ONLY udeb_contents + ADD CONSTRAINT udeb_contents_binary_fkey + FOREIGN KEY (binary_id) REFERENCES binaries(id) + ON DELETE CASCADE;""") + + c.execute("""CREATE INDEX ind_deb_contents_binary ON deb_contents(binary_id);""" ) + + arches_q = """PREPARE arches_q(int) as + SELECT s.architecture, a.arch_string + FROM suite_architectures s + JOIN architecture a ON (s.architecture=a.id) + WHERE suite = $1""" + + suites = self.suites() + + for suite in [i.lower() for i in suites]: + suite_id = DBConn().get_suite_id(suite) + arch_list = arches(c, suite_id) + arch_list = arches(c, suite_id) + + for (arch_id,arch_str) in arch_list: + c.execute( "CREATE INDEX ind_deb_contents_%s_%s ON deb_contents (arch,suite) WHERE (arch=2 OR arch=%d) AND suite=$d"%(arch_str,suite,arch_id,suite_id) ) + + for section, sname in [("debian-installer","main"), + ("non-free/debian-installer", "nonfree")]: + c.execute( "CREATE INDEX ind_udeb_contents_%s_%s ON udeb_contents (section,suite) WHERE section=%s AND suite=$d"%(sname,suite,section,suite_id) ) + + + Column | Type | Modifiers + ------------+---------+----------- + package | text | not null + suite | integer | not null + component | integer | not null + priority | integer | + section | integer | not null + type | integer | not null + maintainer | text | + + c.execute("""CREATE TABLE deb_contents ( + file text, + section text, + package text, + binary_id integer, + arch integer, + suite integer, + component integer)""" ) + + +CREATE FUNCTION update_contents_for_override() RETURNS trigger AS $update_contents_for_override$ +BEGIN + UPDATE deb_contents SET section=NEW.section, component=NEW.component + WHERE deb_contents.package=OLD.package + + +DELETE FROM +NEW.last_date := current_timestamp; +NEW.last_user := current_user; +RETURN NEW; +END; +$update_contents_for_override$ LANGUAGE plpgsql; + + + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply process-new update 14, rollback issued. Error message : %s" % (str(msg)) +""" + INSERT INTO deb_contents SELECT (p.path||'/'||n.file) AS file, + s.section AS section, + b.package AS package, + b.id AS binary_id, + b.architecture AS arch, + o.suite AS suited, + o.component AS componentd, + o.type AS otype_id + FROM content_associations c + JOIN content_file_paths p ON (c.filepath=p.id) + JOIN content_file_names n ON (c.filename=n.id) + JOIN binaries b ON (b.id=c.binary_pkg) + JOIN architecture a ON (b.architecture = a.id) + JOIN override o ON (o.package=b.package) + JOIN bin_associations ba on ba.suite=o.suite and ba.bin=b.id + JOIN section s ON (s.id=o.section) + where b.type='deb'; + + INSERT INTO udeb_contents SELECT (p.path||'/'||n.file) AS file, + s.section AS section, + b.package AS package, + b.id AS binary_id, + b.architecture AS arch, + o.suite AS suited, + o.component AS componentd, + o.type AS otype_id + FROM content_associations c + JOIN content_file_paths p ON (c.filepath=p.id) + JOIN content_file_names n ON (c.filename=n.id) + JOIN binaries b ON (b.id=c.binary_pkg) + JOIN architecture a ON (b.architecture = a.id) + JOIN override o ON (o.package=b.package) + JOIN section s ON (s.id=o.section) + where b.type='udeb' +""" + +""" +CREATE INDEX ind_archid ON contents(arch); +CREATE INDEX ind_archid_amd64 ON contents(arch) WHERE arch=16; +CREATE INDEX ind_suite ON contents(suite); +CREATE INDEX ind_suite_unstable ON contents(suite) WHERE suite=5; +CREATE INDEX ind_overridetype ON contents(otype); +CREATE INDEX ind_overridetype_deb ON contents(otype) WHERE otype=7; +CREATE INDEX ind_packagetype ON contents(packagetype); +CREATE INDEX ind_packagetype_deb ON contents(packagetype) WHERE packagetype='deb'; +CREATE INDEX ind_package ON contents(package); + + CREATE INDEX ind_suite_otype ON contents(suite, otype) WHERE suite=5 AND otype=7; + CREATE INDEX ind_suite_otype_arch ON contents(suite, otype, arch) WHERE suite=5 AND otype=7 AND arch=16; + CREATE INDEX ind_suite_otype_package ON contents(suite, otype, packagetype) WHERE suite=5 AND otype=7 AND packagetype='deb'; + CREATE INDEX ind_suite_otype_package_notdeb ON contents(suite, otype, packagetype) WHERE suite=5 AND otype=7 AND packagetype!='deb'; + """ + +CREATE INDEX ind_deb_contents_binary ON deb_contents(binary_id); + +CREATE INDEX ind_deb_contents_arch_alpha_unstable ON deb_contents(arch) where (arch=2 or arch=3) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hurd_i386_unstable ON deb_contents(arch) where (arch=2 or arch=4) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hppa_unstable ON deb_contents(arch) where (arch=2 or arch=5) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_arm_unstable ON deb_contents(arch) where (arch=2 or arch=6) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_i386_unstable ON deb_contents(arch) where (arch=2 or arch=7) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_m68k_unstable ON deb_contents(arch) where (arch=2 or arch=8) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mips_unstable ON deb_contents(arch) where (arch=2 or arch=9) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mipsel_unstable ON deb_contents(arch) where (arch=2 or arch=10) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_powerpc_unstable ON deb_contents(arch) where (arch=2 or arch=11) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sh_unstable ON deb_contents(arch) where (arch=2 or arch=12) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sparc_unstable ON deb_contents(arch) where (arch=2 or arch=13) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_s390_unstable ON deb_contents(arch) where (arch=2 or arch=14) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_ia64_unstable ON deb_contents(arch) where (arch=2 or arch=15) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_amd64_unstable ON deb_contents(arch) where (arch=2 or arch=16) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_armel_unstable ON deb_contents(arch) where (arch=2 or arch=17) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_unstable ON deb_contents(arch) where (arch=2 or arch=25) AND suite=5 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_unstable ON deb_contents(arch) where (arch=2 or arch=26) AND suite=5 AND otype=7; + +CREATE INDEX ind_deb_contents_arch_alpha_stable ON deb_contents(arch) where (arch=2 or arch=3) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hurd_i386_stable ON deb_contents(arch) where (arch=2 or arch=4) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hppa_stable ON deb_contents(arch) where (arch=2 or arch=5) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_arm_stable ON deb_contents(arch) where (arch=2 or arch=6) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_i386_stable ON deb_contents(arch) where (arch=2 or arch=7) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_m68k_stable ON deb_contents(arch) where (arch=2 or arch=8) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mips_stable ON deb_contents(arch) where (arch=2 or arch=9) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mipsel_stable ON deb_contents(arch) where (arch=2 or arch=10) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_powerpc_stable ON deb_contents(arch) where (arch=2 or arch=11) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sh_stable ON deb_contents(arch) where (arch=2 or arch=12) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sparc_stable ON deb_contents(arch) where (arch=2 or arch=13) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_s390_stable ON deb_contents(arch) where (arch=2 or arch=14) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_ia64_stable ON deb_contents(arch) where (arch=2 or arch=15) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_amd64_stable ON deb_contents(arch) where (arch=2 or arch=16) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_armel_stable ON deb_contents(arch) where (arch=2 or arch=17) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_stable ON deb_contents(arch) where (arch=2 or arch=25) AND suite=2 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_stable ON deb_contents(arch) where (arch=2 or arch=26) AND suite=2 AND otype=7; + +CREATE INDEX ind_deb_contents_arch_alpha_testing ON deb_contents(arch) where (arch=2 or arch=3) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hurd_i386_testing ON deb_contents(arch) where (arch=2 or arch=4) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hppa_testing ON deb_contents(arch) where (arch=2 or arch=5) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_arm_testing ON deb_contents(arch) where (arch=2 or arch=6) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_i386_testing ON deb_contents(arch) where (arch=2 or arch=7) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_m68k_testing ON deb_contents(arch) where (arch=2 or arch=8) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mips_testing ON deb_contents(arch) where (arch=2 or arch=9) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mipsel_testing ON deb_contents(arch) where (arch=2 or arch=10) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_powerpc_testing ON deb_contents(arch) where (arch=2 or arch=11) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sh_testing ON deb_contents(arch) where (arch=2 or arch=12) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sparc_testing ON deb_contents(arch) where (arch=2 or arch=13) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_s390_testing ON deb_contents(arch) where (arch=2 or arch=14) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_ia64_testing ON deb_contents(arch) where (arch=2 or arch=15) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_amd64_testing ON deb_contents(arch) where (arch=2 or arch=16) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_armel_testing ON deb_contents(arch) where (arch=2 or arch=17) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_testing ON deb_contents(arch) where (arch=2 or arch=25) AND suite=4 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_testing ON deb_contents(arch) where (arch=2 or arch=26) AND suite=4 AND otype=7; + +CREATE INDEX ind_deb_contents_arch_alpha_oldstable ON deb_contents(arch) where (arch=2 or arch=3) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hurd_i386_oldstable ON deb_contents(arch) where (arch=2 or arch=4) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_hppa_oldstable ON deb_contents(arch) where (arch=2 or arch=5) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_arm_oldstable ON deb_contents(arch) where (arch=2 or arch=6) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_i386_oldstable ON deb_contents(arch) where (arch=2 or arch=7) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_m68k_oldstable ON deb_contents(arch) where (arch=2 or arch=8) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mips_oldstable ON deb_contents(arch) where (arch=2 or arch=9) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_mipsel_oldstable ON deb_contents(arch) where (arch=2 or arch=10) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_powerpc_oldstable ON deb_contents(arch) where (arch=2 or arch=11) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sh_oldstable ON deb_contents(arch) where (arch=2 or arch=12) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_sparc_oldstable ON deb_contents(arch) where (arch=2 or arch=13) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_s390_oldstable ON deb_contents(arch) where (arch=2 or arch=14) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_ia64_oldstable ON deb_contents(arch) where (arch=2 or arch=15) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_amd64_oldstable ON deb_contents(arch) where (arch=2 or arch=16) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_armel_oldstable ON deb_contents(arch) where (arch=2 or arch=17) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_oldstable ON deb_contents(arch) where (arch=2 or arch=25) AND suite=14 AND otype=7; +CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_oldstable ON deb_contents(arch) where (arch=2 or arch=26) AND suite=14 AND otype=7; diff --git a/dak/update_db.py b/dak/update_db.py index 5fe6918a..e197edd6 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError Cnf = None projectB = None -required_database_schema = 13 +required_database_schema = 14 ################################################################################ @@ -110,9 +110,10 @@ Updates dak's database schema to the lastest version. You should disable crontab try: # Build a connect string - connect_str = "dbname=%s"% (Cnf["DB::Name"]) - if Cnf["DB::Host"] != '': connect_str += " host=%s" % (Cnf["DB::Host"]) - if Cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(Cnf["DB::Port"])) +# connect_str = "dbname=%s"% (Cnf["DB::Name"]) + connect_str = "dbname=%s"% "projectbstew" +# if Cnf["DB::Host"] != '': connect_str += " host=%s" % (Cnf["DB::Host"]) +# if Cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(Cnf["DB::Port"])) self.db = psycopg2.connect(connect_str) diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 58dd7fc5..a1ac7038 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -73,7 +73,8 @@ class DBConn(Singleton): ## Connection functions def __createconn(self): cnf = Config() - connstr = "dbname=%s" % cnf["DB::Name"] + connstr = "dbname=%s" % "projectbstew" #cnf["DB::Name"] + print( "connstr: %s "% connstr) if cnf["DB::Host"]: connstr += " host=%s" % cnf["DB::Host"] if cnf["DB::Port"] and cnf["DB::Port"] != "-1": @@ -524,7 +525,8 @@ class DBConn(Singleton): file_id = self.get_or_set_contents_file_id(file) path_id = self.get_or_set_contents_path_id(path) - c.execute("""INSERT INTO content_associations + c.execute("""INSERT INTO deb_contents + (binary_pkg, filepath, filename) VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) ) diff --git a/daklib/utils.py b/daklib/utils.py index 073b8133..04570c95 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -48,7 +48,8 @@ from regexes import re_html_escaping, html_escaping, re_single_line_field, \ ################################################################################ -default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties +#default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties +default_config = "/home/stew/etc/dak/dak.conf" #: default dak config, defines host properties default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used alias_cache = None #: Cache for email alias checks @@ -1558,7 +1559,7 @@ apt_pkg.init() Cnf = apt_pkg.newConfiguration() apt_pkg.ReadConfigFileISC(Cnf,default_config) -if which_conf_file() != default_config: - apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) +#if which_conf_file() != default_config: +# apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) ############################################################################### -- 2.39.2