From b7e5c9a6deda1018da9023d1d01acec86e165ba4 Mon Sep 17 00:00:00 2001 From: Mike O'Connor Date: Sat, 14 Mar 2009 10:50:54 -0400 Subject: [PATCH] fix things I broke yesterday with the contents merge * alter the pending_content_associations table to include arch * only allow debs with more than 3 ar members during bootstrap Signed-off-by: Mike O'Connor --- dak/contents.py | 4 +-- dak/dakdb/update9.py | 61 ++++++++++++++++++++++++++++++++++++++ dak/process_accepted.py | 2 +- dak/process_unchecked.py | 2 +- daklib/.#dbconn.py | 1 + daklib/binary.py | 22 ++++++++------ daklib/database.py | 16 +++++++--- daklib/dbconn.py | 11 ++++--- templates/missing-contents | 8 ++--- 9 files changed, 102 insertions(+), 25 deletions(-) create mode 100644 dak/dakdb/update9.py create mode 120000 daklib/.#dbconn.py diff --git a/dak/contents.py b/dak/contents.py index 1efb361f..ebeb2bc7 100755 --- a/dak/contents.py +++ b/dak/contents.py @@ -283,9 +283,9 @@ class Contents(object): log.debug( "scanning: %s" % (deb[1]) ) debfile = os.path.join( pooldir, deb[1] ) if os.path.exists( debfile ): - Binary(debfile, self.reject).scan_package( deb[0] ) + Binary(debfile, self.reject).scan_package(deb[0],True) else: - log.error( "missing .deb: %s" % deb[1] ) + log.error("missing .deb: %s" % deb[1]) def generate(self): """ diff --git a/dak/dakdb/update9.py b/dak/dakdb/update9.py new file mode 100644 index 00000000..09785773 --- /dev/null +++ b/dak/dakdb/update9.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Debian Archive Kit Database Update Script +Copyright © 2008 Michael Casadevall +Copyright © 2009 Mike O'Connor + +Debian Archive Kit Database Update Script 8 +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# * Ganneff ponders how to best write the text to -devel. (need to tell em in +# case they find more bugs). "We fixed the fucking idiotic broken implementation +# to be less so" is probably not the nicest, even if perfect valid, way to say so + +################################################################################ + +import psycopg2 +import time +from daklib.dak_exceptions import DBUpdateError +from daklib.utils import get_conf + +################################################################################ + +def do_update(self): + print "pending_contents should distinguish by arch" + Cnf = get_conf() + + try: + c = self.db.cursor() + + c.execute("DELETE FROM pending_content_associations") + c.execute("""ALTER TABLE pending_content_associations + ADD COLUMN architecture integer NOT NULL""") + c.execute("""ALTER TABLE ONLY pending_content_associations + ADD CONSTRAINT pending_content_assiciations_arch + FOREIGN KEY (architecture) + REFERENCES architecture(id) + ON DELETE CASCADE""") + c.execute("UPDATE config SET value = '9' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)) diff --git a/dak/process_accepted.py b/dak/process_accepted.py index 97c5d0d5..e55ac54d 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -390,7 +390,7 @@ def install (): suite_id = database.get_suite_id(suite) projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id)) - if not database.copy_temporary_contents(package, version, newfile, reject): + if not database.copy_temporary_contents(package, version, architecture, newfile, reject): print "REJECT\n" + reject_message, projectB.query("ROLLBACK") raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (package, newfile ) diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index 938f839c..5bd8bf2c 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -562,7 +562,7 @@ def check_files(): # Check the version and for file overwrites reject(Upload.check_binary_against_db(f),"") - Binary(f, reject).scan_package( ) + Binary(f, reject).scan_package() # Checks for a source package... else: diff --git a/daklib/.#dbconn.py b/daklib/.#dbconn.py new file mode 120000 index 00000000..ed69a0a0 --- /dev/null +++ b/daklib/.#dbconn.py @@ -0,0 +1 @@ +stew@tang.vireo.org.14091:1236689528 \ No newline at end of file diff --git a/daklib/binary.py b/daklib/binary.py index 1f630a39..2e8028f2 100755 --- a/daklib/binary.py +++ b/daklib/binary.py @@ -127,7 +127,7 @@ class Binary(object): finally: os.chdir( cwd ) - def valid_deb(self): + def valid_deb(self, relaxed=False): """ Check deb contents making sure the .deb contains: 1. debian-binary @@ -137,9 +137,14 @@ class Binary(object): """ self.__scan_ar() rejected = not self.chunks - if len(self.chunks) < 3: - rejected = True - self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks))) + if relaxed: + if en(self.chunks) < 3: + rejected = True + self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks))) + else + if en(self.chunks) != 3: + rejected = True + self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks))) if self.chunks[0] != "debian-binary": rejected = True self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0])) @@ -152,7 +157,7 @@ class Binary(object): return not rejected - def scan_package(self, bootstrap_id=0): + def scan_package(self, bootstrap_id=0, relaxed=False): """ Unpack the .deb, do sanity checking, and gather info from it. @@ -168,7 +173,7 @@ class Binary(object): @return True if the deb is valid and contents were imported """ result = False - rejected = not self.valid_deb() + rejected = not self.valid_deb(relaxed) if not rejected: self.__unpack() @@ -196,6 +201,7 @@ class Binary(object): traceback.print_exc() os.chdir(cwd) + self._cleanup() return result def check_utf8_package(self, package): @@ -213,7 +219,7 @@ class Binary(object): @return True if the deb is valid and contents were imported """ - rejected = not self.valid_deb() + rejected = not self.valid_deb(True) self.__unpack() if not rejected and self.tmpdir: @@ -240,6 +246,4 @@ class Binary(object): os.chdir(cwd) -if __name__ == "__main__": - Binary( "/srv/ftp.debian.org/queue/accepted/halevt_0.1.3-2_amd64.deb" ).scan_package() diff --git a/daklib/database.py b/daklib/database.py index 90b8e6da..49c18f9e 100755 --- a/daklib/database.py +++ b/daklib/database.py @@ -838,7 +838,7 @@ def get_suites(pkgname, src=False): ################################################################################ -def copy_temporary_contents(package, version, deb, reject): +def copy_temporary_contents(package, version, arch, deb, reject): """ copy the previously stored contents from the temp table to the permanant one @@ -848,8 +848,13 @@ def copy_temporary_contents(package, version, deb, reject): # first see if contents exist: + arch_id = database.get_architecture_id (architecture) + exists = projectB.query("""SELECT 1 FROM pending_content_associations - WHERE package='%s' LIMIT 1""" % package ).getresult() + WHERE package='%s' + AND version='%s' + AND architecture=%d LIMIT 1""" + % package, version, arch_id ).getresult() if not exists: # This should NOT happen. We should have added contents @@ -858,6 +863,7 @@ def copy_temporary_contents(package, version, deb, reject): subst = { "__PACKAGE__": package, "__VERSION__": version, + "__ARCH__": arch, "__TO_ADDRESS__": Cnf["Dinstall::MyAdminAddress"], "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"] } @@ -870,10 +876,12 @@ def copy_temporary_contents(package, version, deb, reject): sql = """INSERT INTO content_associations(binary_pkg,filepath,filename) SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations WHERE package='%s' - AND version='%s'""" % (package, version) + AND version='%s' + AND architecture=%d""" % (package, version, arch_id) projectB.query(sql) projectB.query("""DELETE from pending_content_associations WHERE package='%s' - AND version='%s'""" % (package, version)) + AND version='%s' + AND architecture=%d""" % (package, version, arch_id)) return exists diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 24cb9044..2c4d1a0e 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -546,11 +546,13 @@ class DBConn(Singleton): c.execute("BEGIN WORK") try: + arch_id = self.get_architecture_id(package[Architecture]) # Remove any already existing recorded files for this package c.execute("""DELETE FROM pending_content_associations WHERE package=%(Package)s - AND version=%(Version)s""", package ) + AND version=%(Version)s + AND arch_id=%d""" % arch_id, package ) for fullpath in fullpaths: (path, file) = os.path.split(fullpath) @@ -562,9 +564,10 @@ class DBConn(Singleton): path_id = self.get_or_set_contents_path_id(path) c.execute("""INSERT INTO pending_content_associations - (package, version, filepath, filename) - VALUES (%%(Package)s, %%(Version)s, '%d', '%d')""" % (path_id, file_id), - package ) + (package, version, architecture, filepath, filename) + VALUES (%%(Package)s, %%(Version)s, '%d', '%d', '%d')""" + % (arch_id, path_id, file_id), package ) + c.execute("COMMIT") return True except: diff --git a/templates/missing-contents b/templates/missing-contents index 90f16dc5..8605e022 100644 --- a/templates/missing-contents +++ b/templates/missing-contents @@ -8,8 +8,8 @@ Content-Transfer-Encoding: 8bit Subject: Missing contents for __PACKAGE__ in accepted queue While processing the accepted queue, I didn't have contents in the -database for __PACKAGE__ version __VERSION__. These contents should -have been put into the database by process-unchecked when the package -first arrived. +database for __PACKAGE__ version __VERSION__ for arch __ARCH__. These +contents should have been put into the database by process-unchecked +when the package first arrived. -This is probably stew's fault. \ No newline at end of file +This is probably stew's fault. -- 2.39.2