]> git.decadent.org.uk Git - dak.git/commitdiff
merge from master
authorMike O'Connor <stew@vireo.org>
Fri, 13 Mar 2009 13:30:41 +0000 (09:30 -0400)
committerMike O'Connor <stew@vireo.org>
Fri, 13 Mar 2009 13:30:41 +0000 (09:30 -0400)
Signed-off-by: Mike O'Connor <stew@vireo.org>
dak/contents.py
dak/process_unchecked.py
daklib/binary.py
daklib/dbconn.py

index 3444071cdb34a9860f5f62f5ae7c0427e3830332..6215574815edc90f549c5d1422584e875dbcc58a 100644 (file)
@@ -91,14 +91,14 @@ log = logging.getLogger()
 
 # get all the arches delivered for a given suite
 # this should probably exist somehere common
-arches_q = """PREPARE arches_q as
+arches_q = """PREPARE arches_q(int) as
               SELECT s.architecture, a.arch_string
               FROM suite_architectures s
               JOIN architecture a ON (s.architecture=a.id)
                   WHERE suite = $1"""
 
 # find me the .deb for a given binary id
-debs_q = """PREPARE debs_q as
+debs_q = """PREPARE debs_q(int, int) as
               SELECT b.id, f.filename FROM bin_assoc_by_arch baa
               JOIN binaries b ON baa.bin=b.id
               JOIN files f ON b.file=f.id
@@ -106,13 +106,13 @@ debs_q = """PREPARE debs_q as
                   AND arch = $2"""
 
 # ask if we already have contents associated with this binary
-olddeb_q = """PREPARE olddeb_q as
+olddeb_q = """PREPARE olddeb_q(int) as
               SELECT 1 FROM content_associations
               WHERE binary_pkg = $1
               LIMIT 1"""
 
 # find me all of the contents for a given .deb
-contents_q = """PREPARE contents_q as
+contents_q = """PREPARE contents_q(int,int,int,int) as
               SELECT (p.path||'/'||n.file) AS fn,
                       comma_separated_list(s.section||'/'||b.package)
               FROM content_associations c
@@ -131,7 +131,7 @@ contents_q = """PREPARE contents_q as
               ORDER BY fn"""
 
 # find me all of the contents for a given .udeb
-udeb_contents_q = """PREPARE udeb_contents_q as
+udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as
               SELECT (p.path||'/'||n.file) as fn,
                       comma_separated_list(s.section||'/'||b.package)
               FROM content_associations c
@@ -178,6 +178,9 @@ class Contents(object):
     def __init__(self):
         self.header = None
 
+    def reject(self, message):
+        log.error("E: %s" % message)
+
     def _getHeader(self):
         """
         Internal method to return the header for Contents.gz files
@@ -226,7 +229,7 @@ class Contents(object):
                     return
 
                 num_tabs = max(1,
-                               int(math.ceil((self._goal_column - len(contents[0])) / 8)))
+                               int(math.ceil((self._goal_column - len(contents[0])-1) / 8)))
                 f.write(contents[0] + ( '\t' * num_tabs ) + contents[-1] + "\n")
 
         finally:
@@ -279,7 +282,7 @@ class Contents(object):
                     else:
                         debfile = os.path.join( pooldir, deb[1] )
                         if os.path.exists( debfile ):
-                            Binary(debfile).scan_package( deb[0] )
+                            Binary(debfile, self.reject).scan_package( deb[0] )
                         else:
                             log.error( "missing .deb: %s" % deb[1] )
 
index 491c5560ec7b7c968b17bbeeb37c4514f778c049..dd97b6c7252d9c9eaf323fb336402a972e19941a 100755 (executable)
@@ -400,7 +400,7 @@ def check_files():
     cursor = DBConn().cursor()
     # Check for packages that have moved from one component to another
     # STU: this should probably be changed to not join on architecture, suite tables but instead to used their cached name->id mappings from DBConn
-    cursor.execute("""PREPARE moved_pkg_q AS
+    cursor.execute("""PREPARE moved_pkg_q(text,text,text) AS
         SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
                     component c, architecture a, files f
         WHERE b.package = $1 AND s.suite_name = $2
@@ -572,7 +572,7 @@ def check_files():
             # Check the version and for file overwrites
             reject(Upload.check_binary_against_db(f),"")
 
-            Binary(f).scan_package()
+            Binary(f, reject).scan_package( )
 
         # Checks for a source package...
         else:
index 042cd266070a300f6e62644f8c709866cf6e7de1..57b0f65ddd30183f67a670733a6a5d3abe8c5ade 100755 (executable)
@@ -24,7 +24,23 @@ Functions related debian binary packages
 
 ################################################################################
 
+# <Ganneff> are we going the xorg way?
+# <Ganneff> a dak without a dak.conf?
+# <stew> automatically detect the wrong settings at runtime?
+# <Ganneff> yes!
+# <mhy> well, we'll probably always need dak.conf (how do you get the database setting
+# <mhy> but removing most of the config into the database seems sane
+# <Ganneff> mhy: dont spoil the fun
+# <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
+# <Ganneff> maybe one answers to sql
+# <stew> we will discover projectb via avahi
+# <mhy> you're both sick
+# <mhy> really fucking sick
+
+################################################################################
+
 import os
+import sys
 import shutil
 import tempfile
 import tarfile
@@ -33,12 +49,33 @@ import traceback
 import atexit
 from debian_bundle import deb822
 from dbconn import DBConn
+from config import Config
+import logging
+import utils
 
 class Binary(object):
-    def __init__(self, filename):
+    def __init__(self, filename, reject=None):
+        """
+        @ptype filename: string
+        @param filename: path of a .deb
+
+        @ptype reject: function
+        @param reject: a function to log reject messages to
+        """
         self.filename = filename
         self.tmpdir = None
         self.chunks = None
+        self.wrapped_reject = reject
+
+    def reject(self, message):
+        """
+        if we were given a reject function, send the reject message,
+        otherwise send it to stderr.
+        """
+        if self.wrapped_reject:
+            self.wrapped_reject(message)
+        else:
+            print >> sys.stderr, message
 
     def __del__(self):
         """
@@ -59,12 +96,11 @@ class Binary(object):
         if not self.chunks:
 
             cmd = "ar t %s" % (self.filename)
-
             (result, output) = commands.getstatusoutput(cmd)
             if result != 0:
                 rejected = True
-                reject("%s: 'ar t' invocation failed." % (self.filename))
-                reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
+                self.reject("%s: 'ar t' invocation failed." % (self.filename))
+                self.reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
             self.chunks = output.split('\n')
 
 
@@ -81,8 +117,8 @@ class Binary(object):
                 cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
                 (result, output) = commands.getstatusoutput(cmd)
                 if result != 0:
-                    reject("%s: '%s' invocation failed." % (filename, cmd))
-                    reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
+                    self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
+                    self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
                 else:
                     self.tmpdir = tmpdir
                     atexit.register( self._cleanup )
@@ -102,16 +138,16 @@ class Binary(object):
         rejected = not self.chunks
         if len(self.chunks) != 3:
             rejected = True
-            reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
+            self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
         if self.chunks[0] != "debian-binary":
             rejected = True
-            reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
+            self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
         if self.chunks[1] != "control.tar.gz":
             rejected = True
-            reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
+            self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
         if self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
             rejected = True
-            reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
+            self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
 
         return not rejected
 
@@ -133,32 +169,73 @@ class Binary(object):
         rejected = not self.valid_deb()
         self.__unpack()
 
+        result = False
+
+        cwd = os.getcwd()
         if not rejected and self.tmpdir:
-            cwd = os.getcwd()
             try:
                 os.chdir(self.tmpdir)
                 if self.chunks[1] == "control.tar.gz":
                     control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
-
-
+                    control.extract('./control', self.tmpdir )
                 if self.chunks[2] == "data.tar.gz":
                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
                 elif self.chunks[2] == "data.tar.bz2":
                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 
                 if bootstrap_id:
-                    return DBConn().insert_content_paths(bootstrap_id, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+                    result = DBConn().insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
                 else:
-                    pkg = deb822.Packages.iter_paragraphs( control.extractfile('./control') ).next()
-                    return DBConn().insert_pending_content_paths(pkg, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+                    pkg = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control'))).next()
+                    result = DBConn().insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
 
             except:
                 traceback.print_exc()
 
-                return False
+        os.chdir(cwd)
+        return result
 
-            finally:
-                os.chdir( cwd )
+    def check_utf8_package(self, package):
+        """
+        Unpack the .deb, do sanity checking, and gather info from it.
+
+        Currently information gathering consists of getting the contents list. In
+        the hopefully near future, it should also include gathering info from the
+        control file.
+
+        @ptype bootstrap_id: int
+        @param bootstrap_id: the id of the binary these packages
+          should be associated or zero meaning we are not bootstrapping
+          so insert into a temporary table
+
+        @return True if the deb is valid and contents were imported
+        """
+        rejected = not self.valid_deb()
+        self.__unpack()
+
+        if not rejected and self.tmpdir:
+            cwd = os.getcwd()
+            try:
+                os.chdir(self.tmpdir)
+                if self.chunks[1] == "control.tar.gz":
+                    control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
+                    control.extract('control', self.tmpdir )
+                if self.chunks[2] == "data.tar.gz":
+                    data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
+                elif self.chunks[2] == "data.tar.bz2":
+                    data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
+
+                for tarinfo in data:
+                    try:
+                        unicode( tarinfo.name )
+                    except:
+                        print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
+
+            except:
+                traceback.print_exc()
+                result = False
+
+            os.chdir(cwd)
 
 if __name__ == "__main__":
     Binary( "/srv/ftp.debian.org/queue/accepted/halevt_0.1.3-2_amd64.deb" ).scan_package()
index 3fad3f50dbba4701bdce5cabaee80eba51c06096..308f5548778810f2ccf8d6877c2dc4127b2f31d6 100755 (executable)
@@ -413,18 +413,22 @@ class DBConn(Singleton):
         @rtype: int
         @return: the database id for the given component
         """
-        values={'value': filename}
-        query = "SELECT id FROM content_file_names WHERE file = %(value)s"
-        id = self.__get_single_id(query, values, cachename='content_file_names')
-        if not id:
-            c = self.db_con.cursor()
-            c.execute( "INSERT INTO content_file_names VALUES (DEFAULT, %(value)s) RETURNING id",
-                       values )
-
-            id = c.fetchone()[0]
-            self.caches['content_file_names'].SetValue(values, id)
-
-        return id
+        try:
+            values={'value': filename}
+            query = "SELECT id FROM content_file_names WHERE file = %(value)s"
+            id = self.__get_single_id(query, values, cachename='content_file_names')
+            if not id:
+                c = self.db_con.cursor()
+                c.execute( "INSERT INTO content_file_names VALUES (DEFAULT, %(value)s) RETURNING id",
+                           values )
+
+                id = c.fetchone()[0]
+                self.caches['content_file_names'].SetValue(values, id)
+
+            return id
+        except:
+            traceback.print_exc()
+            raise
 
     def get_or_set_contents_path_id(self, path):
         """
@@ -439,18 +443,22 @@ class DBConn(Singleton):
         @rtype: int
         @return: the database id for the given component
         """
-        values={'value': path}
-        query = "SELECT id FROM content_file_paths WHERE path = %(value)s"
-        id = self.__get_single_id(query, values, cachename='content_path_names')
-        if not id:
-            c = self.db_con.cursor()
-            c.execute( "INSERT INTO content_file_paths VALUES (DEFAULT, %(value)s) RETURNING id",
-                       values )
-
-            id = c.fetchone()[0]
-            self.caches['content_path_names'].SetValue(values, id)
-
-        return id
+        try:
+            values={'value': path}
+            query = "SELECT id FROM content_file_paths WHERE path = %(value)s"
+            id = self.__get_single_id(query, values, cachename='content_path_names')
+            if not id:
+                c = self.db_con.cursor()
+                c.execute( "INSERT INTO content_file_paths VALUES (DEFAULT, %(value)s) RETURNING id",
+                           values )
+
+                id = c.fetchone()[0]
+                self.caches['content_path_names'].SetValue(values, id)
+
+            return id
+        except:
+            traceback.print_exc()
+            raise
 
     def get_suite_architectures(self, suite):
         """