Merge branch 'dbtests'

diff --git a/config/debian/dak.conf b/config/debian/dak.conf
index 01bd8581d209b2bddc2ec75a7ad9f96da73d960a..bc9da6d8d64548b8adfb6a8fd574b48d58f8469b 100644 (file)
--- a/config/debian/dak.conf
+++ b/config/debian/dak.conf
@@ -454,6 +454,10 @@ DB
   Name "projectb";
   Host "";
   Port 5433;
+  // PoolSize should be at least ThreadCount + 1
+  PoolSize 17;
+  // MaxOverflow shouldn't exceed postgresql.conf's max_connections - PoolSize
+  MaxOverflow 13;
 };
 
 Archive

diff --git a/daklib/contents.py b/daklib/contents.py
index 740c0b2942951a5ff6f38df9581e9dd376698222..ed982d8dabfac25c3d97b18e479b15cfa75073f3 100755 (executable)
--- a/daklib/contents.py
+++ b/daklib/contents.py
@@ -210,14 +210,19 @@ class ContentsScanner(object):
         '''
         The class method scan_all() scans all binaries using multiple threads.
         The number of binaries to be scanned can be limited with the limit
-        argument.
+        argument. Returns the number of processed and remaining packages as a
+        dict.
         '''
         session = DBConn().session()
         query = session.query(DBBinary).filter(DBBinary.contents == None)
+        remaining = query.count
         if limit is not None:
             query = query.limit(limit)
+        processed = query.count()
         threadpool = ThreadPool()
         for binary in query.yield_per(100):
             threadpool.queueTask(ContentsScanner(binary).scan)
         threadpool.joinAll()
+        remaining = remaining()
         session.close()
+        return { 'processed': processed, 'remaining': remaining }

diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 0f3a690cc785d91ed64169d8a0ed088264eb8ff3..3806e31e2d09c9922c58014e9c0462d13e0a368b 100755 (executable)
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -512,9 +512,8 @@ class DBBinary(ORMObject):
         iso8859-1 encoding.
         '''
         fullpath = self.poolfile.fullpath
-        debdata = Popen(['dpkg-deb', '--fsys-tarfile', fullpath],
-            stdout = PIPE).stdout
-        tar = TarFile.open(fileobj = debdata, mode = 'r|')
+        dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
+        tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
         for member in tar.getmembers():
             if member.isfile():
                 try:
@@ -523,7 +522,8 @@ class DBBinary(ORMObject):
                     name = member.name.decode('iso8859-1')
                 yield normpath(name)
         tar.close()
-        debdata.close()
+        dpkg.stdout.close()
+        dpkg.wait()
 
 __all__.append('DBBinary')
 
@@ -3306,8 +3306,14 @@ class DBConn(object):
             connstr = "postgres:///%s" % cnf["DB::Name"]
             if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
                 connstr += "?port=%s" % cnf["DB::Port"]
-
-        self.db_pg   = create_engine(connstr, echo=self.debug)
+        if not cnf.has_key('DB::PoolSize'):
+            cnf['DB::PoolSize'] = '5'
+        if not cnf.has_key('DB::MaxOverflow'):
+            cnf['DB::MaxOverflow'] = '10'
+
+        self.db_pg   = create_engine(connstr, echo=self.debug,
+            pool_size=int(cnf['DB::PoolSize']),
+            max_overflow=int(cnf['DB::MaxOverflow']))
         self.db_meta = MetaData()
         self.db_meta.bind = self.db_pg
         self.db_smaker = sessionmaker(bind=self.db_pg,

diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py
index 7402601925178ddbfd1d3fa90796733461a7cabb..49db68489e0e084f6dba79e8acbdf379140381cd 100755 (executable)
--- a/tests/dbtest_contents.py
+++ b/tests/dbtest_contents.py
@@ -154,6 +154,9 @@ class ContentsTestCase(DBDakTestCase):
             self.otype['udeb'], self.comp['main'])
         self.assertEqual('tests/fixtures/ftp/squeeze/main/Contents-i386.gz', \
             cw.output_filename())
+        # test delete cascading
+        self.session.delete(self.binary['hello_2.2-1_i386'])
+        self.session.commit()
 
     def test_scan_contents(self):
         self.setup_binaries()