From: Torsten Werner <twerner@debian.org>
Date: Wed, 2 Mar 2011 20:57:08 +0000 (+0100)
Subject: Merge branch 'dbtests'
X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=504b5a17d6b16ecfc7aea32e4a1e92658db829fe;hp=8cc76224a38ffd1edca7cec7c83e58b1362edcb7;p=dak.git

Merge branch 'dbtests'
---

diff --git a/config/debian/dak.conf b/config/debian/dak.conf
index 01bd8581..bc9da6d8 100644
--- a/config/debian/dak.conf
+++ b/config/debian/dak.conf
@@ -454,6 +454,10 @@ DB
   Name "projectb";
   Host "";
   Port 5433;
+  // PoolSize should be at least ThreadCount + 1
+  PoolSize 17;
+  // MaxOverflow shouldn't exceed postgresql.conf's max_connections - PoolSize
+  MaxOverflow 13;
 };
 
 Archive
diff --git a/daklib/contents.py b/daklib/contents.py
index 740c0b29..ed982d8d 100755
--- a/daklib/contents.py
+++ b/daklib/contents.py
@@ -210,14 +210,19 @@ class ContentsScanner(object):
         '''
         The class method scan_all() scans all binaries using multiple threads.
         The number of binaries to be scanned can be limited with the limit
-        argument.
+        argument. Returns the number of processed and remaining packages as a
+        dict.
         '''
         session = DBConn().session()
         query = session.query(DBBinary).filter(DBBinary.contents == None)
+        remaining = query.count
         if limit is not None:
             query = query.limit(limit)
+        processed = query.count()
         threadpool = ThreadPool()
         for binary in query.yield_per(100):
             threadpool.queueTask(ContentsScanner(binary).scan)
         threadpool.joinAll()
+        remaining = remaining()
         session.close()
+        return { 'processed': processed, 'remaining': remaining }
diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 0f3a690c..3806e31e 100755
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -512,9 +512,8 @@ class DBBinary(ORMObject):
         iso8859-1 encoding.
         '''
         fullpath = self.poolfile.fullpath
-        debdata = Popen(['dpkg-deb', '--fsys-tarfile', fullpath],
-            stdout = PIPE).stdout
-        tar = TarFile.open(fileobj = debdata, mode = 'r|')
+        dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
+        tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
         for member in tar.getmembers():
             if member.isfile():
                 try:
@@ -523,7 +522,8 @@ class DBBinary(ORMObject):
                     name = member.name.decode('iso8859-1')
                 yield normpath(name)
         tar.close()
-        debdata.close()
+        dpkg.stdout.close()
+        dpkg.wait()
 
 __all__.append('DBBinary')
 
@@ -3306,8 +3306,14 @@ class DBConn(object):
             connstr = "postgres:///%s" % cnf["DB::Name"]
             if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
                 connstr += "?port=%s" % cnf["DB::Port"]
-
-        self.db_pg   = create_engine(connstr, echo=self.debug)
+        if not cnf.has_key('DB::PoolSize'):
+            cnf['DB::PoolSize'] = '5'
+        if not cnf.has_key('DB::MaxOverflow'):
+            cnf['DB::MaxOverflow'] = '10'
+
+        self.db_pg   = create_engine(connstr, echo=self.debug,
+            pool_size=int(cnf['DB::PoolSize']),
+            max_overflow=int(cnf['DB::MaxOverflow']))
         self.db_meta = MetaData()
         self.db_meta.bind = self.db_pg
         self.db_smaker = sessionmaker(bind=self.db_pg,
diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py
index 74026019..49db6848 100755
--- a/tests/dbtest_contents.py
+++ b/tests/dbtest_contents.py
@@ -154,6 +154,9 @@ class ContentsTestCase(DBDakTestCase):
             self.otype['udeb'], self.comp['main'])
         self.assertEqual('tests/fixtures/ftp/squeeze/main/Contents-i386.gz', \
             cw.output_filename())
+        # test delete cascading
+        self.session.delete(self.binary['hello_2.2-1_i386'])
+        self.session.commit()
 
     def test_scan_contents(self):
         self.setup_binaries()