3 Helper code for packages generation.
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2011 Torsten Werner <twerner@debian.org>
7 @copyright: 2011 Mark Hymers <mhy@debian.org>
8 @license: GNU General Public License version 2 or later
11 ################################################################################
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
29 from daklib.dbconn import *
30 from daklib.config import Config
32 from multiprocessing import Pool
33 from subprocess import Popen, PIPE
37 class PackagesScanner(object):
39 PackagesScanner provides a threadsafe method scan() to scan the metadata of
42 def __init__(self, binary_id):
44 The argument binary_id is the id of the DBBinary object that
47 self.binary_id = binary_id
49 def scan(self, dummy_arg = None):
51 This method does the actual scan and fills in the associated metadata
52 property. It commits any changes to the database. The argument dummy_arg
53 is ignored but needed by our threadpool implementation.
55 session = DBConn().session()
56 binary = session.query(DBBinary).get(self.binary_id)
57 fileset = set(binary.read_control())
59 #if len(fileset) == 0:
60 # fileset.add('EMPTY_PACKAGE')
61 #for filename in fileset:
62 # binary.contents.append(BinContents(file = filename))
67 def scan_all(class_, limit = None):
69 The class method scan_all() scans all binaries using multiple threads.
70 The number of binaries to be scanned can be limited with the limit
71 argument. Returns the number of processed and remaining packages as a
74 session = DBConn().session()
75 query = session.query(DBBinary).filter(DBBinary.contents == None)
76 remaining = query.count
78 query = query.limit(limit)
79 processed = query.count()
81 for binary in query.yield_per(100):
82 pool.apply_async(scan_helper, (binary.binary_id, ))
85 remaining = remaining()
87 return { 'processed': processed, 'remaining': remaining }
89 def scan_helper(binary_id):
91 This function runs in a subprocess.
93 scanner = PackagesScanner(binary_id)