5 Import known_changes files
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2009 Mike O'Connor <stew@debian.org>
9 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
29 ################################################################################
35 from daklib.dbconn import DBConn,get_knownchange
37 from daklib.config import Config
39 # where in dak.conf all of our configuration will be stowed
40 options_prefix = "KnownChanges"
41 options_prefix = "%s::Options" % options_prefix
43 log = logging.getLogger()
45 ################################################################################
48 def usage (exit_code=0):
49 print """Usage: dak import-known-changes [options]
53 run with n threads concurrently
56 show verbose information messages
59 supress all output but errors
64 def check_signature (sig_filename, data_filename=""):
66 "/home/joerg/keyring/keyrings/debian-keyring.gpg",
67 "/home/joerg/keyring/keyrings/debian-keyring.pgp",
68 "/home/joerg/keyring/keyrings/debian-maintainers.gpg",
69 "/home/joerg/keyring/keyrings/debian-role-keys.gpg",
70 "/home/joerg/keyring/keyrings/emeritus-keyring.pgp",
71 "/home/joerg/keyring/keyrings/emeritus-keyring.gpg",
72 "/home/joerg/keyring/keyrings/removed-keys.gpg",
73 "/home/joerg/keyring/keyrings/removed-keys.pgp"
76 keyringargs = " ".join(["--keyring %s" % x for x in keyrings ])
78 # Build the command line
79 status_read, status_write = os.pipe()
80 cmd = "gpgv --status-fd %s %s %s" % (status_write, keyringargs, sig_filename)
82 # Invoke gpgv on the file
83 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
85 # Process the status-fd output
86 (keywords, internal_error) = process_gpgv_output(status)
88 # If we failed to parse the status-fd output, let's just whine and bail now
90 warn("Couldn't parse signature")
93 # usually one would check for bad things here. We, however, do not care.
95 # Next check gpgv exited with a zero return code
97 warn("Couldn't parse signature")
100 # Sanity check the good stuff we expect
101 if not keywords.has_key("VALIDSIG"):
102 warn("Couldn't parse signature")
104 args = keywords["VALIDSIG"]
106 warn("Couldn't parse signature")
108 fingerprint = args[0]
113 class EndOfChanges(object):
114 """something enqueued to signify the last change"""
118 class OneAtATime(object):
120 a one space queue which sits between multiple possible producers
121 and multiple possible consumers
124 self.next_in_line = None
125 self.next_lock = threading.Condition()
127 def enqueue(self, next):
128 self.next_lock.acquire()
129 while self.next_in_line:
130 self.next_lock.wait()
132 assert( not self.next_in_line )
133 self.next_in_line = next
134 self.next_lock.notify()
135 self.next_lock.release()
138 self.next_lock.acquire()
139 while not self.next_in_line:
140 self.next_lock.wait()
141 result = self.next_in_line
143 if isinstance(next, EndOfChanges):
146 self.next_in_line = None
147 self.next_lock.notify()
148 self.next_lock.release()
151 class ChangesToImport(object):
152 """A changes file to be enqueued to be processed"""
153 def __init__(self, queue, checkdir, changesfile, count):
155 self.checkdir = checkdir
156 self.changesfile = changesfile
159 class ChangesGenerator(threading.Thread):
160 """enqueues changes files to be imported"""
161 def __init__(self, queue):
163 self.session = DBConn().session()
167 for directory in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
168 checkdir = cnf["Dir::Queue::%s" % (directory) ]
169 if os.path.exists(checkdir):
170 print "Looking into %s" % (checkdir)
172 for dirpath, dirnames, filenames in os.walk(checkdir, topdown=False):
174 # Empty directory (or only subdirectories), next
176 for changesfile in filenames:
177 if not changesfile.endswith(".changes"):
178 # Only interested in changes files.
182 if not get_knownchange(session):
183 self.queue.enqueue(ChangesToImport(directory, checkdir, changesfile, count))
185 self.queue.enqueue(EndOfChanges())
187 class ImportThread(threading.Thread):
188 def __init__(self, queue):
190 self.session = DBConn().session()
195 to_import = queue.dequeue()
199 print( "Directory %s, file %7d, failures %3d. (%s)" % (to_import.dirpath[-10:], to_import.count, failure, to_import.changesfile) )
202 changes.changes_file = to_import.changesfile
203 changesfile = os.path.join(to_import.dirpath, to_import.changesfile)
204 changes.changes = parse_changes(changesfile, signing_rules=-1)
205 changes.changes["fingerprint"] = check_signature(changesfile)
206 changes.add_known_changes(to_import.queue, self.session)
207 self.session.commit()
209 except InvalidDscError, line:
210 warn("syntax error in .dsc file '%s', line %s." % (f, line))
213 except ChangesUnicodeError:
214 warn("found invalid changes file, not properly utf-8 encoded")
217 print "Directory %s, file %7d, failures %3d. (%s)" % (dirpath[-10:], count, failure, changesfile)
224 arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
225 ('j',"concurrency", "%s::%s" % (options_prefix,"Concurrency"),"HasArg"),
226 ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
227 ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
230 args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments,sys.argv)
234 if (len(args) < 1) or not commands.has_key(args[0]):
237 if cnf.has_key("%s::%s" % (options_prefix,"Help")):
241 if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
244 elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
248 logging.basicConfig( level=level,
249 format='%(asctime)s %(levelname)s %(message)s',
250 stream = sys.stderr )
252 if Config().has_key( "%s::%s" %(options_prefix,"Concurrency")):
253 num_threads = int(Config()[ "%s::%s" %(options_prefix,"Suite")])
257 ChangesGenerator(queue).start()
259 for i in range(num_threads):
260 ImportThread(queue).start()
262 def which_suites(session):
264 return a list of suites to operate on
266 if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
267 suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
269 suites = Config().SubTree("Suite").List()
271 return [get_suite(s.lower(), session) for s in suites]
274 if __name__ == '__main__':