]> git.decadent.org.uk Git - dak.git/blob - dak/process_new.py
dinstall
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2 # vim:set et ts=4 sw=4:
3
4 """ Handles NEW and BYHAND packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
11 """
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
16
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25
26 ################################################################################
27
28 # 23:12|<aj> I will not hush!
29 # 23:12|<elmo> :>
30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
31 # 23:13|<aj> I shall not be silenced!
32 # 23:13|<aj> The world shall know!
33 # 23:13|<aj> The world *must* know!
34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
35 # 23:13|<aj> yay powerpuff girls!!
36 # 23:13|<aj> buttercup's my favourite, who's yours?
37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
38 # 23:14|<aj> *AREN'T YOU*?!
39 # 23:15|<aj> I will not be treated like this.
40 # 23:15|<aj> I shall have my revenge.
41 # 23:15|<aj> I SHALL!!!
42
43 ################################################################################
44
45 from __future__ import with_statement
46
47 import copy
48 import errno
49 import os
50 import readline
51 import stat
52 import sys
53 import time
54 import contextlib
55 import pwd
56 import apt_pkg, apt_inst
57 import examine_package
58
59 from daklib.dbconn import *
60 from daklib.queue import *
61 from daklib import daklog
62 from daklib import utils
63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
65 from daklib.summarystats import SummaryStats
66 from daklib.config import Config
67
68 # Globals
69 Options = None
70 Logger = None
71
72 Priorities = None
73 Sections = None
74
75 ################################################################################
76 ################################################################################
77 ################################################################################
78
79 def recheck(upload, session):
80     upload.recheck()
81     if len(upload.rejects) > 0:
82         answer = "XXX"
83         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
84             answer = 'S'
85
86         print "REJECT\n%s" % '\n'.join(upload.rejects)
87         prompt = "[R]eject, Skip, Quit ?"
88
89         while prompt.find(answer) == -1:
90             answer = utils.our_raw_input(prompt)
91             m = re_default_answer.match(prompt)
92             if answer == "":
93                 answer = m.group(1)
94             answer = answer[:1].upper()
95
96         if answer == 'R':
97             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
98             os.unlink(upload.pkg.changes_file[:-8]+".dak")
99             return 0
100         elif answer == 'S':
101             return 0
102         elif answer == 'Q':
103             end()
104             sys.exit(0)
105
106     return 1
107
108 ################################################################################
109
110 def indiv_sg_compare (a, b):
111     """Sort by source name, source, version, 'have source', and
112        finally by filename."""
113     # Sort by source version
114     q = apt_pkg.VersionCompare(a["version"], b["version"])
115     if q:
116         return -q
117
118     # Sort by 'have source'
119     a_has_source = a["architecture"].get("source")
120     b_has_source = b["architecture"].get("source")
121     if a_has_source and not b_has_source:
122         return -1
123     elif b_has_source and not a_has_source:
124         return 1
125
126     return cmp(a["filename"], b["filename"])
127
128 ############################################################
129
130 def sg_compare (a, b):
131     a = a[1]
132     b = b[1]
133     """Sort by have note, source already in database and time of oldest upload."""
134     # Sort by have note
135     a_note_state = a["note_state"]
136     b_note_state = b["note_state"]
137     if a_note_state < b_note_state:
138         return -1
139     elif a_note_state > b_note_state:
140         return 1
141     # Sort by source already in database (descending)
142     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
143     if source_in_database:
144         return -source_in_database
145
146     # Sort by time of oldest upload
147     return cmp(a["oldest"], b["oldest"])
148
149 def sort_changes(changes_files, session):
150     """Sort into source groups, then sort each source group by version,
151     have source, filename.  Finally, sort the source groups by have
152     note, time of oldest upload of each source upload."""
153     if len(changes_files) == 1:
154         return changes_files
155
156     sorted_list = []
157     cache = {}
158     # Read in all the .changes files
159     for filename in changes_files:
160         u = Upload()
161         try:
162             u.pkg.load_dot_dak(filename)
163             u.update_subst()
164             cache[filename] = copy.copy(u.pkg.changes)
165             cache[filename]["filename"] = filename
166         except:
167             sorted_list.append(filename)
168             break
169     # Divide the .changes into per-source groups
170     per_source = {}
171     for filename in cache.keys():
172         source = cache[filename]["source"]
173         if not per_source.has_key(source):
174             per_source[source] = {}
175             per_source[source]["list"] = []
176         per_source[source]["list"].append(cache[filename])
177     # Determine oldest time and have note status for each source group
178     for source in per_source.keys():
179         q = session.query(DBSource).filter_by(source = source).all()
180         per_source[source]["source_in_database"] = len(q)>0
181         source_list = per_source[source]["list"]
182         first = source_list[0]
183         oldest = os.stat(first["filename"])[stat.ST_MTIME]
184         have_note = 0
185         for d in per_source[source]["list"]:
186             mtime = os.stat(d["filename"])[stat.ST_MTIME]
187             if mtime < oldest:
188                 oldest = mtime
189             have_note += has_new_comment(d["source"], d["version"], session)
190         per_source[source]["oldest"] = oldest
191         if not have_note:
192             per_source[source]["note_state"] = 0; # none
193         elif have_note < len(source_list):
194             per_source[source]["note_state"] = 1; # some
195         else:
196             per_source[source]["note_state"] = 2; # all
197         per_source[source]["list"].sort(indiv_sg_compare)
198     per_source_items = per_source.items()
199     per_source_items.sort(sg_compare)
200     for i in per_source_items:
201         for j in i[1]["list"]:
202             sorted_list.append(j["filename"])
203     return sorted_list
204
205 ################################################################################
206
207 class Section_Completer:
208     def __init__ (self, session):
209         self.sections = []
210         self.matches = []
211         for s, in session.query(Section.section):
212             self.sections.append(s)
213
214     def complete(self, text, state):
215         if state == 0:
216             self.matches = []
217             n = len(text)
218             for word in self.sections:
219                 if word[:n] == text:
220                     self.matches.append(word)
221         try:
222             return self.matches[state]
223         except IndexError:
224             return None
225
226 ############################################################
227
228 class Priority_Completer:
229     def __init__ (self, session):
230         self.priorities = []
231         self.matches = []
232         for p, in session.query(Priority.priority):
233             self.priorities.append(p)
234
235     def complete(self, text, state):
236         if state == 0:
237             self.matches = []
238             n = len(text)
239             for word in self.priorities:
240                 if word[:n] == text:
241                     self.matches.append(word)
242         try:
243             return self.matches[state]
244         except IndexError:
245             return None
246
247 ################################################################################
248
249 def print_new (new, upload, indexed, file=sys.stdout):
250     check_valid(new)
251     broken = False
252     index = 0
253     for pkg in new.keys():
254         index += 1
255         section = new[pkg]["section"]
256         priority = new[pkg]["priority"]
257         if new[pkg]["section id"] == -1:
258             section += "[!]"
259             broken = True
260         if new[pkg]["priority id"] == -1:
261             priority += "[!]"
262             broken = True
263         if indexed:
264             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
265         else:
266             line = "%-20s %-20s %-20s" % (pkg, priority, section)
267         line = line.strip()+'\n'
268         file.write(line)
269     notes = get_new_comments(upload.pkg.changes.get("source"))
270     for note in notes:
271         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
272               % (note.author, note.version, note.notedate, note.comment)
273         print "-" * 72
274     return broken, len(notes) > 0
275
276 ################################################################################
277
278 def index_range (index):
279     if index == 1:
280         return "1"
281     else:
282         return "1-%s" % (index)
283
284 ################################################################################
285 ################################################################################
286
287 def edit_new (new, upload):
288     # Write the current data to a temporary file
289     (fd, temp_filename) = utils.temp_filename()
290     temp_file = os.fdopen(fd, 'w')
291     print_new (new, upload, indexed=0, file=temp_file)
292     temp_file.close()
293     # Spawn an editor on that file
294     editor = os.environ.get("EDITOR","vi")
295     result = os.system("%s %s" % (editor, temp_filename))
296     if result != 0:
297         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
298     # Read the edited data back in
299     temp_file = utils.open_file(temp_filename)
300     lines = temp_file.readlines()
301     temp_file.close()
302     os.unlink(temp_filename)
303     # Parse the new data
304     for line in lines:
305         line = line.strip()
306         if line == "":
307             continue
308         s = line.split()
309         # Pad the list if necessary
310         s[len(s):3] = [None] * (3-len(s))
311         (pkg, priority, section) = s[:3]
312         if not new.has_key(pkg):
313             utils.warn("Ignoring unknown package '%s'" % (pkg))
314         else:
315             # Strip off any invalid markers, print_new will readd them.
316             if section.endswith("[!]"):
317                 section = section[:-3]
318             if priority.endswith("[!]"):
319                 priority = priority[:-3]
320             for f in new[pkg]["files"]:
321                 upload.pkg.files[f]["section"] = section
322                 upload.pkg.files[f]["priority"] = priority
323             new[pkg]["section"] = section
324             new[pkg]["priority"] = priority
325
326 ################################################################################
327
328 def edit_index (new, upload, index):
329     priority = new[index]["priority"]
330     section = new[index]["section"]
331     ftype = new[index]["type"]
332     done = 0
333     while not done:
334         print "\t".join([index, priority, section])
335
336         answer = "XXX"
337         if ftype != "dsc":
338             prompt = "[B]oth, Priority, Section, Done ? "
339         else:
340             prompt = "[S]ection, Done ? "
341         edit_priority = edit_section = 0
342
343         while prompt.find(answer) == -1:
344             answer = utils.our_raw_input(prompt)
345             m = re_default_answer.match(prompt)
346             if answer == "":
347                 answer = m.group(1)
348             answer = answer[:1].upper()
349
350         if answer == 'P':
351             edit_priority = 1
352         elif answer == 'S':
353             edit_section = 1
354         elif answer == 'B':
355             edit_priority = edit_section = 1
356         elif answer == 'D':
357             done = 1
358
359         # Edit the priority
360         if edit_priority:
361             readline.set_completer(Priorities.complete)
362             got_priority = 0
363             while not got_priority:
364                 new_priority = utils.our_raw_input("New priority: ").strip()
365                 if new_priority not in Priorities.priorities:
366                     print "E: '%s' is not a valid priority, try again." % (new_priority)
367                 else:
368                     got_priority = 1
369                     priority = new_priority
370
371         # Edit the section
372         if edit_section:
373             readline.set_completer(Sections.complete)
374             got_section = 0
375             while not got_section:
376                 new_section = utils.our_raw_input("New section: ").strip()
377                 if new_section not in Sections.sections:
378                     print "E: '%s' is not a valid section, try again." % (new_section)
379                 else:
380                     got_section = 1
381                     section = new_section
382
383         # Reset the readline completer
384         readline.set_completer(None)
385
386     for f in new[index]["files"]:
387         upload.pkg.files[f]["section"] = section
388         upload.pkg.files[f]["priority"] = priority
389     new[index]["priority"] = priority
390     new[index]["section"] = section
391     return new
392
393 ################################################################################
394
395 def edit_overrides (new, upload, session):
396     print
397     done = 0
398     while not done:
399         print_new (new, upload, indexed=1)
400         new_index = {}
401         index = 0
402         for i in new.keys():
403             index += 1
404             new_index[index] = i
405
406         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
407
408         got_answer = 0
409         while not got_answer:
410             answer = utils.our_raw_input(prompt)
411             if not answer.isdigit():
412                 answer = answer[:1].upper()
413             if answer == "E" or answer == "D":
414                 got_answer = 1
415             elif re_isanum.match (answer):
416                 answer = int(answer)
417                 if (answer < 1) or (answer > index):
418                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
419                 else:
420                     got_answer = 1
421
422         if answer == 'E':
423             edit_new(new, upload)
424         elif answer == 'D':
425             done = 1
426         else:
427             edit_index (new, upload, new_index[answer])
428
429     return new
430
431 ################################################################################
432
433 def edit_note(note, upload, session):
434     # Write the current data to a temporary file
435     (fd, temp_filename) = utils.temp_filename()
436     editor = os.environ.get("EDITOR","vi")
437     answer = 'E'
438     while answer == 'E':
439         os.system("%s %s" % (editor, temp_filename))
440         temp_file = utils.open_file(temp_filename)
441         newnote = temp_file.read().rstrip()
442         temp_file.close()
443         print "New Note:"
444         print utils.prefix_multi_line_string(newnote,"  ")
445         prompt = "[D]one, Edit, Abandon, Quit ?"
446         answer = "XXX"
447         while prompt.find(answer) == -1:
448             answer = utils.our_raw_input(prompt)
449             m = re_default_answer.search(prompt)
450             if answer == "":
451                 answer = m.group(1)
452             answer = answer[:1].upper()
453     os.unlink(temp_filename)
454     if answer == 'A':
455         return
456     elif answer == 'Q':
457         end()
458         sys.exit(0)
459
460     comment = NewComment()
461     comment.package = upload.pkg.changes["source"]
462     comment.version = upload.pkg.changes["version"]
463     comment.comment = newnote
464     comment.author  = utils.whoami()
465     comment.trainee = bool(Options["Trainee"])
466     session.add(comment)
467     session.commit()
468
469 ################################################################################
470
471 def check_pkg (upload):
472     try:
473         less_fd = os.popen("less -R -", 'w', 0)
474         stdout_fd = sys.stdout
475         try:
476             sys.stdout = less_fd
477             changes = utils.parse_changes (upload.pkg.changes_file)
478             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
479             files = upload.pkg.files
480             for f in files.keys():
481                 if files[f].has_key("new"):
482                     ftype = files[f]["type"]
483                     if ftype == "deb":
484                         examine_package.check_deb(changes['distribution'], f)
485                     elif ftype == "dsc":
486                         examine_package.check_dsc(changes['distribution'], f)
487         finally:
488             examine_package.output_package_relations()
489             sys.stdout = stdout_fd
490     except IOError, e:
491         if e.errno == errno.EPIPE:
492             utils.warn("[examine_package] Caught EPIPE; skipping.")
493             pass
494         else:
495             raise
496     except KeyboardInterrupt:
497         utils.warn("[examine_package] Caught C-c; skipping.")
498         pass
499
500 ################################################################################
501
502 ## FIXME: horribly Debian specific
503
504 def do_bxa_notification(upload):
505     files = upload.pkg.files
506     summary = ""
507     for f in files.keys():
508         if files[f]["type"] == "deb":
509             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
510             summary += "\n"
511             summary += "Package: %s\n" % (control.Find("Package"))
512             summary += "Description: %s\n" % (control.Find("Description"))
513     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
514     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
515     utils.send_mail(bxa_mail)
516
517 ################################################################################
518
519 def add_overrides (new, upload, session):
520     changes = upload.pkg.changes
521     files = upload.pkg.files
522     srcpkg = changes.get("source")
523
524     for suite in changes["suite"].keys():
525         suite_id = get_suite(suite).suite_id
526         for pkg in new.keys():
527             component_id = get_component(new[pkg]["component"]).component_id
528             type_id = get_override_type(new[pkg]["type"]).overridetype_id
529             priority_id = new[pkg]["priority id"]
530             section_id = new[pkg]["section id"]
531             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
532             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
533                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
534             for f in new[pkg]["files"]:
535                 if files[f].has_key("new"):
536                     del files[f]["new"]
537             del new[pkg]
538
539     session.commit()
540
541     if Config().FindB("Dinstall::BXANotify"):
542         do_bxa_notification(upload)
543
544 ################################################################################
545
546 def prod_maintainer (note, upload):
547     cnf = Config()
548     # Here we prepare an editor and get them ready to prod...
549     (fd, temp_filename) = utils.temp_filename()
550     temp_file = os.fdopen(fd, 'w')
551     if len(note) > 0:
552         for line in note:
553             temp_file.write(line)
554     temp_file.close()
555     editor = os.environ.get("EDITOR","vi")
556     answer = 'E'
557     while answer == 'E':
558         os.system("%s %s" % (editor, temp_filename))
559         temp_fh = utils.open_file(temp_filename)
560         prod_message = "".join(temp_fh.readlines())
561         temp_fh.close()
562         print "Prod message:"
563         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
564         prompt = "[P]rod, Edit, Abandon, Quit ?"
565         answer = "XXX"
566         while prompt.find(answer) == -1:
567             answer = utils.our_raw_input(prompt)
568             m = re_default_answer.search(prompt)
569             if answer == "":
570                 answer = m.group(1)
571             answer = answer[:1].upper()
572     os.unlink(temp_filename)
573     if answer == 'A':
574         return
575     elif answer == 'Q':
576         end()
577         sys.exit(0)
578     # Otherwise, do the proding...
579     user_email_address = utils.whoami() + " <%s>" % (
580         cnf["Dinstall::MyAdminAddress"])
581
582     Subst = upload.Subst
583
584     Subst["__FROM_ADDRESS__"] = user_email_address
585     Subst["__PROD_MESSAGE__"] = prod_message
586     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
587
588     prod_mail_message = utils.TemplateSubst(
589         Subst,cnf["Dir::Templates"]+"/process-new.prod")
590
591     # Send the prod mail if appropriate
592     if not cnf["Dinstall::Options::No-Mail"]:
593         utils.send_mail(prod_mail_message)
594
595     print "Sent proding message"
596
597 ################################################################################
598
599 def do_new(upload, session):
600     print "NEW\n"
601     files = upload.pkg.files
602     changes = upload.pkg.changes
603     cnf = Config()
604
605     # Make a copy of distribution we can happily trample on
606     changes["suite"] = copy.copy(changes["distribution"])
607
608     # Fix up the list of target suites
609     for suite in changes["suite"].keys():
610         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
611         if override:
612             (olderr, newerr) = (get_suite(suite, session) == None,
613                                 get_suite(override, session) == None)
614             if olderr or newerr:
615                 (oinv, newinv) = ("", "")
616                 if olderr: oinv = "invalid "
617                 if newerr: ninv = "invalid "
618                 print "warning: overriding %ssuite %s to %ssuite %s" % (
619                         oinv, suite, ninv, override)
620             del changes["suite"][suite]
621             changes["suite"][override] = 1
622     # Validate suites
623     for suite in changes["suite"].keys():
624         if get_suite(suite, session) is None:
625             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
626
627     # The main NEW processing loop
628     done = 0
629     while not done:
630         # Find out what's new
631         new = determine_new(changes, files)
632
633         if not new:
634             break
635
636         answer = "XXX"
637         if Options["No-Action"] or Options["Automatic"]:
638             answer = 'S'
639
640         (broken, note) = print_new(new, upload, indexed=0)
641         prompt = ""
642
643         if not broken and not note:
644             prompt = "Add overrides, "
645         if broken:
646             print "W: [!] marked entries must be fixed before package can be processed."
647         if note:
648             print "W: note must be removed before package can be processed."
649             prompt += "RemOve all notes, Remove note, "
650
651         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
652
653         while prompt.find(answer) == -1:
654             answer = utils.our_raw_input(prompt)
655             m = re_default_answer.search(prompt)
656             if answer == "":
657                 answer = m.group(1)
658             answer = answer[:1].upper()
659
660         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
661             utils.warn("Trainees can't do that")
662             continue
663
664         if answer == 'A' and not Options["Trainee"]:
665             try:
666                 check_daily_lock()
667                 done = add_overrides (new, upload, session)
668                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
669             except CantGetLockError:
670                 print "Hello? Operator! Give me the number for 911!"
671                 print "Dinstall in the locked area, cant process packages, come back later"
672         elif answer == 'C':
673             check_pkg(upload)
674         elif answer == 'E' and not Options["Trainee"]:
675             new = edit_overrides (new, upload, session)
676         elif answer == 'M' and not Options["Trainee"]:
677             upload.pkg.remove_known_changes()
678             aborted = upload.do_reject(manual=1,
679                                        reject_message=Options["Manual-Reject"],
680                                        note=get_new_comments(changes.get("source", ""), session=session))
681             if not aborted:
682                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
683                 os.unlink(upload.pkg.changes_file[:-8]+".dak")
684                 done = 1
685         elif answer == 'N':
686             edit_note(get_new_comments(changes.get("source", ""), session=session),
687                       upload, session)
688         elif answer == 'P' and not Options["Trainee"]:
689             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
690                             upload)
691             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
692         elif answer == 'R' and not Options["Trainee"]:
693             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
694             if confirm == "y":
695                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
696                     session.delete(c)
697                 session.commit()
698         elif answer == 'O' and not Options["Trainee"]:
699             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
700             if confirm == "y":
701                 for c in get_new_comments(changes.get("source", ""), session=session):
702                     session.delete(c)
703                 session.commit()
704
705         elif answer == 'S':
706             done = 1
707         elif answer == 'Q':
708             end()
709             sys.exit(0)
710
711 ################################################################################
712 ################################################################################
713 ################################################################################
714
715 def usage (exit_code=0):
716     print """Usage: dak process-new [OPTION]... [CHANGES]...
717   -a, --automatic           automatic run
718   -h, --help                show this help and exit.
719   -m, --manual-reject=MSG   manual reject with `msg'
720   -n, --no-action           don't do anything
721   -t, --trainee             FTP Trainee mode
722   -V, --version             display the version number and exit"""
723     sys.exit(exit_code)
724
725 ################################################################################
726
727 def do_byhand(upload, session):
728     done = 0
729     while not done:
730         files = upload.pkg.files
731         will_install = 1
732         byhand = []
733
734         for f in files.keys():
735             if files[f]["type"] == "byhand":
736                 if os.path.exists(f):
737                     print "W: %s still present; please process byhand components and try again." % (f)
738                     will_install = 0
739                 else:
740                     byhand.append(f)
741
742         answer = "XXXX"
743         if Options["No-Action"]:
744             answer = "S"
745         if will_install:
746             if Options["Automatic"] and not Options["No-Action"]:
747                 answer = 'A'
748             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
749         else:
750             prompt = "Manual reject, [S]kip, Quit ?"
751
752         while prompt.find(answer) == -1:
753             answer = utils.our_raw_input(prompt)
754             m = re_default_answer.search(prompt)
755             if answer == "":
756                 answer = m.group(1)
757             answer = answer[:1].upper()
758
759         if answer == 'A':
760             try:
761                 check_daily_lock()
762                 done = 1
763                 for f in byhand:
764                     del files[f]
765                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
766             except CantGetLockError:
767                 print "Hello? Operator! Give me the number for 911!"
768                 print "Dinstall in the locked area, cant process packages, come back later"
769         elif answer == 'M':
770             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
771             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
772             os.unlink(upload.pkg.changes_file[:-8]+".dak")
773             done = 1
774         elif answer == 'S':
775             done = 1
776         elif answer == 'Q':
777             end()
778             sys.exit(0)
779
780 ################################################################################
781
782 def check_daily_lock():
783     """
784     Raises CantGetLockError if the dinstall daily.lock exists.
785     """
786
787     cnf = Config()
788     try:
789         os.open(cnf["Process-New::DinstallLockFile"],
790                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
791     except OSError, e:
792         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
793             raise CantGetLockError
794
795     os.unlink(cnf["Process-New::DinstallLockFile"])
796
797
798 @contextlib.contextmanager
799 def lock_package(package):
800     """
801     Lock C{package} so that noone else jumps in processing it.
802
803     @type package: string
804     @param package: source package name to lock
805     """
806
807     path = os.path.join(Config()["Process-New::LockDir"], package)
808     try:
809         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
810     except OSError, e:
811         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
812             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
813             raise AlreadyLockedError, user
814
815     try:
816         yield fd
817     finally:
818         os.unlink(path)
819
820 def _accept(upload):
821     if Options["No-Action"]:
822         return
823     (summary, short_summary) = upload.build_summaries()
824     upload.accept(summary, short_summary, targetqueue)
825     os.unlink(upload.pkg.changes_file[:-8]+".dak")
826
827 def do_accept(upload):
828     print "ACCEPT"
829     cnf = Config()
830     if not Options["No-Action"]:
831         (summary, short_summary) = upload.build_summaries()
832
833         if cnf.FindB("Dinstall::SecurityQueueHandling"):
834             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
835             upload.move_to_queue(get_policy_queue('embargoed'))
836             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
837             # Check for override disparities
838             upload.Subst["__SUMMARY__"] = summary
839         else:
840             # Just a normal upload, accept it...
841             _accept(upload)
842
843 def do_pkg(changes_file, session):
844     u = Upload()
845     u.pkg.load_dot_dak(changes_file)
846     u.update_subst()
847
848     cnf = Config()
849     bcc = "X-DAK: dak process-new"
850     if cnf.has_key("Dinstall::Bcc"):
851         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
852     else:
853         u.Subst["__BCC__"] = bcc
854
855     files = u.pkg.files
856
857     try:
858         with lock_package(u.pkg.changes["source"]):
859             if not recheck(u, session):
860                 return
861
862             (new, byhand) = check_status(files)
863             if new or byhand:
864                 if new:
865                     do_new(u, session)
866                 if byhand:
867                     do_byhand(u, session)
868                 (new, byhand) = check_status(files)
869
870             if not new and not byhand:
871                 try:
872                     check_daily_lock()
873                     do_accept(u)
874                 except CantGetLockError:
875                     print "Hello? Operator! Give me the number for 911!"
876                     print "Dinstall in the locked area, cant process packages, come back later"
877     except AlreadyLockedError, e:
878         print "Seems to be locked by %s already, skipping..." % (e)
879
880 ################################################################################
881
882 def end():
883     accept_count = SummaryStats().accept_count
884     accept_bytes = SummaryStats().accept_bytes
885
886     if accept_count:
887         sets = "set"
888         if accept_count > 1:
889             sets = "sets"
890         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
891         Logger.log(["total",accept_count,accept_bytes])
892
893     if not Options["No-Action"] and not Options["Trainee"]:
894         Logger.close()
895
896 ################################################################################
897
898 def main():
899     global Options, Logger, Sections, Priorities
900
901     print "NO NEW PROCESSING CURRENTLY AVAILABLE"
902     print "(Go and do something more interesting)"
903     sys.exit(0)
904
905     cnf = Config()
906     session = DBConn().session()
907
908     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
909                  ('h',"help","Process-New::Options::Help"),
910                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
911                  ('t',"trainee","Process-New::Options::Trainee"),
912                  ('n',"no-action","Process-New::Options::No-Action")]
913
914     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
915         if not cnf.has_key("Process-New::Options::%s" % (i)):
916             cnf["Process-New::Options::%s" % (i)] = ""
917
918     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
919     if len(changes_files) == 0:
920         changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
921
922     Options = cnf.SubTree("Process-New::Options")
923
924     if Options["Help"]:
925         usage()
926
927     if not Options["No-Action"]:
928         try:
929             Logger = daklog.Logger(cnf, "process-new")
930         except CantOpenError, e:
931             Options["Trainee"] = "True"
932
933     Sections = Section_Completer(session)
934     Priorities = Priority_Completer(session)
935     readline.parse_and_bind("tab: complete")
936
937     if len(changes_files) > 1:
938         sys.stderr.write("Sorting changes...\n")
939     changes_files = sort_changes(changes_files, session)
940
941     # Kill me now? **FIXME**
942     cnf["Dinstall::Options::No-Mail"] = ""
943
944     for changes_file in changes_files:
945         changes_file = utils.validate_changes_file_arg(changes_file, 0)
946         if not changes_file:
947             continue
948         print "\n" + changes_file
949
950         do_pkg (changes_file, session)
951
952     end()
953
954 ################################################################################
955
956 if __name__ == '__main__':
957     main()