]> git.decadent.org.uk Git - dak.git/blob - dak/process_new.py
de01787926ab82e7bfc6b6dfbaef208037808ea9
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2 # vim:set et ts=4 sw=4:
3
4 """ Handles NEW and BYHAND packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
11 """
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
16
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25
26 ################################################################################
27
28 # 23:12|<aj> I will not hush!
29 # 23:12|<elmo> :>
30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
31 # 23:13|<aj> I shall not be silenced!
32 # 23:13|<aj> The world shall know!
33 # 23:13|<aj> The world *must* know!
34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
35 # 23:13|<aj> yay powerpuff girls!!
36 # 23:13|<aj> buttercup's my favourite, who's yours?
37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
38 # 23:14|<aj> *AREN'T YOU*?!
39 # 23:15|<aj> I will not be treated like this.
40 # 23:15|<aj> I shall have my revenge.
41 # 23:15|<aj> I SHALL!!!
42
43 ################################################################################
44
45 from __future__ import with_statement
46
47 import copy
48 import errno
49 import os
50 import readline
51 import stat
52 import sys
53 import time
54 import contextlib
55 import pwd
56 import apt_pkg, apt_inst
57 import examine_package
58
59 from daklib.dbconn import *
60 from daklib.queue import *
61 from daklib import daklog
62 from daklib import utils
63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
65 from daklib.summarystats import SummaryStats
66 from daklib.config import Config
67
68 # Globals
69 Options = None
70 Logger = None
71
72 Priorities = None
73 Sections = None
74
75 ################################################################################
76 ################################################################################
77 ################################################################################
78
79 def recheck(upload, session):
80     files = upload.pkg.files
81
82     cnf = Config()
83     for f in files.keys():
84         # The .orig.tar.gz can disappear out from under us is it's a
85         # duplicate of one in the archive.
86         if not files.has_key(f):
87             continue
88         # Check that the source still exists
89         if files[f]["type"] == "deb":
90             source_version = files[f]["source version"]
91             source_package = files[f]["source package"]
92             if not upload.pkg.changes["architecture"].has_key("source") \
93                and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()):
94                 source_epochless_version = re_no_epoch.sub('', source_version)
95                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
96                 found = 0
97                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
98                     if cnf.has_key("Dir::Queue::%s" % (q)):
99                         if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
100                             found = 1
101                 if not found:
102                     upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
103
104         # Version and file overwrite checks
105         if files[f]["type"] == "deb":
106             upload.check_binary_against_db(f, session)
107         elif files[f]["type"] == "dsc":
108             upload.check_source_against_db(f, session)
109             upload.check_dsc_against_db(f, session)
110
111     if len(upload.rejects) > 0:
112         answer = "XXX"
113         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
114             answer = 'S'
115
116         print "REJECT\n" + upload.rejects.join("\n"),
117         prompt = "[R]eject, Skip, Quit ?"
118
119         while prompt.find(answer) == -1:
120             answer = utils.our_raw_input(prompt)
121             m = re_default_answer.match(prompt)
122             if answer == "":
123                 answer = m.group(1)
124             answer = answer[:1].upper()
125
126         if answer == 'R':
127             upload.do_reject(manual=0, reject_message=upload.rejects.join("\n"))
128             os.unlink(upload.pkg.changes_file[:-8]+".dak")
129             return 0
130         elif answer == 'S':
131             return 0
132         elif answer == 'Q':
133             end()
134             sys.exit(0)
135
136     return 1
137
138 ################################################################################
139
140 def indiv_sg_compare (a, b):
141     """Sort by source name, source, version, 'have source', and
142        finally by filename."""
143     # Sort by source version
144     q = apt_pkg.VersionCompare(a["version"], b["version"])
145     if q:
146         return -q
147
148     # Sort by 'have source'
149     a_has_source = a["architecture"].get("source")
150     b_has_source = b["architecture"].get("source")
151     if a_has_source and not b_has_source:
152         return -1
153     elif b_has_source and not a_has_source:
154         return 1
155
156     return cmp(a["filename"], b["filename"])
157
158 ############################################################
159
160 def sg_compare (a, b):
161     a = a[1]
162     b = b[1]
163     """Sort by have note, source already in database and time of oldest upload."""
164     # Sort by have note
165     a_note_state = a["note_state"]
166     b_note_state = b["note_state"]
167     if a_note_state < b_note_state:
168         return -1
169     elif a_note_state > b_note_state:
170         return 1
171     # Sort by source already in database (descending)
172     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
173     if source_in_database:
174         return -source_in_database
175
176     # Sort by time of oldest upload
177     return cmp(a["oldest"], b["oldest"])
178
179 def sort_changes(changes_files, session):
180     """Sort into source groups, then sort each source group by version,
181     have source, filename.  Finally, sort the source groups by have
182     note, time of oldest upload of each source upload."""
183     if len(changes_files) == 1:
184         return changes_files
185
186     sorted_list = []
187     cache = {}
188     # Read in all the .changes files
189     for filename in changes_files:
190         u = Upload()
191         try:
192             u.pkg.load_dot_dak(filename)
193             u.update_subst()
194             cache[filename] = copy.copy(u.pkg.changes)
195             cache[filename]["filename"] = filename
196         except:
197             sorted_list.append(filename)
198             break
199     # Divide the .changes into per-source groups
200     per_source = {}
201     for filename in cache.keys():
202         source = cache[filename]["source"]
203         if not per_source.has_key(source):
204             per_source[source] = {}
205             per_source[source]["list"] = []
206         per_source[source]["list"].append(cache[filename])
207     # Determine oldest time and have note status for each source group
208     for source in per_source.keys():
209         q = session.query(DBSource).filter_by(source = source).all()
210         per_source[source]["source_in_database"] = len(q)>0
211         source_list = per_source[source]["list"]
212         first = source_list[0]
213         oldest = os.stat(first["filename"])[stat.ST_MTIME]
214         have_note = 0
215         for d in per_source[source]["list"]:
216             mtime = os.stat(d["filename"])[stat.ST_MTIME]
217             if mtime < oldest:
218                 oldest = mtime
219             have_note += has_new_comment(d["source"], d["version"], session)
220         per_source[source]["oldest"] = oldest
221         if not have_note:
222             per_source[source]["note_state"] = 0; # none
223         elif have_note < len(source_list):
224             per_source[source]["note_state"] = 1; # some
225         else:
226             per_source[source]["note_state"] = 2; # all
227         per_source[source]["list"].sort(indiv_sg_compare)
228     per_source_items = per_source.items()
229     per_source_items.sort(sg_compare)
230     for i in per_source_items:
231         for j in i[1]["list"]:
232             sorted_list.append(j["filename"])
233     return sorted_list
234
235 ################################################################################
236
237 class Section_Completer:
238     def __init__ (self, session):
239         self.sections = []
240         self.matches = []
241         for s, in session.query(Section.section):
242             self.sections.append(s)
243
244     def complete(self, text, state):
245         if state == 0:
246             self.matches = []
247             n = len(text)
248             for word in self.sections:
249                 if word[:n] == text:
250                     self.matches.append(word)
251         try:
252             return self.matches[state]
253         except IndexError:
254             return None
255
256 ############################################################
257
258 class Priority_Completer:
259     def __init__ (self, session):
260         self.priorities = []
261         self.matches = []
262         for p, in session.query(Priority.priority):
263             self.priorities.append(p)
264
265     def complete(self, text, state):
266         if state == 0:
267             self.matches = []
268             n = len(text)
269             for word in self.priorities:
270                 if word[:n] == text:
271                     self.matches.append(word)
272         try:
273             return self.matches[state]
274         except IndexError:
275             return None
276
277 ################################################################################
278
279 def print_new (new, upload, indexed, file=sys.stdout):
280     check_valid(new)
281     broken = False
282     index = 0
283     for pkg in new.keys():
284         index += 1
285         section = new[pkg]["section"]
286         priority = new[pkg]["priority"]
287         if new[pkg]["section id"] == -1:
288             section += "[!]"
289             broken = True
290         if new[pkg]["priority id"] == -1:
291             priority += "[!]"
292             broken = True
293         if indexed:
294             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
295         else:
296             line = "%-20s %-20s %-20s" % (pkg, priority, section)
297         line = line.strip()+'\n'
298         file.write(line)
299     notes = get_new_comments(upload.pkg.changes.get("source"))
300     for note in notes:
301         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
302               % (note.author, note.version, note.notedate, note.comment)
303         print "-" * 72
304     return broken, len(notes) > 0
305
306 ################################################################################
307
308 def index_range (index):
309     if index == 1:
310         return "1"
311     else:
312         return "1-%s" % (index)
313
314 ################################################################################
315 ################################################################################
316
317 def edit_new (new, upload):
318     # Write the current data to a temporary file
319     (fd, temp_filename) = utils.temp_filename()
320     temp_file = os.fdopen(fd, 'w')
321     print_new (new, upload, indexed=0, file=temp_file)
322     temp_file.close()
323     # Spawn an editor on that file
324     editor = os.environ.get("EDITOR","vi")
325     result = os.system("%s %s" % (editor, temp_filename))
326     if result != 0:
327         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
328     # Read the edited data back in
329     temp_file = utils.open_file(temp_filename)
330     lines = temp_file.readlines()
331     temp_file.close()
332     os.unlink(temp_filename)
333     # Parse the new data
334     for line in lines:
335         line = line.strip()
336         if line == "":
337             continue
338         s = line.split()
339         # Pad the list if necessary
340         s[len(s):3] = [None] * (3-len(s))
341         (pkg, priority, section) = s[:3]
342         if not new.has_key(pkg):
343             utils.warn("Ignoring unknown package '%s'" % (pkg))
344         else:
345             # Strip off any invalid markers, print_new will readd them.
346             if section.endswith("[!]"):
347                 section = section[:-3]
348             if priority.endswith("[!]"):
349                 priority = priority[:-3]
350             for f in new[pkg]["files"]:
351                 upload.pkg.files[f]["section"] = section
352                 upload.pkg.files[f]["priority"] = priority
353             new[pkg]["section"] = section
354             new[pkg]["priority"] = priority
355
356 ################################################################################
357
358 def edit_index (new, upload, index):
359     priority = new[index]["priority"]
360     section = new[index]["section"]
361     ftype = new[index]["type"]
362     done = 0
363     while not done:
364         print "\t".join([index, priority, section])
365
366         answer = "XXX"
367         if ftype != "dsc":
368             prompt = "[B]oth, Priority, Section, Done ? "
369         else:
370             prompt = "[S]ection, Done ? "
371         edit_priority = edit_section = 0
372
373         while prompt.find(answer) == -1:
374             answer = utils.our_raw_input(prompt)
375             m = re_default_answer.match(prompt)
376             if answer == "":
377                 answer = m.group(1)
378             answer = answer[:1].upper()
379
380         if answer == 'P':
381             edit_priority = 1
382         elif answer == 'S':
383             edit_section = 1
384         elif answer == 'B':
385             edit_priority = edit_section = 1
386         elif answer == 'D':
387             done = 1
388
389         # Edit the priority
390         if edit_priority:
391             readline.set_completer(Priorities.complete)
392             got_priority = 0
393             while not got_priority:
394                 new_priority = utils.our_raw_input("New priority: ").strip()
395                 if new_priority not in Priorities.priorities:
396                     print "E: '%s' is not a valid priority, try again." % (new_priority)
397                 else:
398                     got_priority = 1
399                     priority = new_priority
400
401         # Edit the section
402         if edit_section:
403             readline.set_completer(Sections.complete)
404             got_section = 0
405             while not got_section:
406                 new_section = utils.our_raw_input("New section: ").strip()
407                 if new_section not in Sections.sections:
408                     print "E: '%s' is not a valid section, try again." % (new_section)
409                 else:
410                     got_section = 1
411                     section = new_section
412
413         # Reset the readline completer
414         readline.set_completer(None)
415
416     for f in new[index]["files"]:
417         upload.pkg.files[f]["section"] = section
418         upload.pkg.files[f]["priority"] = priority
419     new[index]["priority"] = priority
420     new[index]["section"] = section
421     return new
422
423 ################################################################################
424
425 def edit_overrides (new, upload, session):
426     print
427     done = 0
428     while not done:
429         print_new (new, upload, indexed=1)
430         new_index = {}
431         index = 0
432         for i in new.keys():
433             index += 1
434             new_index[index] = i
435
436         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
437
438         got_answer = 0
439         while not got_answer:
440             answer = utils.our_raw_input(prompt)
441             if not answer.isdigit():
442                 answer = answer[:1].upper()
443             if answer == "E" or answer == "D":
444                 got_answer = 1
445             elif re_isanum.match (answer):
446                 answer = int(answer)
447                 if (answer < 1) or (answer > index):
448                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
449                 else:
450                     got_answer = 1
451
452         if answer == 'E':
453             edit_new(new, upload)
454         elif answer == 'D':
455             done = 1
456         else:
457             edit_index (new, upload, new_index[answer])
458
459     return new
460
461 ################################################################################
462
463 def edit_note(note, upload, session):
464     # Write the current data to a temporary file
465     (fd, temp_filename) = utils.temp_filename()
466     editor = os.environ.get("EDITOR","vi")
467     answer = 'E'
468     while answer == 'E':
469         os.system("%s %s" % (editor, temp_filename))
470         temp_file = utils.open_file(temp_filename)
471         newnote = temp_file.read().rstrip()
472         temp_file.close()
473         print "New Note:"
474         print utils.prefix_multi_line_string(newnote,"  ")
475         prompt = "[D]one, Edit, Abandon, Quit ?"
476         answer = "XXX"
477         while prompt.find(answer) == -1:
478             answer = utils.our_raw_input(prompt)
479             m = re_default_answer.search(prompt)
480             if answer == "":
481                 answer = m.group(1)
482             answer = answer[:1].upper()
483     os.unlink(temp_filename)
484     if answer == 'A':
485         return
486     elif answer == 'Q':
487         end()
488         sys.exit(0)
489
490     comment = NewComment()
491     comment.package = upload.pkg.changes["source"]
492     comment.version = upload.pkg.changes["version"]
493     comment.comment = newnote
494     comment.author  = utils.whoami()
495     comment.trainee = bool(Options["Trainee"])
496     session.add(comment)
497     session.commit()
498
499 ################################################################################
500
501 def check_pkg (upload):
502     try:
503         less_fd = os.popen("less -R -", 'w', 0)
504         stdout_fd = sys.stdout
505         try:
506             sys.stdout = less_fd
507             changes = utils.parse_changes (upload.pkg.changes_file)
508             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
509             files = upload.pkg.files
510             for f in files.keys():
511                 if files[f].has_key("new"):
512                     ftype = files[f]["type"]
513                     if ftype == "deb":
514                         examine_package.check_deb(changes['distribution'], f)
515                     elif ftype == "dsc":
516                         examine_package.check_dsc(changes['distribution'], f)
517         finally:
518             examine_package.output_package_relations()
519             sys.stdout = stdout_fd
520     except IOError, e:
521         if e.errno == errno.EPIPE:
522             utils.warn("[examine_package] Caught EPIPE; skipping.")
523             pass
524         else:
525             raise
526     except KeyboardInterrupt:
527         utils.warn("[examine_package] Caught C-c; skipping.")
528         pass
529
530 ################################################################################
531
532 ## FIXME: horribly Debian specific
533
534 def do_bxa_notification(upload):
535     files = upload.pkg.files
536     summary = ""
537     for f in files.keys():
538         if files[f]["type"] == "deb":
539             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
540             summary += "\n"
541             summary += "Package: %s\n" % (control.Find("Package"))
542             summary += "Description: %s\n" % (control.Find("Description"))
543     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
544     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
545     utils.send_mail(bxa_mail)
546
547 ################################################################################
548
549 def add_overrides (new, upload, session):
550     changes = upload.pkg.changes
551     files = upload.pkg.files
552     srcpkg = changes.get("source")
553
554     for suite in changes["suite"].keys():
555         suite_id = get_suite(suite).suite_id
556         for pkg in new.keys():
557             component_id = get_component(new[pkg]["component"]).component_id
558             type_id = get_override_type(new[pkg]["type"]).overridetype_id
559             priority_id = new[pkg]["priority id"]
560             section_id = new[pkg]["section id"]
561             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
562             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
563                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
564             for f in new[pkg]["files"]:
565                 if files[f].has_key("new"):
566                     del files[f]["new"]
567             del new[pkg]
568
569     session.commit()
570
571     if Config().FindB("Dinstall::BXANotify"):
572         do_bxa_notification(upload)
573
574 ################################################################################
575
576 def prod_maintainer (note, upload):
577     cnf = Config()
578     # Here we prepare an editor and get them ready to prod...
579     (fd, temp_filename) = utils.temp_filename()
580     temp_file = os.fdopen(fd, 'w')
581     if len(note) > 0:
582         for line in note:
583             temp_file.write(line)
584     temp_file.close()
585     editor = os.environ.get("EDITOR","vi")
586     answer = 'E'
587     while answer == 'E':
588         os.system("%s %s" % (editor, temp_filename))
589         temp_fh = utils.open_file(temp_filename)
590         prod_message = "".join(temp_fh.readlines())
591         temp_fh.close()
592         print "Prod message:"
593         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
594         prompt = "[P]rod, Edit, Abandon, Quit ?"
595         answer = "XXX"
596         while prompt.find(answer) == -1:
597             answer = utils.our_raw_input(prompt)
598             m = re_default_answer.search(prompt)
599             if answer == "":
600                 answer = m.group(1)
601             answer = answer[:1].upper()
602     os.unlink(temp_filename)
603     if answer == 'A':
604         return
605     elif answer == 'Q':
606         end()
607         sys.exit(0)
608     # Otherwise, do the proding...
609     user_email_address = utils.whoami() + " <%s>" % (
610         cnf["Dinstall::MyAdminAddress"])
611
612     Subst = upload.Subst
613
614     Subst["__FROM_ADDRESS__"] = user_email_address
615     Subst["__PROD_MESSAGE__"] = prod_message
616     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
617
618     prod_mail_message = utils.TemplateSubst(
619         Subst,cnf["Dir::Templates"]+"/process-new.prod")
620
621     # Send the prod mail if appropriate
622     if not cnf["Dinstall::Options::No-Mail"]:
623         utils.send_mail(prod_mail_message)
624
625     print "Sent proding message"
626
627 ################################################################################
628
629 def do_new(upload, session):
630     print "NEW\n"
631     files = upload.pkg.files
632     changes = upload.pkg.changes
633     cnf = Config()
634
635     # Make a copy of distribution we can happily trample on
636     changes["suite"] = copy.copy(changes["distribution"])
637
638     # Fix up the list of target suites
639     for suite in changes["suite"].keys():
640         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
641         if override:
642             (olderr, newerr) = (get_suite(suite, session) == None,
643                                 get_suite(override, session) == None)
644             if olderr or newerr:
645                 (oinv, newinv) = ("", "")
646                 if olderr: oinv = "invalid "
647                 if newerr: ninv = "invalid "
648                 print "warning: overriding %ssuite %s to %ssuite %s" % (
649                         oinv, suite, ninv, override)
650             del changes["suite"][suite]
651             changes["suite"][override] = 1
652     # Validate suites
653     for suite in changes["suite"].keys():
654         if get_suite(suite, session) is None:
655             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
656
657     # The main NEW processing loop
658     done = 0
659     while not done:
660         # Find out what's new
661         new = determine_new(changes, files)
662
663         if not new:
664             break
665
666         answer = "XXX"
667         if Options["No-Action"] or Options["Automatic"]:
668             answer = 'S'
669
670         (broken, note) = print_new(new, upload, indexed=0)
671         prompt = ""
672
673         if not broken and not note:
674             prompt = "Add overrides, "
675         if broken:
676             print "W: [!] marked entries must be fixed before package can be processed."
677         if note:
678             print "W: note must be removed before package can be processed."
679             prompt += "RemOve all notes, Remove note, "
680
681         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
682
683         while prompt.find(answer) == -1:
684             answer = utils.our_raw_input(prompt)
685             m = re_default_answer.search(prompt)
686             if answer == "":
687                 answer = m.group(1)
688             answer = answer[:1].upper()
689
690         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
691             utils.warn("Trainees can't do that")
692             continue
693
694         if answer == 'A' and not Options["Trainee"]:
695             try:
696                 check_daily_lock()
697                 done = add_overrides (new, upload, session)
698                 Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (upload.pkg.changes_file)])
699             except CantGetLockError:
700                 print "Hello? Operator! Give me the number for 911!"
701                 print "Dinstall in the locked area, cant process packages, come back later"
702         elif answer == 'C':
703             check_pkg(upload)
704         elif answer == 'E' and not Options["Trainee"]:
705             new = edit_overrides (new, upload, session)
706         elif answer == 'M' and not Options["Trainee"]:
707             aborted = upload.do_reject(manual=1,
708                                        reject_message=Options["Manual-Reject"],
709                                        note=get_new_comments(changes.get("source", ""), session=session))
710             if not aborted:
711                 Logger.log([utils.getusername(), "NEW REJECT: %s" % (upload.pkg.changes_file)])
712                 os.unlink(upload.pkg.changes_file[:-8]+".dak")
713                 done = 1
714         elif answer == 'N':
715             edit_note(get_new_comments(changes.get("source", ""), session=session),
716                       upload, session)
717         elif answer == 'P' and not Options["Trainee"]:
718             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
719                             upload)
720             Logger.log([utils.getusername(), "NEW PROD: %s" % (upload.pkg.changes_file)])
721         elif answer == 'R' and not Options["Trainee"]:
722             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
723             if confirm == "y":
724                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
725                     session.delete(c)
726                 session.commit()
727         elif answer == 'O' and not Options["Trainee"]:
728             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
729             if confirm == "y":
730                 for c in get_new_comments(changes.get("source", ""), session=session):
731                     session.delete(c)
732                 session.commit()
733
734         elif answer == 'S':
735             done = 1
736         elif answer == 'Q':
737             end()
738             sys.exit(0)
739
740 ################################################################################
741 ################################################################################
742 ################################################################################
743
744 def usage (exit_code=0):
745     print """Usage: dak process-new [OPTION]... [CHANGES]...
746   -a, --automatic           automatic run
747   -h, --help                show this help and exit.
748   -m, --manual-reject=MSG   manual reject with `msg'
749   -n, --no-action           don't do anything
750   -t, --trainee             FTP Trainee mode
751   -V, --version             display the version number and exit"""
752     sys.exit(exit_code)
753
754 ################################################################################
755
756 def do_byhand(upload, session):
757     done = 0
758     while not done:
759         files = upload.pkg.files
760         will_install = 1
761         byhand = []
762
763         for f in files.keys():
764             if files[f]["type"] == "byhand":
765                 if os.path.exists(f):
766                     print "W: %s still present; please process byhand components and try again." % (f)
767                     will_install = 0
768                 else:
769                     byhand.append(f)
770
771         answer = "XXXX"
772         if Options["No-Action"]:
773             answer = "S"
774         if will_install:
775             if Options["Automatic"] and not Options["No-Action"]:
776                 answer = 'A'
777             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
778         else:
779             prompt = "Manual reject, [S]kip, Quit ?"
780
781         while prompt.find(answer) == -1:
782             answer = utils.our_raw_input(prompt)
783             m = re_default_answer.search(prompt)
784             if answer == "":
785                 answer = m.group(1)
786             answer = answer[:1].upper()
787
788         if answer == 'A':
789             try:
790                 check_daily_lock()
791                 done = 1
792                 for f in byhand:
793                     del files[f]
794                 Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
795             except CantGetLockError:
796                 print "Hello? Operator! Give me the number for 911!"
797                 print "Dinstall in the locked area, cant process packages, come back later"
798         elif answer == 'M':
799             Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (upload.pkg.changes_file)])
800             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
801             os.unlink(upload.pkg.changes_file[:-8]+".dak")
802             done = 1
803         elif answer == 'S':
804             done = 1
805         elif answer == 'Q':
806             end()
807             sys.exit(0)
808
809 ################################################################################
810
811 def check_daily_lock():
812     """
813     Raises CantGetLockError if the dinstall daily.lock exists.
814     """
815
816     cnf = Config()
817     try:
818         os.open(cnf["Process-New::DinstallLockFile"],
819                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
820     except OSError, e:
821         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
822             raise CantGetLockError
823
824     os.unlink(cnf["Process-New::DinstallLockFile"])
825
826
827 @contextlib.contextmanager
828 def lock_package(package):
829     """
830     Lock C{package} so that noone else jumps in processing it.
831
832     @type package: string
833     @param package: source package name to lock
834     """
835
836     path = os.path.join(Config()["Process-New::LockDir"], package)
837     try:
838         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
839     except OSError, e:
840         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
841             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
842             raise AlreadyLockedError, user
843
844     try:
845         yield fd
846     finally:
847         os.unlink(path)
848
849 def _accept(upload):
850     if Options["No-Action"]:
851         return
852     (summary, short_summary) = upload.build_summaries()
853     upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
854     os.unlink(upload.pkg.changes_file[:-8]+".dak")
855
856 def do_accept(upload):
857     print "ACCEPT"
858     cnf = Config()
859     if not Options["No-Action"]:
860         (summary, short_summary) = upload.build_summaries()
861
862     if cnf.FindB("Dinstall::SecurityQueueHandling"):
863          upload.dump_vars(cnf["Dir::Queue::Embargoed"])
864          upload.move_to_dir(cnf["Dir::Queue::Embargoed"])
865          upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
866          # Check for override disparities
867          upload.Subst["__SUMMARY__"] = summary
868     else:
869         # Just a normal upload, accept it...
870         _accept(upload)
871
872 def check_status(files):
873     new = byhand = 0
874     for f in files.keys():
875         if files[f]["type"] == "byhand":
876             byhand = 1
877         elif files[f].has_key("new"):
878             new = 1
879     return (new, byhand)
880
881 def do_pkg(changes_file, session):
882     u = Upload()
883     u.pkg.load_dot_dak(changes_file)
884     u.update_subst()
885
886     cnf = Config()
887     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
888     if cnf.has_key("Dinstall::Bcc"):
889         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
890     else:
891         u.Subst["__BCC__"] = bcc
892
893     files = u.pkg.files
894
895     try:
896         with lock_package(u.pkg.changes["source"]):
897             if not recheck(u, session):
898                 return
899
900             (new, byhand) = check_status(files)
901             if new or byhand:
902                 if new:
903                     do_new(u, session)
904                 if byhand:
905                     do_byhand(u, session)
906                 (new, byhand) = check_status(files)
907
908             if not new and not byhand:
909                 try:
910                     check_daily_lock()
911                     do_accept(u)
912                 except CantGetLockError:
913                     print "Hello? Operator! Give me the number for 911!"
914                     print "Dinstall in the locked area, cant process packages, come back later"
915     except AlreadyLockedError, e:
916         print "Seems to be locked by %s already, skipping..." % (e)
917
918 ################################################################################
919
920 def end():
921     accept_count = SummaryStats().accept_count
922     accept_bytes = SummaryStats().accept_bytes
923
924     if accept_count:
925         sets = "set"
926         if accept_count > 1:
927             sets = "sets"
928         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
929         Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
930
931     if not Options["No-Action"] and not Options["Trainee"]:
932         Logger.close()
933
934 ################################################################################
935
936 def main():
937     global Options, Logger, Sections, Priorities
938
939     cnf = Config()
940     session = DBConn().session()
941
942     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
943                  ('h',"help","Process-New::Options::Help"),
944                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
945                  ('t',"trainee","Process-New::Options::Trainee"),
946                  ('n',"no-action","Process-New::Options::No-Action")]
947
948     for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
949         if not cnf.has_key("Process-New::Options::%s" % (i)):
950             cnf["Process-New::Options::%s" % (i)] = ""
951
952     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
953     if len(changes_files) == 0:
954         changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
955
956     Options = cnf.SubTree("Process-New::Options")
957
958     if Options["Help"]:
959         usage()
960
961     if not Options["No-Action"]:
962         try:
963             Logger = daklog.Logger(cnf, "process-new")
964         except CantOpenError, e:
965             Options["Trainee"] = "True"
966
967     Sections = Section_Completer(session)
968     Priorities = Priority_Completer(session)
969     readline.parse_and_bind("tab: complete")
970
971     if len(changes_files) > 1:
972         sys.stderr.write("Sorting changes...\n")
973     changes_files = sort_changes(changes_files, session)
974
975     # Kill me now? **FIXME**
976     cnf["Dinstall::Options::No-Mail"] = ""
977
978     for changes_file in changes_files:
979         changes_file = utils.validate_changes_file_arg(changes_file, 0)
980         if not changes_file:
981             continue
982         print "\n" + changes_file
983
984         do_pkg (changes_file, session)
985
986     end()
987
988 ################################################################################
989
990 if __name__ == '__main__':
991     main()