]> git.decadent.org.uk Git - dak.git/blob - dak/process_new.py
Merge commit 'lamby/master' into merge
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2 # vim:set et ts=4 sw=4:
3
4 """ Handles NEW and BYHAND packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
11 """
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
16
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25
26 ################################################################################
27
28 # 23:12|<aj> I will not hush!
29 # 23:12|<elmo> :>
30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
31 # 23:13|<aj> I shall not be silenced!
32 # 23:13|<aj> The world shall know!
33 # 23:13|<aj> The world *must* know!
34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
35 # 23:13|<aj> yay powerpuff girls!!
36 # 23:13|<aj> buttercup's my favourite, who's yours?
37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
38 # 23:14|<aj> *AREN'T YOU*?!
39 # 23:15|<aj> I will not be treated like this.
40 # 23:15|<aj> I shall have my revenge.
41 # 23:15|<aj> I SHALL!!!
42
43 ################################################################################
44
45 from __future__ import with_statement
46
47 import copy
48 import errno
49 import os
50 import readline
51 import stat
52 import sys
53 import time
54 import contextlib
55 import pwd
56 import apt_pkg, apt_inst
57 import examine_package
58
59 from daklib.dbconn import *
60 from daklib.queue import *
61 from daklib import daklog
62 from daklib import utils
63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
65 from daklib.summarystats import SummaryStats
66 from daklib.config import Config
67
68 # Globals
69 Options = None
70 Logger = None
71
72 Priorities = None
73 Sections = None
74
75 ################################################################################
76 ################################################################################
77 ################################################################################
78
79 def recheck(upload, session):
80     files = upload.pkg.files
81
82     cnf = Config()
83     for f in files.keys():
84         # The .orig.tar.gz can disappear out from under us is it's a
85         # duplicate of one in the archive.
86         if not files.has_key(f):
87             continue
88         # Check that the source still exists
89         if files[f]["type"] == "deb":
90             source_version = files[f]["source version"]
91             source_package = files[f]["source package"]
92             if not upload.pkg.changes["architecture"].has_key("source") \
93                and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()):
94                 source_epochless_version = re_no_epoch.sub('', source_version)
95                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
96                 found = 0
97                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
98                     if cnf.has_key("Dir::Queue::%s" % (q)):
99                         if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
100                             found = 1
101                 if not found:
102                     upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
103
104         # Version and file overwrite checks
105         if files[f]["type"] == "deb":
106             upload.check_binary_against_db(f, session)
107         elif files[f]["type"] == "dsc":
108             upload.check_source_against_db(f, session)
109             upload.check_dsc_against_db(f, session)
110
111     if len(upload.rejects) > 0:
112         answer = "XXX"
113         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
114             answer = 'S'
115
116         print "REJECT\n%s" % '\n'.join(upload.rejects)
117         prompt = "[R]eject, Skip, Quit ?"
118
119         while prompt.find(answer) == -1:
120             answer = utils.our_raw_input(prompt)
121             m = re_default_answer.match(prompt)
122             if answer == "":
123                 answer = m.group(1)
124             answer = answer[:1].upper()
125
126         if answer == 'R':
127             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
128             os.unlink(upload.pkg.changes_file[:-8]+".dak")
129             return 0
130         elif answer == 'S':
131             return 0
132         elif answer == 'Q':
133             end()
134             sys.exit(0)
135
136     return 1
137
138 ################################################################################
139
140 def indiv_sg_compare (a, b):
141     """Sort by source name, source, version, 'have source', and
142        finally by filename."""
143     # Sort by source version
144     q = apt_pkg.VersionCompare(a["version"], b["version"])
145     if q:
146         return -q
147
148     # Sort by 'have source'
149     a_has_source = a["architecture"].get("source")
150     b_has_source = b["architecture"].get("source")
151     if a_has_source and not b_has_source:
152         return -1
153     elif b_has_source and not a_has_source:
154         return 1
155
156     return cmp(a["filename"], b["filename"])
157
158 ############################################################
159
160 def sg_compare (a, b):
161     a = a[1]
162     b = b[1]
163     """Sort by have note, source already in database and time of oldest upload."""
164     # Sort by have note
165     a_note_state = a["note_state"]
166     b_note_state = b["note_state"]
167     if a_note_state < b_note_state:
168         return -1
169     elif a_note_state > b_note_state:
170         return 1
171     # Sort by source already in database (descending)
172     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
173     if source_in_database:
174         return -source_in_database
175
176     # Sort by time of oldest upload
177     return cmp(a["oldest"], b["oldest"])
178
179 def sort_changes(changes_files, session):
180     """Sort into source groups, then sort each source group by version,
181     have source, filename.  Finally, sort the source groups by have
182     note, time of oldest upload of each source upload."""
183     if len(changes_files) == 1:
184         return changes_files
185
186     sorted_list = []
187     cache = {}
188     # Read in all the .changes files
189     for filename in changes_files:
190         u = Upload()
191         try:
192             u.pkg.load_dot_dak(filename)
193             u.update_subst()
194             cache[filename] = copy.copy(u.pkg.changes)
195             cache[filename]["filename"] = filename
196         except:
197             sorted_list.append(filename)
198             break
199     # Divide the .changes into per-source groups
200     per_source = {}
201     for filename in cache.keys():
202         source = cache[filename]["source"]
203         if not per_source.has_key(source):
204             per_source[source] = {}
205             per_source[source]["list"] = []
206         per_source[source]["list"].append(cache[filename])
207     # Determine oldest time and have note status for each source group
208     for source in per_source.keys():
209         q = session.query(DBSource).filter_by(source = source).all()
210         per_source[source]["source_in_database"] = len(q)>0
211         source_list = per_source[source]["list"]
212         first = source_list[0]
213         oldest = os.stat(first["filename"])[stat.ST_MTIME]
214         have_note = 0
215         for d in per_source[source]["list"]:
216             mtime = os.stat(d["filename"])[stat.ST_MTIME]
217             if mtime < oldest:
218                 oldest = mtime
219             have_note += has_new_comment(d["source"], d["version"], session)
220         per_source[source]["oldest"] = oldest
221         if not have_note:
222             per_source[source]["note_state"] = 0; # none
223         elif have_note < len(source_list):
224             per_source[source]["note_state"] = 1; # some
225         else:
226             per_source[source]["note_state"] = 2; # all
227         per_source[source]["list"].sort(indiv_sg_compare)
228     per_source_items = per_source.items()
229     per_source_items.sort(sg_compare)
230     for i in per_source_items:
231         for j in i[1]["list"]:
232             sorted_list.append(j["filename"])
233     return sorted_list
234
235 ################################################################################
236
237 class Section_Completer:
238     def __init__ (self, session):
239         self.sections = []
240         self.matches = []
241         for s, in session.query(Section.section):
242             self.sections.append(s)
243
244     def complete(self, text, state):
245         if state == 0:
246             self.matches = []
247             n = len(text)
248             for word in self.sections:
249                 if word[:n] == text:
250                     self.matches.append(word)
251         try:
252             return self.matches[state]
253         except IndexError:
254             return None
255
256 ############################################################
257
258 class Priority_Completer:
259     def __init__ (self, session):
260         self.priorities = []
261         self.matches = []
262         for p, in session.query(Priority.priority):
263             self.priorities.append(p)
264
265     def complete(self, text, state):
266         if state == 0:
267             self.matches = []
268             n = len(text)
269             for word in self.priorities:
270                 if word[:n] == text:
271                     self.matches.append(word)
272         try:
273             return self.matches[state]
274         except IndexError:
275             return None
276
277 ################################################################################
278
279 def print_new (new, upload, indexed, file=sys.stdout):
280     check_valid(new)
281     broken = False
282     index = 0
283     for pkg in new.keys():
284         index += 1
285         section = new[pkg]["section"]
286         priority = new[pkg]["priority"]
287         if new[pkg]["section id"] == -1:
288             section += "[!]"
289             broken = True
290         if new[pkg]["priority id"] == -1:
291             priority += "[!]"
292             broken = True
293         if indexed:
294             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
295         else:
296             line = "%-20s %-20s %-20s" % (pkg, priority, section)
297         line = line.strip()+'\n'
298         file.write(line)
299     notes = get_new_comments(upload.pkg.changes.get("source"))
300     for note in notes:
301         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
302               % (note.author, note.version, note.notedate, note.comment)
303         print "-" * 72
304     return broken, len(notes) > 0
305
306 ################################################################################
307
308 def index_range (index):
309     if index == 1:
310         return "1"
311     else:
312         return "1-%s" % (index)
313
314 ################################################################################
315 ################################################################################
316
317 def edit_new (new, upload):
318     # Write the current data to a temporary file
319     (fd, temp_filename) = utils.temp_filename()
320     temp_file = os.fdopen(fd, 'w')
321     print_new (new, upload, indexed=0, file=temp_file)
322     temp_file.close()
323     # Spawn an editor on that file
324     editor = os.environ.get("EDITOR","vi")
325     result = os.system("%s %s" % (editor, temp_filename))
326     if result != 0:
327         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
328     # Read the edited data back in
329     temp_file = utils.open_file(temp_filename)
330     lines = temp_file.readlines()
331     temp_file.close()
332     os.unlink(temp_filename)
333     # Parse the new data
334     for line in lines:
335         line = line.strip()
336         if line == "":
337             continue
338         s = line.split()
339         # Pad the list if necessary
340         s[len(s):3] = [None] * (3-len(s))
341         (pkg, priority, section) = s[:3]
342         if not new.has_key(pkg):
343             utils.warn("Ignoring unknown package '%s'" % (pkg))
344         else:
345             # Strip off any invalid markers, print_new will readd them.
346             if section.endswith("[!]"):
347                 section = section[:-3]
348             if priority.endswith("[!]"):
349                 priority = priority[:-3]
350             for f in new[pkg]["files"]:
351                 upload.pkg.files[f]["section"] = section
352                 upload.pkg.files[f]["priority"] = priority
353             new[pkg]["section"] = section
354             new[pkg]["priority"] = priority
355
356 ################################################################################
357
358 def edit_index (new, upload, index):
359     priority = new[index]["priority"]
360     section = new[index]["section"]
361     ftype = new[index]["type"]
362     done = 0
363     while not done:
364         print "\t".join([index, priority, section])
365
366         answer = "XXX"
367         if ftype != "dsc":
368             prompt = "[B]oth, Priority, Section, Done ? "
369         else:
370             prompt = "[S]ection, Done ? "
371         edit_priority = edit_section = 0
372
373         while prompt.find(answer) == -1:
374             answer = utils.our_raw_input(prompt)
375             m = re_default_answer.match(prompt)
376             if answer == "":
377                 answer = m.group(1)
378             answer = answer[:1].upper()
379
380         if answer == 'P':
381             edit_priority = 1
382         elif answer == 'S':
383             edit_section = 1
384         elif answer == 'B':
385             edit_priority = edit_section = 1
386         elif answer == 'D':
387             done = 1
388
389         # Edit the priority
390         if edit_priority:
391             readline.set_completer(Priorities.complete)
392             got_priority = 0
393             while not got_priority:
394                 new_priority = utils.our_raw_input("New priority: ").strip()
395                 if new_priority not in Priorities.priorities:
396                     print "E: '%s' is not a valid priority, try again." % (new_priority)
397                 else:
398                     got_priority = 1
399                     priority = new_priority
400
401         # Edit the section
402         if edit_section:
403             readline.set_completer(Sections.complete)
404             got_section = 0
405             while not got_section:
406                 new_section = utils.our_raw_input("New section: ").strip()
407                 if new_section not in Sections.sections:
408                     print "E: '%s' is not a valid section, try again." % (new_section)
409                 else:
410                     got_section = 1
411                     section = new_section
412
413         # Reset the readline completer
414         readline.set_completer(None)
415
416     for f in new[index]["files"]:
417         upload.pkg.files[f]["section"] = section
418         upload.pkg.files[f]["priority"] = priority
419     new[index]["priority"] = priority
420     new[index]["section"] = section
421     return new
422
423 ################################################################################
424
425 def edit_overrides (new, upload, session):
426     print
427     done = 0
428     while not done:
429         print_new (new, upload, indexed=1)
430         new_index = {}
431         index = 0
432         for i in new.keys():
433             index += 1
434             new_index[index] = i
435
436         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
437
438         got_answer = 0
439         while not got_answer:
440             answer = utils.our_raw_input(prompt)
441             if not answer.isdigit():
442                 answer = answer[:1].upper()
443             if answer == "E" or answer == "D":
444                 got_answer = 1
445             elif re_isanum.match (answer):
446                 answer = int(answer)
447                 if (answer < 1) or (answer > index):
448                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
449                 else:
450                     got_answer = 1
451
452         if answer == 'E':
453             edit_new(new, upload)
454         elif answer == 'D':
455             done = 1
456         else:
457             edit_index (new, upload, new_index[answer])
458
459     return new
460
461 ################################################################################
462
463 def edit_note(note, upload, session):
464     # Write the current data to a temporary file
465     (fd, temp_filename) = utils.temp_filename()
466     editor = os.environ.get("EDITOR","vi")
467     answer = 'E'
468     while answer == 'E':
469         os.system("%s %s" % (editor, temp_filename))
470         temp_file = utils.open_file(temp_filename)
471         newnote = temp_file.read().rstrip()
472         temp_file.close()
473         print "New Note:"
474         print utils.prefix_multi_line_string(newnote,"  ")
475         prompt = "[D]one, Edit, Abandon, Quit ?"
476         answer = "XXX"
477         while prompt.find(answer) == -1:
478             answer = utils.our_raw_input(prompt)
479             m = re_default_answer.search(prompt)
480             if answer == "":
481                 answer = m.group(1)
482             answer = answer[:1].upper()
483     os.unlink(temp_filename)
484     if answer == 'A':
485         return
486     elif answer == 'Q':
487         end()
488         sys.exit(0)
489
490     comment = NewComment()
491     comment.package = upload.pkg.changes["source"]
492     comment.version = upload.pkg.changes["version"]
493     comment.comment = newnote
494     comment.author  = utils.whoami()
495     comment.trainee = bool(Options["Trainee"])
496     session.add(comment)
497     session.commit()
498
499 ################################################################################
500
501 def check_pkg (upload):
502     try:
503         less_fd = os.popen("less -R -", 'w', 0)
504         stdout_fd = sys.stdout
505         try:
506             sys.stdout = less_fd
507             changes = utils.parse_changes (upload.pkg.changes_file)
508             examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
509             files = upload.pkg.files
510             for f in files.keys():
511                 if files[f].has_key("new"):
512                     ftype = files[f]["type"]
513                     if ftype == "deb":
514                         examine_package.check_deb(changes['distribution'], f)
515                     elif ftype == "dsc":
516                         examine_package.check_dsc(changes['distribution'], f)
517         finally:
518             examine_package.output_package_relations()
519             sys.stdout = stdout_fd
520     except IOError, e:
521         if e.errno == errno.EPIPE:
522             utils.warn("[examine_package] Caught EPIPE; skipping.")
523             pass
524         else:
525             raise
526     except KeyboardInterrupt:
527         utils.warn("[examine_package] Caught C-c; skipping.")
528         pass
529
530 ################################################################################
531
532 ## FIXME: horribly Debian specific
533
534 def do_bxa_notification(upload):
535     files = upload.pkg.files
536     summary = ""
537     for f in files.keys():
538         if files[f]["type"] == "deb":
539             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
540             summary += "\n"
541             summary += "Package: %s\n" % (control.Find("Package"))
542             summary += "Description: %s\n" % (control.Find("Description"))
543     upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
544     bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
545     utils.send_mail(bxa_mail)
546
547 ################################################################################
548
549 def add_overrides (new, upload, session):
550     changes = upload.pkg.changes
551     files = upload.pkg.files
552     srcpkg = changes.get("source")
553
554     for suite in changes["suite"].keys():
555         suite_id = get_suite(suite).suite_id
556         for pkg in new.keys():
557             component_id = get_component(new[pkg]["component"]).component_id
558             type_id = get_override_type(new[pkg]["type"]).overridetype_id
559             priority_id = new[pkg]["priority id"]
560             section_id = new[pkg]["section id"]
561             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
562             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
563                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
564             for f in new[pkg]["files"]:
565                 if files[f].has_key("new"):
566                     del files[f]["new"]
567             del new[pkg]
568
569     session.commit()
570
571     if Config().FindB("Dinstall::BXANotify"):
572         do_bxa_notification(upload)
573
574 ################################################################################
575
576 def prod_maintainer (note, upload):
577     cnf = Config()
578     # Here we prepare an editor and get them ready to prod...
579     (fd, temp_filename) = utils.temp_filename()
580     temp_file = os.fdopen(fd, 'w')
581     if len(note) > 0:
582         for line in note:
583             temp_file.write(line)
584     temp_file.close()
585     editor = os.environ.get("EDITOR","vi")
586     answer = 'E'
587     while answer == 'E':
588         os.system("%s %s" % (editor, temp_filename))
589         temp_fh = utils.open_file(temp_filename)
590         prod_message = "".join(temp_fh.readlines())
591         temp_fh.close()
592         print "Prod message:"
593         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
594         prompt = "[P]rod, Edit, Abandon, Quit ?"
595         answer = "XXX"
596         while prompt.find(answer) == -1:
597             answer = utils.our_raw_input(prompt)
598             m = re_default_answer.search(prompt)
599             if answer == "":
600                 answer = m.group(1)
601             answer = answer[:1].upper()
602     os.unlink(temp_filename)
603     if answer == 'A':
604         return
605     elif answer == 'Q':
606         end()
607         sys.exit(0)
608     # Otherwise, do the proding...
609     user_email_address = utils.whoami() + " <%s>" % (
610         cnf["Dinstall::MyAdminAddress"])
611
612     Subst = upload.Subst
613
614     Subst["__FROM_ADDRESS__"] = user_email_address
615     Subst["__PROD_MESSAGE__"] = prod_message
616     Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
617
618     prod_mail_message = utils.TemplateSubst(
619         Subst,cnf["Dir::Templates"]+"/process-new.prod")
620
621     # Send the prod mail if appropriate
622     if not cnf["Dinstall::Options::No-Mail"]:
623         utils.send_mail(prod_mail_message)
624
625     print "Sent proding message"
626
627 ################################################################################
628
629 def do_new(upload, session):
630     print "NEW\n"
631     files = upload.pkg.files
632     changes = upload.pkg.changes
633     cnf = Config()
634
635     # Make a copy of distribution we can happily trample on
636     changes["suite"] = copy.copy(changes["distribution"])
637
638     # Fix up the list of target suites
639     for suite in changes["suite"].keys():
640         override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
641         if override:
642             (olderr, newerr) = (get_suite(suite, session) == None,
643                                 get_suite(override, session) == None)
644             if olderr or newerr:
645                 (oinv, newinv) = ("", "")
646                 if olderr: oinv = "invalid "
647                 if newerr: ninv = "invalid "
648                 print "warning: overriding %ssuite %s to %ssuite %s" % (
649                         oinv, suite, ninv, override)
650             del changes["suite"][suite]
651             changes["suite"][override] = 1
652     # Validate suites
653     for suite in changes["suite"].keys():
654         if get_suite(suite, session) is None:
655             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
656
657     # The main NEW processing loop
658     done = 0
659     while not done:
660         # Find out what's new
661         new = determine_new(changes, files)
662
663         if not new:
664             break
665
666         answer = "XXX"
667         if Options["No-Action"] or Options["Automatic"]:
668             answer = 'S'
669
670         (broken, note) = print_new(new, upload, indexed=0)
671         prompt = ""
672
673         if not broken and not note:
674             prompt = "Add overrides, "
675         if broken:
676             print "W: [!] marked entries must be fixed before package can be processed."
677         if note:
678             print "W: note must be removed before package can be processed."
679             prompt += "RemOve all notes, Remove note, "
680
681         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
682
683         while prompt.find(answer) == -1:
684             answer = utils.our_raw_input(prompt)
685             m = re_default_answer.search(prompt)
686             if answer == "":
687                 answer = m.group(1)
688             answer = answer[:1].upper()
689
690         if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
691             utils.warn("Trainees can't do that")
692             continue
693
694         if answer == 'A' and not Options["Trainee"]:
695             try:
696                 check_daily_lock()
697                 done = add_overrides (new, upload, session)
698                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
699             except CantGetLockError:
700                 print "Hello? Operator! Give me the number for 911!"
701                 print "Dinstall in the locked area, cant process packages, come back later"
702         elif answer == 'C':
703             check_pkg(upload)
704         elif answer == 'E' and not Options["Trainee"]:
705             new = edit_overrides (new, upload, session)
706         elif answer == 'M' and not Options["Trainee"]:
707             aborted = upload.do_reject(manual=1,
708                                        reject_message=Options["Manual-Reject"],
709                                        note=get_new_comments(changes.get("source", ""), session=session))
710             if not aborted:
711                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
712                 os.unlink(upload.pkg.changes_file[:-8]+".dak")
713                 done = 1
714         elif answer == 'N':
715             edit_note(get_new_comments(changes.get("source", ""), session=session),
716                       upload, session)
717         elif answer == 'P' and not Options["Trainee"]:
718             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
719                             upload)
720             Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
721         elif answer == 'R' and not Options["Trainee"]:
722             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
723             if confirm == "y":
724                 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
725                     session.delete(c)
726                 session.commit()
727         elif answer == 'O' and not Options["Trainee"]:
728             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
729             if confirm == "y":
730                 for c in get_new_comments(changes.get("source", ""), session=session):
731                     session.delete(c)
732                 session.commit()
733
734         elif answer == 'S':
735             done = 1
736         elif answer == 'Q':
737             end()
738             sys.exit(0)
739
740 ################################################################################
741 ################################################################################
742 ################################################################################
743
744 def usage (exit_code=0):
745     print """Usage: dak process-new [OPTION]... [CHANGES]...
746   -a, --automatic           automatic run
747   -h, --help                show this help and exit.
748   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
749   -m, --manual-reject=MSG   manual reject with `msg'
750   -n, --no-action           don't do anything
751   -t, --trainee             FTP Trainee mode
752   -V, --version             display the version number and exit"""
753     sys.exit(exit_code)
754
755 ################################################################################
756
757 def do_byhand(upload, session):
758     done = 0
759     while not done:
760         files = upload.pkg.files
761         will_install = 1
762         byhand = []
763
764         for f in files.keys():
765             if files[f]["type"] == "byhand":
766                 if os.path.exists(f):
767                     print "W: %s still present; please process byhand components and try again." % (f)
768                     will_install = 0
769                 else:
770                     byhand.append(f)
771
772         answer = "XXXX"
773         if Options["No-Action"]:
774             answer = "S"
775         if will_install:
776             if Options["Automatic"] and not Options["No-Action"]:
777                 answer = 'A'
778             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
779         else:
780             prompt = "Manual reject, [S]kip, Quit ?"
781
782         while prompt.find(answer) == -1:
783             answer = utils.our_raw_input(prompt)
784             m = re_default_answer.search(prompt)
785             if answer == "":
786                 answer = m.group(1)
787             answer = answer[:1].upper()
788
789         if answer == 'A':
790             try:
791                 check_daily_lock()
792                 done = 1
793                 for f in byhand:
794                     del files[f]
795                 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
796             except CantGetLockError:
797                 print "Hello? Operator! Give me the number for 911!"
798                 print "Dinstall in the locked area, cant process packages, come back later"
799         elif answer == 'M':
800             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
801             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
802             os.unlink(upload.pkg.changes_file[:-8]+".dak")
803             done = 1
804         elif answer == 'S':
805             done = 1
806         elif answer == 'Q':
807             end()
808             sys.exit(0)
809
810 ################################################################################
811
812 def check_daily_lock():
813     """
814     Raises CantGetLockError if the dinstall daily.lock exists.
815     """
816
817     cnf = Config()
818     try:
819         os.open(cnf["Process-New::DinstallLockFile"],
820                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
821     except OSError, e:
822         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
823             raise CantGetLockError
824
825     os.unlink(cnf["Process-New::DinstallLockFile"])
826
827
828 @contextlib.contextmanager
829 def lock_package(package):
830     """
831     Lock C{package} so that noone else jumps in processing it.
832
833     @type package: string
834     @param package: source package name to lock
835     """
836
837     path = os.path.join(Config()["Process-New::LockDir"], package)
838     try:
839         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
840     except OSError, e:
841         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
842             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
843             raise AlreadyLockedError, user
844
845     try:
846         yield fd
847     finally:
848         os.unlink(path)
849
850 # def move_to_dir (upload, dest, perms=0660, changesperms=0664):
851 #     utils.move (upload.pkg.changes_file, dest, perms=changesperms)
852 #     file_keys = upload.pkg.files.keys()
853 #     for f in file_keys:
854 #         utils.move (f, dest, perms=perms)
855
856 # def is_source_in_queue_dir(qdir):
857 #     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
858 #                 and x.endswith(".changes") ]
859 #     for entry in entries:
860 #         # read the .dak
861 #         u = queue.Upload(Cnf)
862 #         u.pkg.changes_file = os.path.join(qdir, entry)
863 #         u.update_vars()
864 #         if not u.pkg.changes["architecture"].has_key("source"):
865 #             # another binary upload, ignore
866 #             continue
867 #         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
868 #             # another version, ignore
869 #             continue
870 #         # found it!
871 #         return True
872 #     return False
873
874 # def move_to_holding(suite, queue_dir):
875 #     print "Moving to %s holding area." % (suite.upper(),)
876 #     if Options["No-Action"]:
877 #       return
878 #     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
879 #     Upload.dump_vars(queue_dir)
880 #     move_to_dir(queue_dir, perms=0664)
881 #     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
882
883 def _accept(upload):
884     if Options["No-Action"]:
885         return
886     (summary, short_summary) = upload.build_summaries()
887     upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
888     os.unlink(upload.pkg.changes_file[:-8]+".dak")
889
890 # def do_accept_stableupdate(upload,suite, q):
891 #     cnf = Config()
892 #     queue_dir = cnf["Dir::Queue::%s" % (q,)]
893 #     if not upload.pkg.changes["architecture"].has_key("source"):
894 #         # It is not a sourceful upload.  So its source may be either in p-u
895 #         # holding, in new, in accepted or already installed.
896 #         if is_source_in_queue_dir(queue_dir):
897 #             # It's in p-u holding, so move it there.
898 #             print "Binary-only upload, source in %s." % (q,)
899 #             move_to_holding(suite, queue_dir)
900 #         elif Upload.source_exists(Upload.pkg.changes["source"],
901 #                 Upload.pkg.changes["version"]):
902 #             # dak tells us that there is source available.  At time of
903 #             # writing this means that it is installed, so put it into
904 #             # accepted.
905 #             print "Binary-only upload, source installed."
906 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
907 #             _accept()
908 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
909 #             # The source is in accepted, the binary cleared NEW: accept it.
910 #             print "Binary-only upload, source in accepted."
911 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
912 #             _accept()
913 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
914 #             # It's in NEW.  We expect the source to land in p-u holding
915 #             # pretty soon.
916 #             print "Binary-only upload, source in new."
917 #             move_to_holding(suite, queue_dir)
918 #         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
919 #             # It's in newstage.  Accept into the holding area
920 #             print "Binary-only upload, source in newstage."
921 #             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
922 #             _accept()
923 #         else:
924 #             # No case applicable.  Bail out.  Return will cause the upload
925 #             # to be skipped.
926 #             print "ERROR"
927 #             print "Stable update failed.  Source not found."
928 #             return
929 #     else:
930 #         # We are handling a sourceful upload.  Move to accepted if currently
931 #         # in p-u holding and to p-u holding otherwise.
932 #         if is_source_in_queue_dir(queue_dir):
933 #             print "Sourceful upload in %s, accepting." % (q,)
934 #             _accept()
935 #         else:
936 #             move_to_holding(suite, queue_dir)
937
938 def do_accept(upload):
939     print "ACCEPT"
940     cnf = Config()
941     if not Options["No-Action"]:
942         (summary, short_summary) = upload.build_summaries()
943 #     if cnf.FindB("Dinstall::SecurityQueueHandling"):
944 #         upload.dump_vars(cnf["Dir::Queue::Embargoed"])
945 #         move_to_dir(cnf["Dir::Queue::Embargoed"])
946 #         upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
947 #         # Check for override disparities
948 #         upload.Subst["__SUMMARY__"] = summary
949 #     else:
950         # Stable updates need to be copied to proposed-updates holding
951         # area instead of accepted.  Sourceful uploads need to go
952         # to it directly, binaries only if the source has not yet been
953         # accepted into p-u.
954         for suite, q in [("proposed-updates", "ProposedUpdates"),
955                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
956             if not upload.pkg.changes["distribution"].has_key(suite):
957                 continue
958             utils.fubar("stable accept not supported yet")
959 #            return do_accept_stableupdate(suite, q)
960         # Just a normal upload, accept it...
961         _accept(upload)
962
963 def check_status(files):
964     new = byhand = 0
965     for f in files.keys():
966         if files[f]["type"] == "byhand":
967             byhand = 1
968         elif files[f].has_key("new"):
969             new = 1
970     return (new, byhand)
971
972 def do_pkg(changes_file, session):
973     u = Upload()
974     u.pkg.load_dot_dak(changes_file)
975     u.update_subst()
976
977     cnf = Config()
978     bcc = "X-DAK: dak process-new"
979     if cnf.has_key("Dinstall::Bcc"):
980         u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
981     else:
982         u.Subst["__BCC__"] = bcc
983
984     files = u.pkg.files
985
986     try:
987         with lock_package(u.pkg.changes["source"]):
988             if not recheck(u, session):
989                 return
990
991             (new, byhand) = check_status(files)
992             if new or byhand:
993                 if new:
994                     do_new(u, session)
995                 if byhand:
996                     do_byhand(u, session)
997                 (new, byhand) = check_status(files)
998
999             if not new and not byhand:
1000                 try:
1001                     check_daily_lock()
1002                     do_accept(u)
1003                 except CantGetLockError:
1004                     print "Hello? Operator! Give me the number for 911!"
1005                     print "Dinstall in the locked area, cant process packages, come back later"
1006     except AlreadyLockedError, e:
1007         print "Seems to be locked by %s already, skipping..." % (e)
1008
1009 ################################################################################
1010
1011 def end():
1012     accept_count = SummaryStats().accept_count
1013     accept_bytes = SummaryStats().accept_bytes
1014
1015     if accept_count:
1016         sets = "set"
1017         if accept_count > 1:
1018             sets = "sets"
1019         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1020         Logger.log(["total",accept_count,accept_bytes])
1021
1022     if not Options["No-Action"] and not Options["Trainee"]:
1023         Logger.close()
1024
1025 ################################################################################
1026
1027 # def do_comments(dir, opref, npref, line, fn):
1028 #     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1029 #         lines = open("%s/%s" % (dir, comm)).readlines()
1030 #         if len(lines) == 0 or lines[0] != line + "\n": continue
1031 #         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1032 #                                 and x.endswith(".changes") ]
1033 #         changes_files = sort_changes(changes_files)
1034 #         for f in changes_files:
1035 #             f = utils.validate_changes_file_arg(f, 0)
1036 #             if not f: continue
1037 #             print "\n" + f
1038 #             fn(f, "".join(lines[1:]))
1039
1040 #         if opref != npref and not Options["No-Action"]:
1041 #             newcomm = npref + comm[len(opref):]
1042 #             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1043
1044 # ################################################################################
1045
1046 # def comment_accept(changes_file, comments):
1047 #     Upload.pkg.changes_file = changes_file
1048 #     Upload.init_vars()
1049 #     Upload.update_vars()
1050 #     Upload.update_subst()
1051 #     files = Upload.pkg.files
1052
1053 #     if not recheck():
1054 #         return # dak wants to REJECT, crap
1055
1056 #     (new, byhand) = check_status(files)
1057 #     if not new and not byhand:
1058 #         do_accept()
1059
1060 # ################################################################################
1061
1062 # def comment_reject(changes_file, comments):
1063 #     Upload.pkg.changes_file = changes_file
1064 #     Upload.init_vars()
1065 #     Upload.update_vars()
1066 #     Upload.update_subst()
1067
1068 #     if not recheck():
1069 #         pass # dak has its own reasons to reject as well, which is fine
1070
1071 #     reject(comments)
1072 #     print "REJECT\n" + reject_message,
1073 #     if not Options["No-Action"]:
1074 #         Upload.do_reject(0, reject_message)
1075 #         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1076
1077 ################################################################################
1078
1079 def main():
1080     global Options, Logger, Sections, Priorities
1081
1082     cnf = Config()
1083     session = DBConn().session()
1084
1085     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
1086                  ('h',"help","Process-New::Options::Help"),
1087                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
1088                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
1089                  ('t',"trainee","Process-New::Options::Trainee"),
1090                  ('n',"no-action","Process-New::Options::No-Action")]
1091
1092     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
1093         if not cnf.has_key("Process-New::Options::%s" % (i)):
1094             cnf["Process-New::Options::%s" % (i)] = ""
1095
1096     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
1097     if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""):
1098         changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
1099
1100     Options = cnf.SubTree("Process-New::Options")
1101
1102     if Options["Help"]:
1103         usage()
1104
1105     if not Options["No-Action"]:
1106         try:
1107             Logger = daklog.Logger(cnf, "process-new")
1108         except CantOpenError, e:
1109             Options["Trainee"] = "True"
1110
1111     Sections = Section_Completer(session)
1112     Priorities = Priority_Completer(session)
1113     readline.parse_and_bind("tab: complete")
1114
1115     if len(changes_files) > 1:
1116         sys.stderr.write("Sorting changes...\n")
1117     changes_files = sort_changes(changes_files, session)
1118
1119     # Kill me now? **FIXME**
1120     cnf["Dinstall::Options::No-Mail"] = ""
1121
1122 #     commentsdir = cnf.get("Process-New::Options::Comments-Dir","")
1123 #     if commentsdir:
1124 #        if changes_files != []:
1125 #            sys.stderr.write("Can't specify any changes files if working with comments-dir")
1126 #            sys.exit(1)
1127 #        do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1128 #        do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1129 #     else:
1130     if True:
1131         for changes_file in changes_files:
1132             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1133             if not changes_file:
1134                 continue
1135             print "\n" + changes_file
1136
1137             do_pkg (changes_file, session)
1138
1139     end()
1140
1141 ################################################################################
1142
1143 if __name__ == '__main__':
1144     main()