]> git.decadent.org.uk Git - dak.git/blob - dak/process_new.py
remove UserExtensions
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2 # vim:set et ts=4 sw=4:
3
4 """ Handles NEW and BYHAND packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
10 """
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
15
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24
25 ################################################################################
26
27 # 23:12|<aj> I will not hush!
28 # 23:12|<elmo> :>
29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
30 # 23:13|<aj> I shall not be silenced!
31 # 23:13|<aj> The world shall know!
32 # 23:13|<aj> The world *must* know!
33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
34 # 23:13|<aj> yay powerpuff girls!!
35 # 23:13|<aj> buttercup's my favourite, who's yours?
36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
37 # 23:14|<aj> *AREN'T YOU*?!
38 # 23:15|<aj> I will not be treated like this.
39 # 23:15|<aj> I shall have my revenge.
40 # 23:15|<aj> I SHALL!!!
41
42 ################################################################################
43
44 from __future__ import with_statement
45
46 import copy
47 import errno
48 import os
49 import readline
50 import stat
51 import sys
52 import time
53 import contextlib
54 import pwd
55 import apt_pkg, apt_inst
56 import examine_package
57
58 from daklib import database
59 from daklib import daklog
60 from daklib import queue
61 from daklib import utils
62 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
63 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
64 from daklib.summarystats import SummaryStats
65
66 # Globals
67 Cnf = None       #: Configuration, apt_pkg.Configuration
68 Options = None
69 Upload = None
70 projectB = None  #: database connection, pgobject
71 Logger = None
72
73 Priorities = None
74 Sections = None
75
76 reject_message = ""
77
78 ################################################################################
79 ################################################################################
80 ################################################################################
81
82 def reject (str, prefix="Rejected: "):
83     global reject_message
84     if str:
85         reject_message += prefix + str + "\n"
86
87 def recheck():
88     global reject_message
89     files = Upload.pkg.files
90     reject_message = ""
91
92     for f in files.keys():
93         # The .orig.tar.gz can disappear out from under us is it's a
94         # duplicate of one in the archive.
95         if not files.has_key(f):
96             continue
97         # Check that the source still exists
98         if files[f]["type"] == "deb":
99             source_version = files[f]["source version"]
100             source_package = files[f]["source package"]
101             if not Upload.pkg.changes["architecture"].has_key("source") \
102                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
103                 source_epochless_version = re_no_epoch.sub('', source_version)
104                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
105                 found = 0
106                 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
107                     if Cnf.has_key("Dir::Queue::%s" % (q)):
108                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
109                             found = 1
110                 if not found:
111                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
112
113         # Version and file overwrite checks
114         if files[f]["type"] == "deb":
115             reject(Upload.check_binary_against_db(f), "")
116         elif files[f]["type"] == "dsc":
117             reject(Upload.check_source_against_db(f), "")
118             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
119             reject(reject_msg, "")
120
121     if reject_message.find("Rejected") != -1:
122         answer = "XXX"
123         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
124             answer = 'S'
125
126         print "REJECT\n" + reject_message,
127         prompt = "[R]eject, Skip, Quit ?"
128
129         while prompt.find(answer) == -1:
130             answer = utils.our_raw_input(prompt)
131             m = re_default_answer.match(prompt)
132             if answer == "":
133                 answer = m.group(1)
134             answer = answer[:1].upper()
135
136         if answer == 'R':
137             Upload.do_reject(0, reject_message)
138             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
139             return 0
140         elif answer == 'S':
141             return 0
142         elif answer == 'Q':
143             end()
144             sys.exit(0)
145
146     return 1
147
148 ################################################################################
149
150 def indiv_sg_compare (a, b):
151     """Sort by source name, source, version, 'have source', and
152        finally by filename."""
153     # Sort by source version
154     q = apt_pkg.VersionCompare(a["version"], b["version"])
155     if q:
156         return -q
157
158     # Sort by 'have source'
159     a_has_source = a["architecture"].get("source")
160     b_has_source = b["architecture"].get("source")
161     if a_has_source and not b_has_source:
162         return -1
163     elif b_has_source and not a_has_source:
164         return 1
165
166     return cmp(a["filename"], b["filename"])
167
168 ############################################################
169
170 def sg_compare (a, b):
171     a = a[1]
172     b = b[1]
173     """Sort by have note, source already in database and time of oldest upload."""
174     # Sort by have note
175     a_note_state = a["note_state"]
176     b_note_state = b["note_state"]
177     if a_note_state < b_note_state:
178         return -1
179     elif a_note_state > b_note_state:
180         return 1
181     # Sort by source already in database (descending)
182     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
183     if source_in_database:
184         return -source_in_database
185
186     # Sort by time of oldest upload
187     return cmp(a["oldest"], b["oldest"])
188
189 def sort_changes(changes_files):
190     """Sort into source groups, then sort each source group by version,
191     have source, filename.  Finally, sort the source groups by have
192     note, time of oldest upload of each source upload."""
193     if len(changes_files) == 1:
194         return changes_files
195
196     sorted_list = []
197     cache = {}
198     # Read in all the .changes files
199     for filename in changes_files:
200         try:
201             Upload.pkg.changes_file = filename
202             Upload.init_vars()
203             Upload.update_vars()
204             cache[filename] = copy.copy(Upload.pkg.changes)
205             cache[filename]["filename"] = filename
206         except:
207             sorted_list.append(filename)
208             break
209     # Divide the .changes into per-source groups
210     per_source = {}
211     for filename in cache.keys():
212         source = cache[filename]["source"]
213         if not per_source.has_key(source):
214             per_source[source] = {}
215             per_source[source]["list"] = []
216         per_source[source]["list"].append(cache[filename])
217     # Determine oldest time and have note status for each source group
218     for source in per_source.keys():
219         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
220         ql = q.getresult()
221         per_source[source]["source_in_database"] = len(ql)>0
222         source_list = per_source[source]["list"]
223         first = source_list[0]
224         oldest = os.stat(first["filename"])[stat.ST_MTIME]
225         have_note = 0
226         for d in per_source[source]["list"]:
227             mtime = os.stat(d["filename"])[stat.ST_MTIME]
228             if mtime < oldest:
229                 oldest = mtime
230             have_note += (database.has_new_comment(d["source"], d["version"], True))
231         per_source[source]["oldest"] = oldest
232         if not have_note:
233             per_source[source]["note_state"] = 0; # none
234         elif have_note < len(source_list):
235             per_source[source]["note_state"] = 1; # some
236         else:
237             per_source[source]["note_state"] = 2; # all
238         per_source[source]["list"].sort(indiv_sg_compare)
239     per_source_items = per_source.items()
240     per_source_items.sort(sg_compare)
241     for i in per_source_items:
242         for j in i[1]["list"]:
243             sorted_list.append(j["filename"])
244     return sorted_list
245
246 ################################################################################
247
248 class Section_Completer:
249     def __init__ (self):
250         self.sections = []
251         self.matches = []
252         q = projectB.query("SELECT section FROM section")
253         for i in q.getresult():
254             self.sections.append(i[0])
255
256     def complete(self, text, state):
257         if state == 0:
258             self.matches = []
259             n = len(text)
260             for word in self.sections:
261                 if word[:n] == text:
262                     self.matches.append(word)
263         try:
264             return self.matches[state]
265         except IndexError:
266             return None
267
268 ############################################################
269
270 class Priority_Completer:
271     def __init__ (self):
272         self.priorities = []
273         self.matches = []
274         q = projectB.query("SELECT priority FROM priority")
275         for i in q.getresult():
276             self.priorities.append(i[0])
277
278     def complete(self, text, state):
279         if state == 0:
280             self.matches = []
281             n = len(text)
282             for word in self.priorities:
283                 if word[:n] == text:
284                     self.matches.append(word)
285         try:
286             return self.matches[state]
287         except IndexError:
288             return None
289
290 ################################################################################
291
292 def print_new (new, indexed, file=sys.stdout):
293     queue.check_valid(new)
294     broken = 0
295     index = 0
296     for pkg in new.keys():
297         index += 1
298         section = new[pkg]["section"]
299         priority = new[pkg]["priority"]
300         if new[pkg]["section id"] == -1:
301             section += "[!]"
302             broken = 1
303         if new[pkg]["priority id"] == -1:
304             priority += "[!]"
305             broken = 1
306         if indexed:
307             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
308         else:
309             line = "%-20s %-20s %-20s" % (pkg, priority, section)
310         line = line.strip()+'\n'
311         file.write(line)
312     note = database.get_new_comments(Upload.pkg.changes.get("source"))
313     if len(note) > 0:
314         for line in note:
315             print line
316     return broken, note
317
318 ################################################################################
319
320 def index_range (index):
321     if index == 1:
322         return "1"
323     else:
324         return "1-%s" % (index)
325
326 ################################################################################
327 ################################################################################
328
329 def edit_new (new):
330     # Write the current data to a temporary file
331     (fd, temp_filename) = utils.temp_filename()
332     temp_file = os.fdopen(fd, 'w')
333     print_new (new, 0, temp_file)
334     temp_file.close()
335     # Spawn an editor on that file
336     editor = os.environ.get("EDITOR","vi")
337     result = os.system("%s %s" % (editor, temp_filename))
338     if result != 0:
339         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
340     # Read the edited data back in
341     temp_file = utils.open_file(temp_filename)
342     lines = temp_file.readlines()
343     temp_file.close()
344     os.unlink(temp_filename)
345     # Parse the new data
346     for line in lines:
347         line = line.strip()
348         if line == "":
349             continue
350         s = line.split()
351         # Pad the list if necessary
352         s[len(s):3] = [None] * (3-len(s))
353         (pkg, priority, section) = s[:3]
354         if not new.has_key(pkg):
355             utils.warn("Ignoring unknown package '%s'" % (pkg))
356         else:
357             # Strip off any invalid markers, print_new will readd them.
358             if section.endswith("[!]"):
359                 section = section[:-3]
360             if priority.endswith("[!]"):
361                 priority = priority[:-3]
362             for f in new[pkg]["files"]:
363                 Upload.pkg.files[f]["section"] = section
364                 Upload.pkg.files[f]["priority"] = priority
365             new[pkg]["section"] = section
366             new[pkg]["priority"] = priority
367
368 ################################################################################
369
370 def edit_index (new, index):
371     priority = new[index]["priority"]
372     section = new[index]["section"]
373     ftype = new[index]["type"]
374     done = 0
375     while not done:
376         print "\t".join([index, priority, section])
377
378         answer = "XXX"
379         if ftype != "dsc":
380             prompt = "[B]oth, Priority, Section, Done ? "
381         else:
382             prompt = "[S]ection, Done ? "
383         edit_priority = edit_section = 0
384
385         while prompt.find(answer) == -1:
386             answer = utils.our_raw_input(prompt)
387             m = re_default_answer.match(prompt)
388             if answer == "":
389                 answer = m.group(1)
390             answer = answer[:1].upper()
391
392         if answer == 'P':
393             edit_priority = 1
394         elif answer == 'S':
395             edit_section = 1
396         elif answer == 'B':
397             edit_priority = edit_section = 1
398         elif answer == 'D':
399             done = 1
400
401         # Edit the priority
402         if edit_priority:
403             readline.set_completer(Priorities.complete)
404             got_priority = 0
405             while not got_priority:
406                 new_priority = utils.our_raw_input("New priority: ").strip()
407                 if new_priority not in Priorities.priorities:
408                     print "E: '%s' is not a valid priority, try again." % (new_priority)
409                 else:
410                     got_priority = 1
411                     priority = new_priority
412
413         # Edit the section
414         if edit_section:
415             readline.set_completer(Sections.complete)
416             got_section = 0
417             while not got_section:
418                 new_section = utils.our_raw_input("New section: ").strip()
419                 if new_section not in Sections.sections:
420                     print "E: '%s' is not a valid section, try again." % (new_section)
421                 else:
422                     got_section = 1
423                     section = new_section
424
425         # Reset the readline completer
426         readline.set_completer(None)
427
428     for f in new[index]["files"]:
429         Upload.pkg.files[f]["section"] = section
430         Upload.pkg.files[f]["priority"] = priority
431     new[index]["priority"] = priority
432     new[index]["section"] = section
433     return new
434
435 ################################################################################
436
437 def edit_overrides (new):
438     print
439     done = 0
440     while not done:
441         print_new (new, 1)
442         new_index = {}
443         index = 0
444         for i in new.keys():
445             index += 1
446             new_index[index] = i
447
448         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
449
450         got_answer = 0
451         while not got_answer:
452             answer = utils.our_raw_input(prompt)
453             if not answer.isdigit():
454                 answer = answer[:1].upper()
455             if answer == "E" or answer == "D":
456                 got_answer = 1
457             elif re_isanum.match (answer):
458                 answer = int(answer)
459                 if (answer < 1) or (answer > index):
460                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
461                 else:
462                     got_answer = 1
463
464         if answer == 'E':
465             edit_new(new)
466         elif answer == 'D':
467             done = 1
468         else:
469             edit_index (new, new_index[answer])
470
471     return new
472
473 ################################################################################
474
475 def edit_note(note):
476     # Write the current data to a temporary file
477     (fd, temp_filename) = utils.temp_filename()
478     editor = os.environ.get("EDITOR","vi")
479     answer = 'E'
480     while answer == 'E':
481         os.system("%s %s" % (editor, temp_filename))
482         temp_file = utils.open_file(temp_filename)
483         newnote = temp_file.read().rstrip()
484         temp_file.close()
485         print "New Note:"
486         print utils.prefix_multi_line_string(newnote,"  ")
487         prompt = "[D]one, Edit, Abandon, Quit ?"
488         answer = "XXX"
489         while prompt.find(answer) == -1:
490             answer = utils.our_raw_input(prompt)
491             m = re_default_answer.search(prompt)
492             if answer == "":
493                 answer = m.group(1)
494             answer = answer[:1].upper()
495     os.unlink(temp_filename)
496     if answer == 'A':
497         return
498     elif answer == 'Q':
499         end()
500         sys.exit(0)
501
502     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"]))
503
504 ################################################################################
505
506 def check_pkg ():
507     try:
508         less_fd = os.popen("less -R -", 'w', 0)
509         stdout_fd = sys.stdout
510         try:
511             sys.stdout = less_fd
512             changes = utils.parse_changes (Upload.pkg.changes_file)
513             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
514             files = Upload.pkg.files
515             for f in files.keys():
516                 if files[f].has_key("new"):
517                     ftype = files[f]["type"]
518                     if ftype == "deb":
519                         examine_package.check_deb(changes['distribution'], f)
520                     elif ftype == "dsc":
521                         examine_package.check_dsc(changes['distribution'], f)
522         finally:
523             examine_package.output_package_relations()
524             sys.stdout = stdout_fd
525     except IOError, e:
526         if e.errno == errno.EPIPE:
527             utils.warn("[examine_package] Caught EPIPE; skipping.")
528             pass
529         else:
530             raise
531     except KeyboardInterrupt:
532         utils.warn("[examine_package] Caught C-c; skipping.")
533         pass
534
535 ################################################################################
536
537 ## FIXME: horribly Debian specific
538
539 def do_bxa_notification():
540     files = Upload.pkg.files
541     summary = ""
542     for f in files.keys():
543         if files[f]["type"] == "deb":
544             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
545             summary += "\n"
546             summary += "Package: %s\n" % (control.Find("Package"))
547             summary += "Description: %s\n" % (control.Find("Description"))
548     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
549     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
550     utils.send_mail(bxa_mail)
551
552 ################################################################################
553
554 def add_overrides (new):
555     changes = Upload.pkg.changes
556     files = Upload.pkg.files
557     srcpkg = changes.get("source")
558
559     projectB.query("BEGIN WORK")
560     for suite in changes["suite"].keys():
561         suite_id = database.get_suite_id(suite)
562         for pkg in new.keys():
563             component_id = database.get_component_id(new[pkg]["component"])
564             type_id = database.get_override_type_id(new[pkg]["type"])
565             priority_id = new[pkg]["priority id"]
566             section_id = new[pkg]["section id"]
567             Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
568             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
569             for f in new[pkg]["files"]:
570                 if files[f].has_key("new"):
571                     del files[f]["new"]
572             del new[pkg]
573
574     projectB.query("COMMIT WORK")
575
576     if Cnf.FindB("Dinstall::BXANotify"):
577         do_bxa_notification()
578
579 ################################################################################
580
581 def prod_maintainer (note):
582     # Here we prepare an editor and get them ready to prod...
583     (fd, temp_filename) = utils.temp_filename()
584     temp_file = os.fdopen(fd, 'w')
585     if len(note) > 0:
586         for line in note:
587             temp_file.write(line)
588     temp_file.close()
589     editor = os.environ.get("EDITOR","vi")
590     answer = 'E'
591     while answer == 'E':
592         os.system("%s %s" % (editor, temp_filename))
593         temp_fh = utils.open_file(temp_filename)
594         prod_message = "".join(temp_fh.readlines())
595         temp_fh.close()
596         print "Prod message:"
597         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
598         prompt = "[P]rod, Edit, Abandon, Quit ?"
599         answer = "XXX"
600         while prompt.find(answer) == -1:
601             answer = utils.our_raw_input(prompt)
602             m = re_default_answer.search(prompt)
603             if answer == "":
604                 answer = m.group(1)
605             answer = answer[:1].upper()
606     os.unlink(temp_filename)
607     if answer == 'A':
608         return
609     elif answer == 'Q':
610         end()
611         sys.exit(0)
612     # Otherwise, do the proding...
613     user_email_address = utils.whoami() + " <%s>" % (
614         Cnf["Dinstall::MyAdminAddress"])
615
616     Subst = Upload.Subst
617
618     Subst["__FROM_ADDRESS__"] = user_email_address
619     Subst["__PROD_MESSAGE__"] = prod_message
620     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
621
622     prod_mail_message = utils.TemplateSubst(
623         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
624
625     # Send the prod mail if appropriate
626     if not Cnf["Dinstall::Options::No-Mail"]:
627         utils.send_mail(prod_mail_message)
628
629     print "Sent proding message"
630
631 ################################################################################
632
633 def do_new():
634     print "NEW\n"
635     files = Upload.pkg.files
636     changes = Upload.pkg.changes
637
638     # Make a copy of distribution we can happily trample on
639     changes["suite"] = copy.copy(changes["distribution"])
640
641     # Fix up the list of target suites
642     for suite in changes["suite"].keys():
643         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
644         if override:
645             (olderr, newerr) = (database.get_suite_id(suite) == -1,
646               database.get_suite_id(override) == -1)
647             if olderr or newerr:
648                 (oinv, newinv) = ("", "")
649                 if olderr: oinv = "invalid "
650                 if newerr: ninv = "invalid "
651                 print "warning: overriding %ssuite %s to %ssuite %s" % (
652                         oinv, suite, ninv, override)
653             del changes["suite"][suite]
654             changes["suite"][override] = 1
655     # Validate suites
656     for suite in changes["suite"].keys():
657         suite_id = database.get_suite_id(suite)
658         if suite_id == -1:
659             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
660
661     # The main NEW processing loop
662     done = 0
663     while not done:
664         # Find out what's new
665         new = queue.determine_new(changes, files, projectB)
666
667         if not new:
668             break
669
670         answer = "XXX"
671         if Options["No-Action"] or Options["Automatic"]:
672             answer = 'S'
673
674         (broken, note) = print_new(new, 0)
675         prompt = ""
676
677         if not broken and not note:
678             prompt = "Add overrides, "
679         if broken:
680             print "W: [!] marked entries must be fixed before package can be processed."
681         if note:
682             print "W: note must be removed before package can be processed."
683             prompt += "RemOve all notes, Remove note, "
684
685         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
686
687         while prompt.find(answer) == -1:
688             answer = utils.our_raw_input(prompt)
689             m = re_default_answer.search(prompt)
690             if answer == "":
691                 answer = m.group(1)
692             answer = answer[:1].upper()
693
694         if answer == 'A' and not Options["Trainee"]:
695             try:
696                 check_daily_lock()
697                 done = add_overrides (new)
698                 Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)])
699             except CantGetLockError:
700                 print "Hello? Operator! Give me the number for 911!"
701                 print "Dinstall in the locked area, cant process packages, come back later"
702         elif answer == 'C':
703             check_pkg()
704         elif answer == 'E' and not Options["Trainee"]:
705             new = edit_overrides (new)
706         elif answer == 'M' and not Options["Trainee"]:
707             aborted = Upload.do_reject(manual=1,
708                                        reject_message=Options["Manual-Reject"],
709                                        note=database.get_new_comments(changes.get("source", "")))
710             if not aborted:
711                 Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)])
712                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
713                 done = 1
714         elif answer == 'N':
715             edit_note(database.get_new_comments(changes.get("source", "")))
716         elif answer == 'P' and not Options["Trainee"]:
717             prod_maintainer(database.get_new_comments(changes.get("source", "")))
718             Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)])
719         elif answer == 'R' and not Options["Trainee"]:
720             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
721             if confirm == "y":
722                 database.delete_new_comments(changes.get("source"), changes.get("version"))
723         elif answer == 'O' and not Options["Trainee"]:
724             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
725             if confirm == "y":
726                 database.delete_all_new_comments(changes.get("source"))
727         elif answer == 'S':
728             done = 1
729         elif answer == 'Q':
730             end()
731             sys.exit(0)
732
733 ################################################################################
734 ################################################################################
735 ################################################################################
736
737 def usage (exit_code=0):
738     print """Usage: dak process-new [OPTION]... [CHANGES]...
739   -a, --automatic           automatic run
740   -h, --help                show this help and exit.
741   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
742   -m, --manual-reject=MSG   manual reject with `msg'
743   -n, --no-action           don't do anything
744   -t, --trainee             FTP Trainee mode
745   -V, --version             display the version number and exit"""
746     sys.exit(exit_code)
747
748 ################################################################################
749
750 def init():
751     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
752
753     Cnf = utils.get_conf()
754
755     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
756                  ('h',"help","Process-New::Options::Help"),
757                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
758                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
759                  ('t',"trainee","Process-New::Options::Trainee"),
760                  ('n',"no-action","Process-New::Options::No-Action")]
761
762     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
763         if not Cnf.has_key("Process-New::Options::%s" % (i)):
764             Cnf["Process-New::Options::%s" % (i)] = ""
765
766     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
767     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
768         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
769
770     Options = Cnf.SubTree("Process-New::Options")
771
772     if Options["Help"]:
773         usage()
774
775     Upload = queue.Upload(Cnf)
776
777     if not Options["No-Action"]:
778         try:
779             Logger = Upload.Logger = daklog.Logger(Cnf, "process-new")
780         except CantOpenError, e:
781             Options["Trainee"] = "True"
782
783     projectB = Upload.projectB
784
785     Sections = Section_Completer()
786     Priorities = Priority_Completer()
787     readline.parse_and_bind("tab: complete")
788
789     return changes_files
790
791 ################################################################################
792
793 def do_byhand():
794     done = 0
795     while not done:
796         files = Upload.pkg.files
797         will_install = 1
798         byhand = []
799
800         for f in files.keys():
801             if files[f]["type"] == "byhand":
802                 if os.path.exists(f):
803                     print "W: %s still present; please process byhand components and try again." % (f)
804                     will_install = 0
805                 else:
806                     byhand.append(f)
807
808         answer = "XXXX"
809         if Options["No-Action"]:
810             answer = "S"
811         if will_install:
812             if Options["Automatic"] and not Options["No-Action"]:
813                 answer = 'A'
814             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
815         else:
816             prompt = "Manual reject, [S]kip, Quit ?"
817
818         while prompt.find(answer) == -1:
819             answer = utils.our_raw_input(prompt)
820             m = re_default_answer.search(prompt)
821             if answer == "":
822                 answer = m.group(1)
823             answer = answer[:1].upper()
824
825         if answer == 'A':
826             try:
827                 check_daily_lock()
828                 done = 1
829                 for f in byhand:
830                     del files[f]
831                 Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)])
832             except CantGetLockError:
833                 print "Hello? Operator! Give me the number for 911!"
834                 print "Dinstall in the locked area, cant process packages, come back later"
835         elif answer == 'M':
836             Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)])
837             Upload.do_reject(1, Options["Manual-Reject"])
838             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
839             done = 1
840         elif answer == 'S':
841             done = 1
842         elif answer == 'Q':
843             end()
844             sys.exit(0)
845
846 ################################################################################
847
848 def check_daily_lock():
849     """
850     Raises CantGetLockError if the dinstall daily.lock exists.
851     """
852
853     try:
854         os.open(Cnf["Process-New::DinstallLockFile"],  os.O_RDONLY | os.O_CREAT | os.O_EXCL)
855     except OSError, e:
856         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
857             raise CantGetLockError
858
859     os.unlink(Cnf["Process-New::DinstallLockFile"])
860
861
862 @contextlib.contextmanager
863 def lock_package(package):
864     """
865     Lock C{package} so that noone else jumps in processing it.
866
867     @type package: string
868     @param package: source package name to lock
869     """
870
871     path = os.path.join(Cnf["Process-New::LockDir"], package)
872     try:
873         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
874     except OSError, e:
875         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
876             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
877             raise AlreadyLockedError, user
878
879     try:
880         yield fd
881     finally:
882         os.unlink(path)
883
884 def move_to_dir (dest, perms=0660, changesperms=0664):
885     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
886     file_keys = Upload.pkg.files.keys()
887     for f in file_keys:
888         utils.move (f, dest, perms=perms)
889
890 def is_source_in_queue_dir(qdir):
891     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
892                 and x.endswith(".changes") ]
893     for entry in entries:
894         # read the .dak
895         u = queue.Upload(Cnf)
896         u.pkg.changes_file = os.path.join(qdir, entry)
897         u.update_vars()
898         if not u.pkg.changes["architecture"].has_key("source"):
899             # another binary upload, ignore
900             continue
901         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
902             # another version, ignore
903             continue
904         # found it!
905         return True
906     return False
907
908 def move_to_holding(suite, queue_dir):
909     print "Moving to %s holding area." % (suite.upper(),)
910     if Options["No-Action"]:
911         return
912     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
913     Upload.dump_vars(queue_dir)
914     move_to_dir(queue_dir, perms=0664)
915     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
916
917 def _accept():
918     if Options["No-Action"]:
919         return
920     (summary, short_summary) = Upload.build_summaries()
921     Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
922     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
923
924 def do_accept_stableupdate(suite, q):
925     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
926     if not Upload.pkg.changes["architecture"].has_key("source"):
927         # It is not a sourceful upload.  So its source may be either in p-u
928         # holding, in new, in accepted or already installed.
929         if is_source_in_queue_dir(queue_dir):
930             # It's in p-u holding, so move it there.
931             print "Binary-only upload, source in %s." % (q,)
932             move_to_holding(suite, queue_dir)
933         elif Upload.source_exists(Upload.pkg.changes["source"],
934                 Upload.pkg.changes["version"]):
935             # dak tells us that there is source available.  At time of
936             # writing this means that it is installed, so put it into
937             # accepted.
938             print "Binary-only upload, source installed."
939             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
940             _accept()
941         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
942             # The source is in accepted, the binary cleared NEW: accept it.
943             print "Binary-only upload, source in accepted."
944             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
945             _accept()
946         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
947             # It's in NEW.  We expect the source to land in p-u holding
948             # pretty soon.
949             print "Binary-only upload, source in new."
950             move_to_holding(suite, queue_dir)
951         elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
952             # It's in newstage.  Accept into the holding area
953             print "Binary-only upload, source in newstage."
954             Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
955             _accept()
956         else:
957             # No case applicable.  Bail out.  Return will cause the upload
958             # to be skipped.
959             print "ERROR"
960             print "Stable update failed.  Source not found."
961             return
962     else:
963         # We are handling a sourceful upload.  Move to accepted if currently
964         # in p-u holding and to p-u holding otherwise.
965         if is_source_in_queue_dir(queue_dir):
966             print "Sourceful upload in %s, accepting." % (q,)
967             _accept()
968         else:
969             move_to_holding(suite, queue_dir)
970
971 def do_accept():
972     print "ACCEPT"
973     if not Options["No-Action"]:
974         (summary, short_summary) = Upload.build_summaries()
975     if Cnf.FindB("Dinstall::SecurityQueueHandling"):
976         Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
977         move_to_dir(Cnf["Dir::Queue::Embargoed"])
978         Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
979         # Check for override disparities
980         Upload.Subst["__SUMMARY__"] = summary
981     else:
982         # Stable updates need to be copied to proposed-updates holding
983         # area instead of accepted.  Sourceful uploads need to go
984         # to it directly, binaries only if the source has not yet been
985         # accepted into p-u.
986         for suite, q in [("proposed-updates", "ProposedUpdates"),
987                 ("oldstable-proposed-updates", "OldProposedUpdates")]:
988             if not Upload.pkg.changes["distribution"].has_key(suite):
989                 continue
990             return do_accept_stableupdate(suite, q)
991         # Just a normal upload, accept it...
992         _accept()
993
994 def check_status(files):
995     new = byhand = 0
996     for f in files.keys():
997         if files[f]["type"] == "byhand":
998             byhand = 1
999         elif files[f].has_key("new"):
1000             new = 1
1001     return (new, byhand)
1002
1003 def do_pkg(changes_file):
1004     Upload.pkg.changes_file = changes_file
1005     Upload.init_vars()
1006     Upload.update_vars()
1007     Upload.update_subst()
1008     files = Upload.pkg.files
1009
1010     try:
1011         with lock_package(Upload.pkg.changes["source"]):
1012             if not recheck():
1013                 return
1014
1015             (new, byhand) = check_status(files)
1016             if new or byhand:
1017                 if new:
1018                     do_new()
1019                 if byhand:
1020                     do_byhand()
1021                 (new, byhand) = check_status(files)
1022
1023             if not new and not byhand:
1024                 try:
1025                     check_daily_lock()
1026                     do_accept()
1027                 except CantGetLockError:
1028                     print "Hello? Operator! Give me the number for 911!"
1029                     print "Dinstall in the locked area, cant process packages, come back later"
1030     except AlreadyLockedError, e:
1031         print "Seems to be locked by %s already, skipping..." % (e)
1032
1033 ################################################################################
1034
1035 def end():
1036     accept_count = SummaryStats().accept_count
1037     accept_bytes = SummaryStats().accept_bytes
1038
1039     if accept_count:
1040         sets = "set"
1041         if accept_count > 1:
1042             sets = "sets"
1043         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
1044         Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
1045
1046     if not Options["No-Action"] and not Options["Trainee"]:
1047         Logger.close()
1048
1049 ################################################################################
1050
1051 def do_comments(dir, opref, npref, line, fn):
1052     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
1053         lines = open("%s/%s" % (dir, comm)).readlines()
1054         if len(lines) == 0 or lines[0] != line + "\n": continue
1055         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
1056                                 and x.endswith(".changes") ]
1057         changes_files = sort_changes(changes_files)
1058         for f in changes_files:
1059             f = utils.validate_changes_file_arg(f, 0)
1060             if not f: continue
1061             print "\n" + f
1062             fn(f, "".join(lines[1:]))
1063
1064         if opref != npref and not Options["No-Action"]:
1065             newcomm = npref + comm[len(opref):]
1066             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1067
1068 ################################################################################
1069
1070 def comment_accept(changes_file, comments):
1071     Upload.pkg.changes_file = changes_file
1072     Upload.init_vars()
1073     Upload.update_vars()
1074     Upload.update_subst()
1075     files = Upload.pkg.files
1076
1077     if not recheck():
1078         return # dak wants to REJECT, crap
1079
1080     (new, byhand) = check_status(files)
1081     if not new and not byhand:
1082         do_accept()
1083
1084 ################################################################################
1085
1086 def comment_reject(changes_file, comments):
1087     Upload.pkg.changes_file = changes_file
1088     Upload.init_vars()
1089     Upload.update_vars()
1090     Upload.update_subst()
1091
1092     if not recheck():
1093         pass # dak has its own reasons to reject as well, which is fine
1094
1095     reject(comments)
1096     print "REJECT\n" + reject_message,
1097     if not Options["No-Action"]:
1098         Upload.do_reject(0, reject_message)
1099         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1100
1101 ################################################################################
1102
1103 def main():
1104     changes_files = init()
1105     if len(changes_files) > 50:
1106         sys.stderr.write("Sorting changes...\n")
1107     changes_files = sort_changes(changes_files)
1108
1109     # Kill me now? **FIXME**
1110     Cnf["Dinstall::Options::No-Mail"] = ""
1111     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1112     if Cnf.has_key("Dinstall::Bcc"):
1113         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1114     else:
1115         Upload.Subst["__BCC__"] = bcc
1116
1117     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1118     if commentsdir:
1119         if changes_files != []:
1120             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1121             sys.exit(1)
1122         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1123         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1124     else:
1125         for changes_file in changes_files:
1126             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1127             if not changes_file:
1128                 continue
1129             print "\n" + changes_file
1130
1131             do_pkg (changes_file)
1132
1133     end()
1134
1135 ################################################################################
1136
1137 if __name__ == '__main__':
1138     main()