]> git.decadent.org.uk Git - dak.git/blob - dak/process_new.py
53fed5f50acc0634c2569865ea315405515ad08f
[dak.git] / dak / process_new.py
1 #!/usr/bin/env python
2 # vim:set et ts=4 sw=4:
3
4 """ Handles NEW and BYHAND packages
5
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
10 """
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
15
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24
25 ################################################################################
26
27 # 23:12|<aj> I will not hush!
28 # 23:12|<elmo> :>
29 # 23:12|<aj> Where there is injustice in the world, I shall be there!
30 # 23:13|<aj> I shall not be silenced!
31 # 23:13|<aj> The world shall know!
32 # 23:13|<aj> The world *must* know!
33 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
34 # 23:13|<aj> yay powerpuff girls!!
35 # 23:13|<aj> buttercup's my favourite, who's yours?
36 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
37 # 23:14|<aj> *AREN'T YOU*?!
38 # 23:15|<aj> I will not be treated like this.
39 # 23:15|<aj> I shall have my revenge.
40 # 23:15|<aj> I SHALL!!!
41
42 ################################################################################
43
44 import copy
45 import errno
46 import os
47 import readline
48 import stat
49 import sys
50 import time
51 import apt_pkg, apt_inst
52 import examine_package
53 from daklib import database
54 from daklib import logging
55 from daklib import queue
56 from daklib import utils
57 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
58 from daklib.dak_exceptions import CantOpenError
59
60 # Globals
61 Cnf = None       #: Configuration, apt_pkg.Configuration
62 Options = None
63 Upload = None
64 projectB = None  #: database connection, pgobject
65 Logger = None
66
67 Priorities = None
68 Sections = None
69
70 reject_message = ""
71
72 ################################################################################
73 ################################################################################
74 ################################################################################
75
76 def reject (str, prefix="Rejected: "):
77     global reject_message
78     if str:
79         reject_message += prefix + str + "\n"
80
81 def recheck():
82     global reject_message
83     files = Upload.pkg.files
84     reject_message = ""
85
86     for f in files.keys():
87         # The .orig.tar.gz can disappear out from under us is it's a
88         # duplicate of one in the archive.
89         if not files.has_key(f):
90             continue
91         # Check that the source still exists
92         if files[f]["type"] == "deb":
93             source_version = files[f]["source version"]
94             source_package = files[f]["source package"]
95             if not Upload.pkg.changes["architecture"].has_key("source") \
96                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
97                 source_epochless_version = re_no_epoch.sub('', source_version)
98                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
99                 found = 0
100                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
101                     if Cnf.has_key("Dir::Queue::%s" % (q)):
102                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
103                             found = 1
104                 if not found:
105                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
106
107         # Version and file overwrite checks
108         if files[f]["type"] == "deb":
109             reject(Upload.check_binary_against_db(f), "")
110         elif files[f]["type"] == "dsc":
111             reject(Upload.check_source_against_db(f), "")
112             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
113             reject(reject_msg, "")
114
115     if reject_message.find("Rejected") != -1:
116         answer = "XXX"
117         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
118             answer = 'S'
119
120         print "REJECT\n" + reject_message,
121         prompt = "[R]eject, Skip, Quit ?"
122
123         while prompt.find(answer) == -1:
124             answer = utils.our_raw_input(prompt)
125             m = re_default_answer.match(prompt)
126             if answer == "":
127                 answer = m.group(1)
128             answer = answer[:1].upper()
129
130         if answer == 'R':
131             Upload.do_reject(0, reject_message)
132             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
133             return 0
134         elif answer == 'S':
135             return 0
136         elif answer == 'Q':
137             end()
138             sys.exit(0)
139
140     return 1
141
142 ################################################################################
143
144 def indiv_sg_compare (a, b):
145     """Sort by source name, source, version, 'have source', and
146        finally by filename."""
147     # Sort by source version
148     q = apt_pkg.VersionCompare(a["version"], b["version"])
149     if q:
150         return -q
151
152     # Sort by 'have source'
153     a_has_source = a["architecture"].get("source")
154     b_has_source = b["architecture"].get("source")
155     if a_has_source and not b_has_source:
156         return -1
157     elif b_has_source and not a_has_source:
158         return 1
159
160     return cmp(a["filename"], b["filename"])
161
162 ############################################################
163
164 def sg_compare (a, b):
165     a = a[1]
166     b = b[1]
167     """Sort by have note, source already in database and time of oldest upload."""
168     # Sort by have note
169     a_note_state = a["note_state"]
170     b_note_state = b["note_state"]
171     if a_note_state < b_note_state:
172         return -1
173     elif a_note_state > b_note_state:
174         return 1
175     # Sort by source already in database (descending)
176     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
177     if source_in_database:
178         return -source_in_database
179
180     # Sort by time of oldest upload
181     return cmp(a["oldest"], b["oldest"])
182
183 def sort_changes(changes_files):
184     """Sort into source groups, then sort each source group by version,
185     have source, filename.  Finally, sort the source groups by have
186     note, time of oldest upload of each source upload."""
187     if len(changes_files) == 1:
188         return changes_files
189
190     sorted_list = []
191     cache = {}
192     # Read in all the .changes files
193     for filename in changes_files:
194         try:
195             Upload.pkg.changes_file = filename
196             Upload.init_vars()
197             Upload.update_vars()
198             cache[filename] = copy.copy(Upload.pkg.changes)
199             cache[filename]["filename"] = filename
200         except:
201             sorted_list.append(filename)
202             break
203     # Divide the .changes into per-source groups
204     per_source = {}
205     for filename in cache.keys():
206         source = cache[filename]["source"]
207         if not per_source.has_key(source):
208             per_source[source] = {}
209             per_source[source]["list"] = []
210         per_source[source]["list"].append(cache[filename])
211     # Determine oldest time and have note status for each source group
212     for source in per_source.keys():
213         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
214         ql = q.getresult()
215         per_source[source]["source_in_database"] = len(ql)>0
216         source_list = per_source[source]["list"]
217         first = source_list[0]
218         oldest = os.stat(first["filename"])[stat.ST_MTIME]
219         have_note = 0
220         for d in per_source[source]["list"]:
221             mtime = os.stat(d["filename"])[stat.ST_MTIME]
222             if mtime < oldest:
223                 oldest = mtime
224             have_note += (database.has_new_comment(d["source"], d["version"]))
225         per_source[source]["oldest"] = oldest
226         if not have_note:
227             per_source[source]["note_state"] = 0; # none
228         elif have_note < len(source_list):
229             per_source[source]["note_state"] = 1; # some
230         else:
231             per_source[source]["note_state"] = 2; # all
232         per_source[source]["list"].sort(indiv_sg_compare)
233     per_source_items = per_source.items()
234     per_source_items.sort(sg_compare)
235     for i in per_source_items:
236         for j in i[1]["list"]:
237             sorted_list.append(j["filename"])
238     return sorted_list
239
240 ################################################################################
241
242 class Section_Completer:
243     def __init__ (self):
244         self.sections = []
245         self.matches = []
246         q = projectB.query("SELECT section FROM section")
247         for i in q.getresult():
248             self.sections.append(i[0])
249
250     def complete(self, text, state):
251         if state == 0:
252             self.matches = []
253             n = len(text)
254             for word in self.sections:
255                 if word[:n] == text:
256                     self.matches.append(word)
257         try:
258             return self.matches[state]
259         except IndexError:
260             return None
261
262 ############################################################
263
264 class Priority_Completer:
265     def __init__ (self):
266         self.priorities = []
267         self.matches = []
268         q = projectB.query("SELECT priority FROM priority")
269         for i in q.getresult():
270             self.priorities.append(i[0])
271
272     def complete(self, text, state):
273         if state == 0:
274             self.matches = []
275             n = len(text)
276             for word in self.priorities:
277                 if word[:n] == text:
278                     self.matches.append(word)
279         try:
280             return self.matches[state]
281         except IndexError:
282             return None
283
284 ################################################################################
285
286 def print_new (new, indexed, file=sys.stdout):
287     queue.check_valid(new)
288     broken = 0
289     index = 0
290     for pkg in new.keys():
291         index += 1
292         section = new[pkg]["section"]
293         priority = new[pkg]["priority"]
294         if new[pkg]["section id"] == -1:
295             section += "[!]"
296             broken = 1
297         if new[pkg]["priority id"] == -1:
298             priority += "[!]"
299             broken = 1
300         if indexed:
301             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
302         else:
303             line = "%-20s %-20s %-20s" % (pkg, priority, section)
304         line = line.strip()+'\n'
305         file.write(line)
306     note = database.get_new_comments(Upload.pkg.changes.get("source"))
307     if len(note) > 0:
308         for line in note:
309             print line
310     return broken, note
311
312 ################################################################################
313
314 def index_range (index):
315     if index == 1:
316         return "1"
317     else:
318         return "1-%s" % (index)
319
320 ################################################################################
321 ################################################################################
322
323 def edit_new (new):
324     # Write the current data to a temporary file
325     (fd, temp_filename) = utils.temp_filename()
326     temp_file = os.fdopen(fd, 'w')
327     print_new (new, 0, temp_file)
328     temp_file.close()
329     # Spawn an editor on that file
330     editor = os.environ.get("EDITOR","vi")
331     result = os.system("%s %s" % (editor, temp_filename))
332     if result != 0:
333         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
334     # Read the edited data back in
335     temp_file = utils.open_file(temp_filename)
336     lines = temp_file.readlines()
337     temp_file.close()
338     os.unlink(temp_filename)
339     # Parse the new data
340     for line in lines:
341         line = line.strip()
342         if line == "":
343             continue
344         s = line.split()
345         # Pad the list if necessary
346         s[len(s):3] = [None] * (3-len(s))
347         (pkg, priority, section) = s[:3]
348         if not new.has_key(pkg):
349             utils.warn("Ignoring unknown package '%s'" % (pkg))
350         else:
351             # Strip off any invalid markers, print_new will readd them.
352             if section.endswith("[!]"):
353                 section = section[:-3]
354             if priority.endswith("[!]"):
355                 priority = priority[:-3]
356             for f in new[pkg]["files"]:
357                 Upload.pkg.files[f]["section"] = section
358                 Upload.pkg.files[f]["priority"] = priority
359             new[pkg]["section"] = section
360             new[pkg]["priority"] = priority
361
362 ################################################################################
363
364 def edit_index (new, index):
365     priority = new[index]["priority"]
366     section = new[index]["section"]
367     ftype = new[index]["type"]
368     done = 0
369     while not done:
370         print "\t".join([index, priority, section])
371
372         answer = "XXX"
373         if ftype != "dsc":
374             prompt = "[B]oth, Priority, Section, Done ? "
375         else:
376             prompt = "[S]ection, Done ? "
377         edit_priority = edit_section = 0
378
379         while prompt.find(answer) == -1:
380             answer = utils.our_raw_input(prompt)
381             m = re_default_answer.match(prompt)
382             if answer == "":
383                 answer = m.group(1)
384             answer = answer[:1].upper()
385
386         if answer == 'P':
387             edit_priority = 1
388         elif answer == 'S':
389             edit_section = 1
390         elif answer == 'B':
391             edit_priority = edit_section = 1
392         elif answer == 'D':
393             done = 1
394
395         # Edit the priority
396         if edit_priority:
397             readline.set_completer(Priorities.complete)
398             got_priority = 0
399             while not got_priority:
400                 new_priority = utils.our_raw_input("New priority: ").strip()
401                 if new_priority not in Priorities.priorities:
402                     print "E: '%s' is not a valid priority, try again." % (new_priority)
403                 else:
404                     got_priority = 1
405                     priority = new_priority
406
407         # Edit the section
408         if edit_section:
409             readline.set_completer(Sections.complete)
410             got_section = 0
411             while not got_section:
412                 new_section = utils.our_raw_input("New section: ").strip()
413                 if new_section not in Sections.sections:
414                     print "E: '%s' is not a valid section, try again." % (new_section)
415                 else:
416                     got_section = 1
417                     section = new_section
418
419         # Reset the readline completer
420         readline.set_completer(None)
421
422     for f in new[index]["files"]:
423         Upload.pkg.files[f]["section"] = section
424         Upload.pkg.files[f]["priority"] = priority
425     new[index]["priority"] = priority
426     new[index]["section"] = section
427     return new
428
429 ################################################################################
430
431 def edit_overrides (new):
432     print
433     done = 0
434     while not done:
435         print_new (new, 1)
436         new_index = {}
437         index = 0
438         for i in new.keys():
439             index += 1
440             new_index[index] = i
441
442         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
443
444         got_answer = 0
445         while not got_answer:
446             answer = utils.our_raw_input(prompt)
447             if not answer.isdigit():
448                 answer = answer[:1].upper()
449             if answer == "E" or answer == "D":
450                 got_answer = 1
451             elif re_isanum.match (answer):
452                 answer = int(answer)
453                 if (answer < 1) or (answer > index):
454                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
455                 else:
456                     got_answer = 1
457
458         if answer == 'E':
459             edit_new(new)
460         elif answer == 'D':
461             done = 1
462         else:
463             edit_index (new, new_index[answer])
464
465     return new
466
467 ################################################################################
468
469 def edit_note(note):
470     # Write the current data to a temporary file
471     (fd, temp_filename) = utils.temp_filename()
472     editor = os.environ.get("EDITOR","vi")
473     answer = 'E'
474     while answer == 'E':
475         os.system("%s %s" % (editor, temp_filename))
476         temp_file = utils.open_file(temp_filename)
477         newnote = temp_file.read().rstrip()
478         temp_file.close()
479         print "New Note:"
480         print utils.prefix_multi_line_string(newnote,"  ")
481         prompt = "[D]one, Edit, Abandon, Quit ?"
482         answer = "XXX"
483         while prompt.find(answer) == -1:
484             answer = utils.our_raw_input(prompt)
485             m = re_default_answer.search(prompt)
486             if answer == "":
487                 answer = m.group(1)
488             answer = answer[:1].upper()
489     os.unlink(temp_filename)
490     if answer == 'A':
491         return
492     elif answer == 'Q':
493         end()
494         sys.exit(0)
495     database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
496
497 ################################################################################
498
499 def check_pkg ():
500     try:
501         less_fd = os.popen("less -R -", 'w', 0)
502         stdout_fd = sys.stdout
503         try:
504             sys.stdout = less_fd
505             changes = utils.parse_changes (Upload.pkg.changes_file)
506             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
507             files = Upload.pkg.files
508             for f in files.keys():
509                 if files[f].has_key("new"):
510                     ftype = files[f]["type"]
511                     if ftype == "deb":
512                         examine_package.check_deb(changes['distribution'], f)
513                     elif ftype == "dsc":
514                         examine_package.check_dsc(changes['distribution'], f)
515         finally:
516             examine_package.output_package_relations()
517             sys.stdout = stdout_fd
518     except IOError, e:
519         if e.errno == errno.EPIPE:
520             utils.warn("[examine_package] Caught EPIPE; skipping.")
521             pass
522         else:
523             raise
524     except KeyboardInterrupt:
525         utils.warn("[examine_package] Caught C-c; skipping.")
526         pass
527
528 ################################################################################
529
530 ## FIXME: horribly Debian specific
531
532 def do_bxa_notification():
533     files = Upload.pkg.files
534     summary = ""
535     for f in files.keys():
536         if files[f]["type"] == "deb":
537             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
538             summary += "\n"
539             summary += "Package: %s\n" % (control.Find("Package"))
540             summary += "Description: %s\n" % (control.Find("Description"))
541     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
542     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
543     utils.send_mail(bxa_mail)
544
545 ################################################################################
546
547 def add_overrides (new):
548     changes = Upload.pkg.changes
549     files = Upload.pkg.files
550
551     projectB.query("BEGIN WORK")
552     for suite in changes["suite"].keys():
553         suite_id = database.get_suite_id(suite)
554         for pkg in new.keys():
555             component_id = database.get_component_id(new[pkg]["component"])
556             type_id = database.get_override_type_id(new[pkg]["type"])
557             priority_id = new[pkg]["priority id"]
558             section_id = new[pkg]["section id"]
559             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
560             for f in new[pkg]["files"]:
561                 if files[f].has_key("new"):
562                     del files[f]["new"]
563             del new[pkg]
564
565     projectB.query("COMMIT WORK")
566
567     if Cnf.FindB("Dinstall::BXANotify"):
568         do_bxa_notification()
569
570 ################################################################################
571
572 def prod_maintainer (note):
573     # Here we prepare an editor and get them ready to prod...
574     (fd, temp_filename) = utils.temp_filename()
575     temp_file = os.fdopen(fd, 'w')
576     if len(note) > 0:
577         for line in note:
578             temp_file.write(line)
579     temp_file.close()
580     editor = os.environ.get("EDITOR","vi")
581     answer = 'E'
582     while answer == 'E':
583         os.system("%s %s" % (editor, temp_filename))
584         temp_fh = utils.open_file(temp_filename)
585         prod_message = "".join(temp_fh.readlines())
586         temp_fh.close()
587         print "Prod message:"
588         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
589         prompt = "[P]rod, Edit, Abandon, Quit ?"
590         answer = "XXX"
591         while prompt.find(answer) == -1:
592             answer = utils.our_raw_input(prompt)
593             m = re_default_answer.search(prompt)
594             if answer == "":
595                 answer = m.group(1)
596             answer = answer[:1].upper()
597         os.unlink(temp_filename)
598         if answer == 'A':
599             return
600         elif answer == 'Q':
601             end()
602             sys.exit(0)
603     # Otherwise, do the proding...
604     user_email_address = utils.whoami() + " <%s>" % (
605         Cnf["Dinstall::MyAdminAddress"])
606
607     Subst = Upload.Subst
608
609     Subst["__FROM_ADDRESS__"] = user_email_address
610     Subst["__PROD_MESSAGE__"] = prod_message
611     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
612
613     prod_mail_message = utils.TemplateSubst(
614         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
615
616     # Send the prod mail if appropriate
617     if not Cnf["Dinstall::Options::No-Mail"]:
618         utils.send_mail(prod_mail_message)
619
620     print "Sent proding message"
621
622 ################################################################################
623
624 def do_new():
625     print "NEW\n"
626     files = Upload.pkg.files
627     changes = Upload.pkg.changes
628
629     # Make a copy of distribution we can happily trample on
630     changes["suite"] = copy.copy(changes["distribution"])
631
632     # Fix up the list of target suites
633     for suite in changes["suite"].keys():
634         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
635         if override:
636             (olderr, newerr) = (database.get_suite_id(suite) == -1,
637               database.get_suite_id(override) == -1)
638             if olderr or newerr:
639                 (oinv, newinv) = ("", "")
640                 if olderr: oinv = "invalid "
641                 if newerr: ninv = "invalid "
642                 print "warning: overriding %ssuite %s to %ssuite %s" % (
643                         oinv, suite, ninv, override)
644             del changes["suite"][suite]
645             changes["suite"][override] = 1
646     # Validate suites
647     for suite in changes["suite"].keys():
648         suite_id = database.get_suite_id(suite)
649         if suite_id == -1:
650             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
651
652     # The main NEW processing loop
653     done = 0
654     while not done:
655         # Find out what's new
656         new = queue.determine_new(changes, files, projectB)
657
658         if not new:
659             break
660
661         answer = "XXX"
662         if Options["No-Action"] or Options["Automatic"]:
663             answer = 'S'
664
665         (broken, note) = print_new(new, 0)
666         prompt = ""
667
668         if not broken and not note:
669             prompt = "Add overrides, "
670         if broken:
671             print "W: [!] marked entries must be fixed before package can be processed."
672         if note:
673             print "W: note must be removed before package can be processed."
674             prompt += "Remove note, "
675
676         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
677
678         while prompt.find(answer) == -1:
679             answer = utils.our_raw_input(prompt)
680             m = re_default_answer.search(prompt)
681             if answer == "":
682                 answer = m.group(1)
683             answer = answer[:1].upper()
684
685         if answer == 'A' and not Options["Trainee"]:
686             done = add_overrides (new)
687         elif answer == 'C':
688             check_pkg()
689         elif answer == 'E' and not Options["Trainee"]:
690             new = edit_overrides (new)
691         elif answer == 'M' and not Options["Trainee"]:
692             aborted = Upload.do_reject(manual=1,
693                                        reject_message=Options["Manual-Reject"],
694                                        note=database.get_new_comments(changes.get("source", "")))
695             if not aborted:
696                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
697                 done = 1
698         elif answer == 'N':
699             edit_note(database.get_new_comments(changes.get("source", "")))
700         elif answer == 'P' and not Options["Trainee"]:
701             prod_maintainer(database.get_new_comments(changes.get("source", "")))
702         elif answer == 'R':
703             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
704             if confirm == "y":
705                 database.delete_new_comments(changes.get("source"), changes.get("version"))
706         elif answer == 'S':
707             done = 1
708         elif answer == 'Q':
709             end()
710             sys.exit(0)
711
712 ################################################################################
713 ################################################################################
714 ################################################################################
715
716 def usage (exit_code=0):
717     print """Usage: dak process-new [OPTION]... [CHANGES]...
718   -a, --automatic           automatic run
719   -h, --help                show this help and exit.
720   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
721   -m, --manual-reject=MSG   manual reject with `msg'
722   -n, --no-action           don't do anything
723   -t, --trainee             FTP Trainee mode
724   -V, --version             display the version number and exit"""
725     sys.exit(exit_code)
726
727 ################################################################################
728
729 def init():
730     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
731
732     Cnf = utils.get_conf()
733
734     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
735                  ('h',"help","Process-New::Options::Help"),
736                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
737                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
738                  ('t',"trainee","Process-New::Options::Trainee"),
739                  ('n',"no-action","Process-New::Options::No-Action")]
740
741     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
742         if not Cnf.has_key("Process-New::Options::%s" % (i)):
743             Cnf["Process-New::Options::%s" % (i)] = ""
744
745     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
746     if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
747         changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
748
749     Options = Cnf.SubTree("Process-New::Options")
750
751     if Options["Help"]:
752         usage()
753
754     Upload = queue.Upload(Cnf)
755
756     if not Options["No-Action"]:
757         try:
758             Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
759         except CantOpenError, e:
760             Options["Trainee"] = "Oh yes"
761
762     projectB = Upload.projectB
763
764     Sections = Section_Completer()
765     Priorities = Priority_Completer()
766     readline.parse_and_bind("tab: complete")
767
768     return changes_files
769
770 ################################################################################
771
772 def do_byhand():
773     done = 0
774     while not done:
775         files = Upload.pkg.files
776         will_install = 1
777         byhand = []
778
779         for f in files.keys():
780             if files[f]["type"] == "byhand":
781                 if os.path.exists(f):
782                     print "W: %s still present; please process byhand components and try again." % (f)
783                     will_install = 0
784                 else:
785                     byhand.append(f)
786
787         answer = "XXXX"
788         if Options["No-Action"]:
789             answer = "S"
790         if will_install:
791             if Options["Automatic"] and not Options["No-Action"]:
792                 answer = 'A'
793             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
794         else:
795             prompt = "Manual reject, [S]kip, Quit ?"
796
797         while prompt.find(answer) == -1:
798             answer = utils.our_raw_input(prompt)
799             m = re_default_answer.search(prompt)
800             if answer == "":
801                 answer = m.group(1)
802             answer = answer[:1].upper()
803
804         if answer == 'A':
805             done = 1
806             for f in byhand:
807                 del files[f]
808         elif answer == 'M':
809             Upload.do_reject(1, Options["Manual-Reject"])
810             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
811             done = 1
812         elif answer == 'S':
813             done = 1
814         elif answer == 'Q':
815             end()
816             sys.exit(0)
817
818 ################################################################################
819
820 def get_accept_lock():
821     retry = 0
822     while retry < 10:
823         try:
824             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
825             retry = 10
826         except OSError, e:
827             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
828                 retry += 1
829                 if (retry >= 10):
830                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
831                 else:
832                     print("Unable to get accepted lock (try %d of 10)" % retry)
833                 time.sleep(60)
834             else:
835                 raise
836
837 def move_to_dir (dest, perms=0660, changesperms=0664):
838     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
839     file_keys = Upload.pkg.files.keys()
840     for f in file_keys:
841         utils.move (f, dest, perms=perms)
842
843 def is_source_in_queue_dir(qdir):
844     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
845                 and x.endswith(".changes") ]
846     for entry in entries:
847         # read the .dak
848         u = queue.Upload(Cnf)
849         u.pkg.changes_file = os.path.join(qdir, entry)
850         u.update_vars()
851         if not u.pkg.changes["architecture"].has_key("source"):
852             # another binary upload, ignore
853             continue
854         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
855             # another version, ignore
856             continue
857         # found it!
858         return True
859     return False
860
861 def move_to_holding(suite, queue_dir):
862     print "Moving to %s holding area." % (suite.upper(),)
863     if Options["No-Action"]:
864         return
865     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
866     Upload.dump_vars(queue_dir)
867     move_to_dir(queue_dir, perms=0664)
868     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
869
870 def _accept():
871     if Options["No-Action"]:
872         return
873     (summary, short_summary) = Upload.build_summaries()
874     Upload.accept(summary, short_summary)
875     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
876
877 def do_accept_stableupdate(suite, q):
878     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
879     if not Upload.pkg.changes["architecture"].has_key("source"):
880         # It is not a sourceful upload.  So its source may be either in p-u
881         # holding, in new, in accepted or already installed.
882         if is_source_in_queue_dir(queue_dir):
883             # It's in p-u holding, so move it there.
884             print "Binary-only upload, source in %s." % (q,)
885             move_to_holding(suite, queue_dir)
886         elif Upload.source_exists(Upload.pkg.changes["source"],
887                 Upload.pkg.changes["version"]):
888             # dak tells us that there is source available.  At time of
889             # writing this means that it is installed, so put it into
890             # accepted.
891             print "Binary-only upload, source installed."
892             _accept()
893         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
894             # The source is in accepted, the binary cleared NEW: accept it.
895             print "Binary-only upload, source in accepted."
896             _accept()
897         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
898             # It's in NEW.  We expect the source to land in p-u holding
899             # pretty soon.
900             print "Binary-only upload, source in new."
901             move_to_holding(suite, queue_dir)
902         else:
903             # No case applicable.  Bail out.  Return will cause the upload
904             # to be skipped.
905             print "ERROR"
906             print "Stable update failed.  Source not found."
907             return
908     else:
909         # We are handling a sourceful upload.  Move to accepted if currently
910         # in p-u holding and to p-u holding otherwise.
911         if is_source_in_queue_dir(queue_dir):
912             print "Sourceful upload in %s, accepting." % (q,)
913             _accept()
914         else:
915             move_to_holding(suite, queue_dir)
916
917 def do_accept():
918     print "ACCEPT"
919     if not Options["No-Action"]:
920         get_accept_lock()
921         (summary, short_summary) = Upload.build_summaries()
922     try:
923         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
924             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
925             move_to_dir(Cnf["Dir::Queue::Embargoed"])
926             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
927             # Check for override disparities
928             Upload.Subst["__SUMMARY__"] = summary
929         else:
930             # Stable updates need to be copied to proposed-updates holding
931             # area instead of accepted.  Sourceful uploads need to go
932             # to it directly, binaries only if the source has not yet been
933             # accepted into p-u.
934             for suite, q in [("proposed-updates", "ProposedUpdates"),
935                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
936                 if not Upload.pkg.changes["distribution"].has_key(suite):
937                     continue
938                 return do_accept_stableupdate(suite, q)
939             # Just a normal upload, accept it...
940             _accept()
941     finally:
942         if not Options["No-Action"]:
943             os.unlink(Cnf["Process-New::AcceptedLockFile"])
944
945 def check_status(files):
946     new = byhand = 0
947     for f in files.keys():
948         if files[f]["type"] == "byhand":
949             byhand = 1
950         elif files[f].has_key("new"):
951             new = 1
952     return (new, byhand)
953
954 def do_pkg(changes_file):
955     Upload.pkg.changes_file = changes_file
956     Upload.init_vars()
957     Upload.update_vars()
958     Upload.update_subst()
959     files = Upload.pkg.files
960
961     if not recheck():
962         return
963
964     (new, byhand) = check_status(files)
965     if new or byhand:
966         if new:
967             do_new()
968         if byhand:
969             do_byhand()
970         (new, byhand) = check_status(files)
971
972     if not new and not byhand:
973         do_accept()
974
975 ################################################################################
976
977 def end():
978     accept_count = Upload.accept_count
979     accept_bytes = Upload.accept_bytes
980
981     if accept_count:
982         sets = "set"
983         if accept_count > 1:
984             sets = "sets"
985         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
986         Logger.log(["total",accept_count,accept_bytes])
987
988     if not Options["No-Action"] and not Options["Trainee"]:
989         Logger.close()
990
991 ################################################################################
992
993 def do_comments(dir, opref, npref, line, fn):
994     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
995         lines = open("%s/%s" % (dir, comm)).readlines()
996         if len(lines) == 0 or lines[0] != line + "\n": continue
997         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
998                                 and x.endswith(".changes") ]
999         changes_files = sort_changes(changes_files)
1000         for f in changes_files:
1001             f = utils.validate_changes_file_arg(f, 0)
1002             if not f: continue
1003             print "\n" + f
1004             fn(f, "".join(lines[1:]))
1005
1006         if opref != npref and not Options["No-Action"]:
1007             newcomm = npref + comm[len(opref):]
1008             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
1009
1010 ################################################################################
1011
1012 def comment_accept(changes_file, comments):
1013     Upload.pkg.changes_file = changes_file
1014     Upload.init_vars()
1015     Upload.update_vars()
1016     Upload.update_subst()
1017     files = Upload.pkg.files
1018
1019     if not recheck():
1020         return # dak wants to REJECT, crap
1021
1022     (new, byhand) = check_status(files)
1023     if not new and not byhand:
1024         do_accept()
1025
1026 ################################################################################
1027
1028 def comment_reject(changes_file, comments):
1029     Upload.pkg.changes_file = changes_file
1030     Upload.init_vars()
1031     Upload.update_vars()
1032     Upload.update_subst()
1033
1034     if not recheck():
1035         pass # dak has its own reasons to reject as well, which is fine
1036
1037     reject(comments)
1038     print "REJECT\n" + reject_message,
1039     if not Options["No-Action"]:
1040         Upload.do_reject(0, reject_message)
1041         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1042
1043 ################################################################################
1044
1045 def main():
1046     changes_files = init()
1047     if len(changes_files) > 50:
1048         sys.stderr.write("Sorting changes...\n")
1049     changes_files = sort_changes(changes_files)
1050
1051     # Kill me now? **FIXME**
1052     Cnf["Dinstall::Options::No-Mail"] = ""
1053     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1054     if Cnf.has_key("Dinstall::Bcc"):
1055         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1056     else:
1057         Upload.Subst["__BCC__"] = bcc
1058
1059     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1060     if commentsdir:
1061         if changes_files != []:
1062             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1063             sys.exit(1)
1064         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1065         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1066     else:
1067         for changes_file in changes_files:
1068             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1069             if not changes_file:
1070                 continue
1071             print "\n" + changes_file
1072             do_pkg (changes_file)
1073
1074     end()
1075
1076 ################################################################################
1077
1078 if __name__ == '__main__':
1079     main()