3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # 23:12|<aj> I will not hush!
24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
25 # 23:13|<aj> I shall not be silenced!
26 # 23:13|<aj> The world shall know!
27 # 23:13|<aj> The world *must* know!
28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
29 # 23:13|<aj> yay powerpuff girls!!
30 # 23:13|<aj> buttercup's my favourite, who's yours?
31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
32 # 23:14|<aj> *AREN'T YOU*?!
33 # 23:15|<aj> I will not be treated like this.
34 # 23:15|<aj> I shall have my revenge.
35 # 23:15|<aj> I SHALL!!!
37 ################################################################################
39 import copy, errno, os, readline, stat, sys, time
40 import apt_pkg, apt_inst
41 import examine_package
42 import daklib.database
59 ################################################################################
60 ################################################################################
61 ################################################################################
63 def reject (str, prefix="Rejected: "):
66 reject_message += prefix + str + "\n"
70 files = Upload.pkg.files
73 for file in files.keys():
74 # The .orig.tar.gz can disappear out from under us is it's a
75 # duplicate of one in the archive.
76 if not files.has_key(file):
78 # Check that the source still exists
79 if files[file]["type"] == "deb":
80 source_version = files[file]["source version"]
81 source_package = files[file]["source package"]
82 if not Upload.pkg.changes["architecture"].has_key("source") \
83 and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
84 source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
85 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
86 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
87 reject("no source found for %s %s (%s)." % (source_package, source_version, file))
89 # Version and file overwrite checks
90 if files[file]["type"] == "deb":
91 reject(Upload.check_binary_against_db(file))
92 elif files[file]["type"] == "dsc":
93 reject(Upload.check_source_against_db(file))
94 (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
95 reject(reject_msg, "")
97 if reject_message.find("Rejected") != -1:
99 if Options["No-Action"] or Options["Automatic"]:
102 print "REJECT\n" + reject_message,
103 prompt = "[R]eject, Skip, Quit ?"
105 while prompt.find(answer) == -1:
106 answer = daklib.utils.our_raw_input(prompt)
107 m = daklib.queue.re_default_answer.match(prompt)
110 answer = answer[:1].upper()
113 Upload.do_reject(0, reject_message)
114 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
124 ################################################################################
126 def indiv_sg_compare (a, b):
127 """Sort by source name, source, version, 'have source', and
128 finally by filename."""
129 # Sort by source version
130 q = apt_pkg.VersionCompare(a["version"], b["version"])
134 # Sort by 'have source'
135 a_has_source = a["architecture"].get("source")
136 b_has_source = b["architecture"].get("source")
137 if a_has_source and not b_has_source:
139 elif b_has_source and not a_has_source:
142 return cmp(a["filename"], b["filename"])
144 ############################################################
146 def sg_compare (a, b):
149 """Sort by have note, time of oldest upload."""
151 a_note_state = a["note_state"]
152 b_note_state = b["note_state"]
153 if a_note_state < b_note_state:
155 elif a_note_state > b_note_state:
158 # Sort by time of oldest upload
159 return cmp(a["oldest"], b["oldest"])
161 def sort_changes(changes_files):
162 """Sort into source groups, then sort each source group by version,
163 have source, filename. Finally, sort the source groups by have
164 note, time of oldest upload of each source upload."""
165 if len(changes_files) == 1:
170 # Read in all the .changes files
171 for filename in changes_files:
173 Upload.pkg.changes_file = filename
176 cache[filename] = copy.copy(Upload.pkg.changes)
177 cache[filename]["filename"] = filename
179 sorted_list.append(filename)
181 # Divide the .changes into per-source groups
183 for filename in cache.keys():
184 source = cache[filename]["source"]
185 if not per_source.has_key(source):
186 per_source[source] = {}
187 per_source[source]["list"] = []
188 per_source[source]["list"].append(cache[filename])
189 # Determine oldest time and have note status for each source group
190 for source in per_source.keys():
191 source_list = per_source[source]["list"]
192 first = source_list[0]
193 oldest = os.stat(first["filename"])[stat.ST_MTIME]
195 for d in per_source[source]["list"]:
196 mtime = os.stat(d["filename"])[stat.ST_MTIME]
199 have_note += (d.has_key("process-new note"))
200 per_source[source]["oldest"] = oldest
202 per_source[source]["note_state"] = 0; # none
203 elif have_note < len(source_list):
204 per_source[source]["note_state"] = 1; # some
206 per_source[source]["note_state"] = 2; # all
207 per_source[source]["list"].sort(indiv_sg_compare)
208 per_source_items = per_source.items()
209 per_source_items.sort(sg_compare)
210 for i in per_source_items:
211 for j in i[1]["list"]:
212 sorted_list.append(j["filename"])
215 ################################################################################
217 class Section_Completer:
220 q = projectB.query("SELECT section FROM section")
221 for i in q.getresult():
222 self.sections.append(i[0])
224 def complete(self, text, state):
228 for word in self.sections:
230 self.matches.append(word)
232 return self.matches[state]
236 ############################################################
238 class Priority_Completer:
241 q = projectB.query("SELECT priority FROM priority")
242 for i in q.getresult():
243 self.priorities.append(i[0])
245 def complete(self, text, state):
249 for word in self.priorities:
251 self.matches.append(word)
253 return self.matches[state]
257 ################################################################################
259 def print_new (new, indexed, file=sys.stdout):
260 daklib.queue.check_valid(new)
263 for pkg in new.keys():
265 section = new[pkg]["section"]
266 priority = new[pkg]["priority"]
267 if new[pkg]["section id"] == -1:
270 if new[pkg]["priority id"] == -1:
274 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
276 line = "%-20s %-20s %-20s" % (pkg, priority, section)
277 line = line.strip()+'\n'
279 note = Upload.pkg.changes.get("process-new note")
286 ################################################################################
288 def index_range (index):
292 return "1-%s" % (index)
294 ################################################################################
295 ################################################################################
298 # Write the current data to a temporary file
299 temp_filename = daklib.utils.temp_filename()
300 temp_file = daklib.utils.open_file(temp_filename, 'w')
301 print_new (new, 0, temp_file)
303 # Spawn an editor on that file
304 editor = os.environ.get("EDITOR","vi")
305 result = os.system("%s %s" % (editor, temp_filename))
307 daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
308 # Read the edited data back in
309 temp_file = daklib.utils.open_file(temp_filename)
310 lines = temp_file.readlines()
312 os.unlink(temp_filename)
319 # Pad the list if necessary
320 s[len(s):3] = [None] * (3-len(s))
321 (pkg, priority, section) = s[:3]
322 if not new.has_key(pkg):
323 daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
325 # Strip off any invalid markers, print_new will readd them.
326 if section.endswith("[!]"):
327 section = section[:-3]
328 if priority.endswith("[!]"):
329 priority = priority[:-3]
330 for file in new[pkg]["files"]:
331 Upload.pkg.files[file]["section"] = section
332 Upload.pkg.files[file]["priority"] = priority
333 new[pkg]["section"] = section
334 new[pkg]["priority"] = priority
336 ################################################################################
338 def edit_index (new, index):
339 priority = new[index]["priority"]
340 section = new[index]["section"]
341 type = new[index]["type"]
344 print "\t".join([index, priority, section])
348 prompt = "[B]oth, Priority, Section, Done ? "
350 prompt = "[S]ection, Done ? "
351 edit_priority = edit_section = 0
353 while prompt.find(answer) == -1:
354 answer = daklib.utils.our_raw_input(prompt)
355 m = daklib.queue.re_default_answer.match(prompt)
358 answer = answer[:1].upper()
365 edit_priority = edit_section = 1
371 readline.set_completer(Priorities.complete)
373 while not got_priority:
374 new_priority = daklib.utils.our_raw_input("New priority: ").strip()
375 if new_priority not in Priorities.priorities:
376 print "E: '%s' is not a valid priority, try again." % (new_priority)
379 priority = new_priority
383 readline.set_completer(Sections.complete)
385 while not got_section:
386 new_section = daklib.utils.our_raw_input("New section: ").strip()
387 if new_section not in Sections.sections:
388 print "E: '%s' is not a valid section, try again." % (new_section)
391 section = new_section
393 # Reset the readline completer
394 readline.set_completer(None)
396 for file in new[index]["files"]:
397 Upload.pkg.files[file]["section"] = section
398 Upload.pkg.files[file]["priority"] = priority
399 new[index]["priority"] = priority
400 new[index]["section"] = section
403 ################################################################################
405 def edit_overrides (new):
416 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
419 while not got_answer:
420 answer = daklib.utils.our_raw_input(prompt)
421 if not answer.isdigit():
422 answer = answer[:1].upper()
423 if answer == "E" or answer == "D":
425 elif daklib.queue.re_isanum.match (answer):
427 if (answer < 1) or (answer > index):
428 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
437 edit_index (new, new_index[answer])
441 ################################################################################
444 # Write the current data to a temporary file
445 temp_filename = daklib.utils.temp_filename()
446 temp_file = daklib.utils.open_file(temp_filename, 'w')
447 temp_file.write(note)
449 editor = os.environ.get("EDITOR","vi")
452 os.system("%s %s" % (editor, temp_filename))
453 temp_file = daklib.utils.open_file(temp_filename)
454 note = temp_file.read().rstrip()
457 print daklib.utils.prefix_multi_line_string(note," ")
458 prompt = "[D]one, Edit, Abandon, Quit ?"
460 while prompt.find(answer) == -1:
461 answer = daklib.utils.our_raw_input(prompt)
462 m = daklib.queue.re_default_answer.search(prompt)
465 answer = answer[:1].upper()
466 os.unlink(temp_filename)
472 Upload.pkg.changes["process-new note"] = note
473 Upload.dump_vars(Cnf["Dir::Queue::New"])
475 ################################################################################
479 less_fd = os.popen("less -R -", 'w', 0)
480 stdout_fd = sys.stdout
483 examine_package.display_changes(Upload.pkg.changes_file)
484 files = Upload.pkg.files
485 for file in files.keys():
486 if files[file].has_key("new"):
487 type = files[file]["type"]
489 examine_package.check_deb(file)
491 examine_package.check_dsc(file)
493 sys.stdout = stdout_fd
495 if errno.errorcode[e.errno] == 'EPIPE':
496 daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
500 except KeyboardInterrupt:
501 daklib.utils.warn("[examine_package] Caught C-c; skipping.")
504 ################################################################################
506 ## FIXME: horribly Debian specific
508 def do_bxa_notification():
509 files = Upload.pkg.files
511 for file in files.keys():
512 if files[file]["type"] == "deb":
513 control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
515 summary += "Package: %s\n" % (control.Find("Package"))
516 summary += "Description: %s\n" % (control.Find("Description"))
517 Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
518 bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
519 daklib.utils.send_mail(bxa_mail)
521 ################################################################################
523 def add_overrides (new):
524 changes = Upload.pkg.changes
525 files = Upload.pkg.files
527 projectB.query("BEGIN WORK")
528 for suite in changes["suite"].keys():
529 suite_id = daklib.database.get_suite_id(suite)
530 for pkg in new.keys():
531 component_id = daklib.database.get_component_id(new[pkg]["component"])
532 type_id = daklib.database.get_override_type_id(new[pkg]["type"])
533 priority_id = new[pkg]["priority id"]
534 section_id = new[pkg]["section id"]
535 projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
536 for file in new[pkg]["files"]:
537 if files[file].has_key("new"):
538 del files[file]["new"]
541 projectB.query("COMMIT WORK")
543 if Cnf.FindB("Dinstall::BXANotify"):
544 do_bxa_notification()
546 ################################################################################
548 def prod_maintainer ():
549 # Here we prepare an editor and get them ready to prod...
550 temp_filename = daklib.utils.temp_filename()
551 editor = os.environ.get("EDITOR","vi")
554 os.system("%s %s" % (editor, temp_filename))
555 file = daklib.utils.open_file(temp_filename)
556 prod_message = "".join(file.readlines())
558 print "Prod message:"
559 print daklib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
560 prompt = "[P]rod, Edit, Abandon, Quit ?"
562 while prompt.find(answer) == -1:
563 answer = daklib.utils.our_raw_input(prompt)
564 m = daklib.queue.re_default_answer.search(prompt)
567 answer = answer[:1].upper()
568 os.unlink(temp_filename)
574 # Otherwise, do the proding...
575 user_email_address = daklib.utils.whoami() + " <%s>" % (
576 Cnf["Dinstall::MyAdminAddress"])
580 Subst["__FROM_ADDRESS__"] = user_email_address
581 Subst["__PROD_MESSAGE__"] = prod_message
582 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
584 prod_mail_message = daklib.utils.TemplateSubst(
585 Subst,Cnf["Dir::Templates"]+"/process-new.prod")
587 # Send the prod mail if appropriate
588 if not Cnf["Dinstall::Options::No-Mail"]:
589 daklib.utils.send_mail(prod_mail_message)
591 print "Sent proding message"
593 ################################################################################
597 files = Upload.pkg.files
598 changes = Upload.pkg.changes
600 # Make a copy of distribution we can happily trample on
601 changes["suite"] = copy.copy(changes["distribution"])
603 # Fix up the list of target suites
604 for suite in changes["suite"].keys():
605 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
607 (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1,
608 daklib.database.get_suite_id(override) == -1)
610 (oinv, newinv) = ("", "")
611 if olderr: oinv = "invalid "
612 if newerr: ninv = "invalid "
613 print "warning: overriding %ssuite %s to %ssuite %s" % (
614 oinv, suite, ninv, override)
615 del changes["suite"][suite]
616 changes["suite"][override] = 1
618 for suite in changes["suite"].keys():
619 suite_id = daklib.database.get_suite_id(suite)
621 daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
623 # The main NEW processing loop
626 # Find out what's new
627 new = daklib.queue.determine_new(changes, files, projectB)
633 if Options["No-Action"] or Options["Automatic"]:
636 (broken, note) = print_new(new, 0)
639 if not broken and not note:
640 prompt = "Add overrides, "
642 print "W: [!] marked entries must be fixed before package can be processed."
644 print "W: note must be removed before package can be processed."
645 prompt += "Remove note, "
647 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
649 while prompt.find(answer) == -1:
650 answer = daklib.utils.our_raw_input(prompt)
651 m = daklib.queue.re_default_answer.search(prompt)
654 answer = answer[:1].upper()
657 done = add_overrides (new)
661 new = edit_overrides (new)
663 aborted = Upload.do_reject(1, Options["Manual-Reject"])
665 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
668 edit_note(changes.get("process-new note", ""))
672 confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
674 del changes["process-new note"]
681 ################################################################################
682 ################################################################################
683 ################################################################################
685 def usage (exit_code=0):
686 print """Usage: dak process-new [OPTION]... [CHANGES]...
687 -a, --automatic automatic run
688 -h, --help show this help and exit.
689 -m, --manual-reject=MSG manual reject with `msg'
690 -n, --no-action don't do anything
691 -V, --version display the version number and exit"""
694 ################################################################################
697 global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
699 Cnf = daklib.utils.get_conf()
701 Arguments = [('a',"automatic","Process-New::Options::Automatic"),
702 ('h',"help","Process-New::Options::Help"),
703 ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
704 ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
705 ('n',"no-action","Process-New::Options::No-Action")]
707 for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
708 if not Cnf.has_key("Process-New::Options::%s" % (i)):
709 Cnf["Process-New::Options::%s" % (i)] = ""
711 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
712 Options = Cnf.SubTree("Process-New::Options")
717 Upload = daklib.queue.Upload(Cnf)
719 if not Options["No-Action"]:
720 Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
722 projectB = Upload.projectB
724 Sections = Section_Completer()
725 Priorities = Priority_Completer()
726 readline.parse_and_bind("tab: complete")
730 ################################################################################
735 files = Upload.pkg.files
739 for file in files.keys():
740 if files[file]["type"] == "byhand":
741 if os.path.exists(file):
742 print "W: %s still present; please process byhand components and try again." % (file)
748 if Options["No-Action"]:
751 if Options["Automatic"] and not Options["No-Action"]:
753 prompt = "[A]ccept, Manual reject, Skip, Quit ?"
755 prompt = "Manual reject, [S]kip, Quit ?"
757 while prompt.find(answer) == -1:
758 answer = daklib.utils.our_raw_input(prompt)
759 m = daklib.queue.re_default_answer.search(prompt)
762 answer = answer[:1].upper()
769 Upload.do_reject(1, Options["Manual-Reject"])
770 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
778 ################################################################################
782 if not Options["No-Action"]:
786 lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
789 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
792 daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
794 print("Unable to get accepted lock (try %d of 10)" % retry)
798 (summary, short_summary) = Upload.build_summaries()
799 Upload.accept(summary, short_summary)
800 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
801 os.unlink(Cnf["Process-New::AcceptedLockFile"])
803 def check_status(files):
805 for file in files.keys():
806 if files[file]["type"] == "byhand":
808 elif files[file].has_key("new"):
812 def do_pkg(changes_file):
813 Upload.pkg.changes_file = changes_file
816 Upload.update_subst()
817 files = Upload.pkg.files
822 (new, byhand) = check_status(files)
828 (new, byhand) = check_status(files)
830 if not new and not byhand:
833 ################################################################################
836 accept_count = Upload.accept_count
837 accept_bytes = Upload.accept_bytes
843 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
844 Logger.log(["total",accept_count,accept_bytes])
846 if not Options["No-Action"]:
849 ################################################################################
851 def do_comments(dir, opref, npref, line, fn):
852 for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
853 lines = open("%s/%s" % (dir, comm)).readlines()
854 if len(lines) == 0 or lines[0] != line + "\n": continue
855 changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
856 and x.endswith(".changes") ]
857 changes_files = sort_changes(changes_files)
858 for f in changes_files:
859 f = daklib.utils.validate_changes_file_arg(f, 0)
862 fn(f, "".join(lines[1:]))
864 if opref != npref and not Options["No-Action"]:
865 newcomm = npref + comm[len(opref):]
866 os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
868 ################################################################################
870 def comment_accept(changes_file, comments):
871 Upload.pkg.changes_file = changes_file
874 Upload.update_subst()
875 files = Upload.pkg.files
878 return # dak wants to REJECT, crap
880 (new, byhand) = check_status(files)
881 if not new and not byhand:
884 ################################################################################
886 def comment_reject(changes_file, comments):
887 Upload.pkg.changes_file = changes_file
890 Upload.update_subst()
891 files = Upload.pkg.files
894 pass # dak has its own reasons to reject as well, which is fine
897 print "REJECT\n" + reject_message,
898 if not Options["No-Action"]:
899 Upload.do_reject(0, reject_message)
900 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
902 ################################################################################
905 changes_files = init()
906 if len(changes_files) > 50:
907 sys.stderr.write("Sorting changes...\n")
908 changes_files = sort_changes(changes_files)
910 # Kill me now? **FIXME**
911 Cnf["Dinstall::Options::No-Mail"] = ""
912 bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
913 if Cnf.has_key("Dinstall::Bcc"):
914 Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
916 Upload.Subst["__BCC__"] = bcc
918 commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
920 if changes_files != []:
921 sys.stderr.write("Can't specify any changes files if working with comments-dir")
923 do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
924 do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
926 for changes_file in changes_files:
927 changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
930 print "\n" + changes_file
931 do_pkg (changes_file)
935 ################################################################################
937 if __name__ == '__main__':