3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # 23:12|<aj> I will not hush!
24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
25 # 23:13|<aj> I shall not be silenced!
26 # 23:13|<aj> The world shall know!
27 # 23:13|<aj> The world *must* know!
28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
29 # 23:13|<aj> yay powerpuff girls!!
30 # 23:13|<aj> buttercup's my favourite, who's yours?
31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
32 # 23:14|<aj> *AREN'T YOU*?!
33 # 23:15|<aj> I will not be treated like this.
34 # 23:15|<aj> I shall have my revenge.
35 # 23:15|<aj> I SHALL!!!
37 ################################################################################
39 import copy, errno, os, readline, stat, sys, time
40 import apt_pkg, apt_inst
41 import examine_package
42 import daklib.database
59 ################################################################################
60 ################################################################################
61 ################################################################################
63 def reject (str, prefix="Rejected: "):
66 reject_message += prefix + str + "\n"
70 files = Upload.pkg.files
73 for file in files.keys():
74 # The .orig.tar.gz can disappear out from under us is it's a
75 # duplicate of one in the archive.
76 if not files.has_key(file):
78 # Check that the source still exists
79 if files[file]["type"] == "deb":
80 source_version = files[file]["source version"]
81 source_package = files[file]["source package"]
82 if not Upload.pkg.changes["architecture"].has_key("source") \
83 and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
84 source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
85 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
86 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
87 reject("no source found for %s %s (%s)." % (source_package, source_version, file))
89 # Version and file overwrite checks
90 if files[file]["type"] == "deb":
91 reject(Upload.check_binary_against_db(file))
92 elif files[file]["type"] == "dsc":
93 reject(Upload.check_source_against_db(file))
94 (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
99 if Options["No-Action"] or Options["Automatic"]:
102 print "REJECT\n" + reject_message,
103 prompt = "[R]eject, Skip, Quit ?"
105 while prompt.find(answer) == -1:
106 answer = daklib.utils.our_raw_input(prompt)
107 m = daklib.queue.re_default_answer.match(prompt)
110 answer = answer[:1].upper()
113 Upload.do_reject(0, reject_message)
114 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
123 ################################################################################
125 def determine_new (changes, files):
128 # Build up a list of potentially new things
129 for file in files.keys():
131 # Skip byhand elements
132 if f["type"] == "byhand":
135 priority = f["priority"]
136 section = f["section"]
138 if section == "non-US/main":
141 component = f["component"]
145 if not new.has_key(pkg):
147 new[pkg]["priority"] = priority
148 new[pkg]["section"] = section
149 new[pkg]["type"] = type
150 new[pkg]["component"] = component
151 new[pkg]["files"] = []
153 old_type = new[pkg]["type"]
155 # source gets trumped by deb or udeb
156 if old_type == "dsc":
157 new[pkg]["priority"] = priority
158 new[pkg]["section"] = section
159 new[pkg]["type"] = type
160 new[pkg]["component"] = component
161 new[pkg]["files"].append(file)
162 if f.has_key("othercomponents"):
163 new[pkg]["othercomponents"] = f["othercomponents"]
165 for suite in changes["suite"].keys():
166 suite_id = database.get_suite_id(suite)
167 for pkg in new.keys():
168 component_id = database.get_component_id(new[pkg]["component"])
169 type_id = database.get_override_type_id(new[pkg]["type"])
170 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
173 for file in new[pkg]["files"]:
174 if files[file].has_key("new"):
175 del files[file]["new"]
178 if changes["suite"].has_key("stable"):
179 print "WARNING: overrides will be added for stable!"
180 if changes["suite"].has_key("oldstable"):
181 print "WARNING: overrides will be added for OLDstable!"
182 for pkg in new.keys():
183 if new[pkg].has_key("othercomponents"):
184 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
188 ################################################################################
190 def indiv_sg_compare (a, b):
191 """Sort by source name, source, version, 'have source', and
192 finally by filename."""
193 # Sort by source version
194 q = apt_pkg.VersionCompare(a["version"], b["version"])
198 # Sort by 'have source'
199 a_has_source = a["architecture"].get("source")
200 b_has_source = b["architecture"].get("source")
201 if a_has_source and not b_has_source:
203 elif b_has_source and not a_has_source:
206 return cmp(a["filename"], b["filename"])
208 ############################################################
210 def sg_compare (a, b):
213 """Sort by have note, time of oldest upload."""
215 a_note_state = a["note_state"]
216 b_note_state = b["note_state"]
217 if a_note_state < b_note_state:
219 elif a_note_state > b_note_state:
222 # Sort by time of oldest upload
223 return cmp(a["oldest"], b["oldest"])
225 def sort_changes(changes_files):
226 """Sort into source groups, then sort each source group by version,
227 have source, filename. Finally, sort the source groups by have
228 note, time of oldest upload of each source upload."""
229 if len(changes_files) == 1:
234 # Read in all the .changes files
235 for filename in changes_files:
237 Upload.pkg.changes_file = filename
240 cache[filename] = copy.copy(Upload.pkg.changes)
241 cache[filename]["filename"] = filename
243 sorted_list.append(filename)
245 # Divide the .changes into per-source groups
247 for filename in cache.keys():
248 source = cache[filename]["source"]
249 if not per_source.has_key(source):
250 per_source[source] = {}
251 per_source[source]["list"] = []
252 per_source[source]["list"].append(cache[filename])
253 # Determine oldest time and have note status for each source group
254 for source in per_source.keys():
255 source_list = per_source[source]["list"]
256 first = source_list[0]
257 oldest = os.stat(first["filename"])[stat.ST_MTIME]
259 for d in per_source[source]["list"]:
260 mtime = os.stat(d["filename"])[stat.ST_MTIME]
263 have_note += (d.has_key("process-new note"))
264 per_source[source]["oldest"] = oldest
266 per_source[source]["note_state"] = 0; # none
267 elif have_note < len(source_list):
268 per_source[source]["note_state"] = 1; # some
270 per_source[source]["note_state"] = 2; # all
271 per_source[source]["list"].sort(indiv_sg_compare)
272 per_source_items = per_source.items()
273 per_source_items.sort(sg_compare)
274 for i in per_source_items:
275 for j in i[1]["list"]:
276 sorted_list.append(j["filename"])
279 ################################################################################
281 class Section_Completer:
284 q = projectB.query("SELECT section FROM section")
285 for i in q.getresult():
286 self.sections.append(i[0])
288 def complete(self, text, state):
292 for word in self.sections:
294 self.matches.append(word)
296 return self.matches[state]
300 ############################################################
302 class Priority_Completer:
305 q = projectB.query("SELECT priority FROM priority")
306 for i in q.getresult():
307 self.priorities.append(i[0])
309 def complete(self, text, state):
313 for word in self.priorities:
315 self.matches.append(word)
317 return self.matches[state]
321 ################################################################################
323 def check_valid (new):
324 for pkg in new.keys():
325 section = new[pkg]["section"]
326 priority = new[pkg]["priority"]
327 type = new[pkg]["type"]
328 new[pkg]["section id"] = database.get_section_id(section)
329 new[pkg]["priority id"] = database.get_priority_id(new[pkg]["priority"])
331 if (section == "debian-installer" and type != "udeb") or \
332 (section != "debian-installer" and type == "udeb"):
333 new[pkg]["section id"] = -1
334 if (priority == "source" and type != "dsc") or \
335 (priority != "source" and type == "dsc"):
336 new[pkg]["priority id"] = -1
338 ################################################################################
340 def print_new (new, indexed, file=sys.stdout):
344 for pkg in new.keys():
346 section = new[pkg]["section"]
347 priority = new[pkg]["priority"]
348 if new[pkg]["section id"] == -1:
351 if new[pkg]["priority id"] == -1:
355 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
357 line = "%-20s %-20s %-20s" % (pkg, priority, section)
358 line = line.strip()+'\n'
360 note = Upload.pkg.changes.get("process-new note")
367 ################################################################################
371 if f.has_key("dbtype"):
373 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
376 daklib.utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
378 # Validate the override type
379 type_id = database.get_override_type_id(type)
381 daklib.utils.fubar("invalid type (%s) for new. Say wha?" % (type))
385 ################################################################################
387 def index_range (index):
391 return "1-%s" % (index)
393 ################################################################################
394 ################################################################################
397 # Write the current data to a temporary file
398 temp_filename = daklib.utils.temp_filename()
399 temp_file = daklib.utils.open_file(temp_filename, 'w')
400 print_new (new, 0, temp_file)
402 # Spawn an editor on that file
403 editor = os.environ.get("EDITOR","vi")
404 result = os.system("%s %s" % (editor, temp_filename))
406 daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
407 # Read the edited data back in
408 temp_file = daklib.utils.open_file(temp_filename)
409 lines = temp_file.readlines()
411 os.unlink(temp_filename)
418 # Pad the list if necessary
419 s[len(s):3] = [None] * (3-len(s))
420 (pkg, priority, section) = s[:3]
421 if not new.has_key(pkg):
422 daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
424 # Strip off any invalid markers, print_new will readd them.
425 if section.endswith("[!]"):
426 section = section[:-3]
427 if priority.endswith("[!]"):
428 priority = priority[:-3]
429 for file in new[pkg]["files"]:
430 Upload.pkg.files[file]["section"] = section
431 Upload.pkg.files[file]["priority"] = priority
432 new[pkg]["section"] = section
433 new[pkg]["priority"] = priority
435 ################################################################################
437 def edit_index (new, index):
438 priority = new[index]["priority"]
439 section = new[index]["section"]
440 type = new[index]["type"]
443 print "\t".join([index, priority, section])
447 prompt = "[B]oth, Priority, Section, Done ? "
449 prompt = "[S]ection, Done ? "
450 edit_priority = edit_section = 0
452 while prompt.find(answer) == -1:
453 answer = daklib.utils.our_raw_input(prompt)
454 m = daklib.queue.re_default_answer.match(prompt)
457 answer = answer[:1].upper()
464 edit_priority = edit_section = 1
470 readline.set_completer(Priorities.complete)
472 while not got_priority:
473 new_priority = daklib.utils.our_raw_input("New priority: ").strip()
474 if new_priority not in Priorities.priorities:
475 print "E: '%s' is not a valid priority, try again." % (new_priority)
478 priority = new_priority
482 readline.set_completer(Sections.complete)
484 while not got_section:
485 new_section = daklib.utils.our_raw_input("New section: ").strip()
486 if new_section not in Sections.sections:
487 print "E: '%s' is not a valid section, try again." % (new_section)
490 section = new_section
492 # Reset the readline completer
493 readline.set_completer(None)
495 for file in new[index]["files"]:
496 Upload.pkg.files[file]["section"] = section
497 Upload.pkg.files[file]["priority"] = priority
498 new[index]["priority"] = priority
499 new[index]["section"] = section
502 ################################################################################
504 def edit_overrides (new):
515 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
518 while not got_answer:
519 answer = daklib.utils.our_raw_input(prompt)
520 if not daklib.utils.str_isnum(answer):
521 answer = answer[:1].upper()
522 if answer == "E" or answer == "D":
524 elif daklib.queue.re_isanum.match (answer):
526 if (answer < 1) or (answer > index):
527 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
536 edit_index (new, new_index[answer])
540 ################################################################################
543 # Write the current data to a temporary file
544 temp_filename = daklib.utils.temp_filename()
545 temp_file = daklib.utils.open_file(temp_filename, 'w')
546 temp_file.write(note)
548 editor = os.environ.get("EDITOR","vi")
551 os.system("%s %s" % (editor, temp_filename))
552 temp_file = daklib.utils.open_file(temp_filename)
553 note = temp_file.read().rstrip()
556 print daklib.utils.prefix_multi_line_string(note," ")
557 prompt = "[D]one, Edit, Abandon, Quit ?"
559 while prompt.find(answer) == -1:
560 answer = daklib.utils.our_raw_input(prompt)
561 m = daklib.queue.re_default_answer.search(prompt)
564 answer = answer[:1].upper()
565 os.unlink(temp_filename)
570 Upload.pkg.changes["process-new note"] = note
571 Upload.dump_vars(Cnf["Dir::Queue::New"])
573 ################################################################################
577 less_fd = os.popen("less -R -", 'w', 0)
578 stdout_fd = sys.stdout
581 examine_package.display_changes(Upload.pkg.changes_file)
582 files = Upload.pkg.files
583 for file in files.keys():
584 if files[file].has_key("new"):
585 type = files[file]["type"]
587 examine_package.check_deb(file)
589 examine_package.check_dsc(file)
591 sys.stdout = stdout_fd
593 if errno.errorcode[e.errno] == 'EPIPE':
594 daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
598 except KeyboardInterrupt:
599 daklib.utils.warn("[examine_package] Caught C-c; skipping.")
602 ################################################################################
604 ## FIXME: horribly Debian specific
606 def do_bxa_notification():
607 files = Upload.pkg.files
609 for file in files.keys():
610 if files[file]["type"] == "deb":
611 control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
613 summary += "Package: %s\n" % (control.Find("Package"))
614 summary += "Description: %s\n" % (control.Find("Description"))
615 Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
616 bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
617 daklib.utils.send_mail(bxa_mail)
619 ################################################################################
621 def add_overrides (new):
622 changes = Upload.pkg.changes
623 files = Upload.pkg.files
625 projectB.query("BEGIN WORK")
626 for suite in changes["suite"].keys():
627 suite_id = database.get_suite_id(suite)
628 for pkg in new.keys():
629 component_id = database.get_component_id(new[pkg]["component"])
630 type_id = database.get_override_type_id(new[pkg]["type"])
631 priority_id = new[pkg]["priority id"]
632 section_id = new[pkg]["section id"]
633 projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
634 for file in new[pkg]["files"]:
635 if files[file].has_key("new"):
636 del files[file]["new"]
639 projectB.query("COMMIT WORK")
641 if Cnf.FindB("Dinstall::BXANotify"):
642 do_bxa_notification()
644 ################################################################################
646 def prod_maintainer ():
647 # Here we prepare an editor and get them ready to prod...
648 temp_filename = daklib.utils.temp_filename()
649 editor = os.environ.get("EDITOR","vi")
652 os.system("%s %s" % (editor, temp_filename))
653 file = daklib.utils.open_file(temp_filename)
654 prod_message = "".join(file.readlines())
656 print "Prod message:"
657 print daklib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
658 prompt = "[P]rod, Edit, Abandon, Quit ?"
660 while prompt.find(answer) == -1:
661 answer = daklib.utils.our_raw_input(prompt)
662 m = daklib.queue.re_default_answer.search(prompt)
665 answer = answer[:1].upper()
666 os.unlink(temp_filename)
671 # Otherwise, do the proding...
672 user_email_address = daklib.utils.whoami() + " <%s>" % (
673 Cnf["Dinstall::MyAdminAddress"])
677 Subst["__FROM_ADDRESS__"] = user_email_address
678 Subst["__PROD_MESSAGE__"] = prod_message
679 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
681 prod_mail_message = daklib.utils.TemplateSubst(
682 Subst,Cnf["Dir::Templates"]+"/process-new.prod")
684 # Send the prod mail if appropriate
685 if not Cnf["Dinstall::Options::No-Mail"]:
686 daklib.utils.send_mail(prod_mail_message)
688 print "Sent proding message"
690 ################################################################################
694 files = Upload.pkg.files
695 changes = Upload.pkg.changes
697 # Make a copy of distribution we can happily trample on
698 changes["suite"] = copy.copy(changes["distribution"])
700 # Fix up the list of target suites
701 for suite in changes["suite"].keys():
702 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
704 del changes["suite"][suite]
705 changes["suite"][override] = 1
707 for suite in changes["suite"].keys():
708 suite_id = database.get_suite_id(suite)
710 daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
712 # The main NEW processing loop
715 # Find out what's new
716 new = determine_new(changes, files)
722 if Options["No-Action"] or Options["Automatic"]:
725 (broken, note) = print_new(new, 0)
728 if not broken and not note:
729 prompt = "Add overrides, "
731 print "W: [!] marked entries must be fixed before package can be processed."
733 print "W: note must be removed before package can be processed."
734 prompt += "Remove note, "
736 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
738 while prompt.find(answer) == -1:
739 answer = daklib.utils.our_raw_input(prompt)
740 m = daklib.queue.re_default_answer.search(prompt)
743 answer = answer[:1].upper()
746 done = add_overrides (new)
750 new = edit_overrides (new)
752 aborted = Upload.do_reject(1, Options["Manual-Reject"])
754 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
757 edit_note(changes.get("process-new note", ""))
761 confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
763 del changes["process-new note"]
769 ################################################################################
770 ################################################################################
771 ################################################################################
773 def usage (exit_code=0):
774 print """Usage: dak process-new [OPTION]... [CHANGES]...
775 -a, --automatic automatic run
776 -h, --help show this help and exit.
777 -m, --manual-reject=MSG manual reject with `msg'
778 -n, --no-action don't do anything
779 -V, --version display the version number and exit"""
782 ################################################################################
785 global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
787 Cnf = daklib.utils.get_conf()
789 Arguments = [('a',"automatic","Process-New::Options::Automatic"),
790 ('h',"help","Process-New::Options::Help"),
791 ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
792 ('n',"no-action","Process-New::Options::No-Action")]
794 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
795 if not Cnf.has_key("Process-New::Options::%s" % (i)):
796 Cnf["Process-New::Options::%s" % (i)] = ""
798 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
799 Options = Cnf.SubTree("Process-New::Options")
804 Upload = daklib.queue.Upload(Cnf)
806 if not Options["No-Action"]:
807 Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
809 projectB = Upload.projectB
811 Sections = Section_Completer()
812 Priorities = Priority_Completer()
813 readline.parse_and_bind("tab: complete")
817 ################################################################################
822 files = Upload.pkg.files
826 for file in files.keys():
827 if files[file]["type"] == "byhand":
828 if os.path.exists(file):
829 print "W: %s still present; please process byhand components and try again." % (file)
835 if Options["No-Action"]:
838 if Options["Automatic"] and not Options["No-Action"]:
840 prompt = "[A]ccept, Manual reject, Skip, Quit ?"
842 prompt = "Manual reject, [S]kip, Quit ?"
844 while prompt.find(answer) == -1:
845 answer = daklib.utils.our_raw_input(prompt)
846 m = daklib.queue.re_default_answer.search(prompt)
849 answer = answer[:1].upper()
856 Upload.do_reject(1, Options["Manual-Reject"])
857 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
864 ################################################################################
868 if not Options["No-Action"]:
872 lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
875 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
878 daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
880 print("Unable to get accepted lock (try %d of 10)" % retry)
884 (summary, short_summary) = Upload.build_summaries()
885 Upload.accept(summary, short_summary)
886 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
887 os.unlink(Cnf["Process-New::AcceptedLockFile"])
889 def check_status(files):
891 for file in files.keys():
892 if files[file]["type"] == "byhand":
894 elif files[file].has_key("new"):
898 def do_pkg(changes_file):
899 Upload.pkg.changes_file = changes_file
902 Upload.update_subst()
903 files = Upload.pkg.files
908 (new, byhand) = check_status(files)
914 (new, byhand) = check_status(files)
916 if not new and not byhand:
919 ################################################################################
922 accept_count = Upload.accept_count
923 accept_bytes = Upload.accept_bytes
929 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
930 Logger.log(["total",accept_count,accept_bytes])
932 if not Options["No-Action"]:
935 ################################################################################
938 changes_files = init()
939 if len(changes_files) > 50:
940 sys.stderr.write("Sorting changes...\n")
941 changes_files = sort_changes(changes_files)
943 # Kill me now? **FIXME**
944 Cnf["Dinstall::Options::No-Mail"] = ""
945 bcc = "X-DAK: dak process-new\nX-Katie: this header is obsolete"
946 if Cnf.has_key("Dinstall::Bcc"):
947 Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
949 Upload.Subst["__BCC__"] = bcc
951 for changes_file in changes_files:
952 changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
955 print "\n" + changes_file
956 do_pkg (changes_file)
960 ################################################################################
962 if __name__ == '__main__':