3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # 23:12|<aj> I will not hush!
24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
25 # 23:13|<aj> I shall not be silenced!
26 # 23:13|<aj> The world shall know!
27 # 23:13|<aj> The world *must* know!
28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
29 # 23:13|<aj> yay powerpuff girls!!
30 # 23:13|<aj> buttercup's my favourite, who's yours?
31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
32 # 23:14|<aj> *AREN'T YOU*?!
33 # 23:15|<aj> I will not be treated like this.
34 # 23:15|<aj> I shall have my revenge.
35 # 23:15|<aj> I SHALL!!!
37 ################################################################################
39 import copy, errno, os, readline, stat, sys, time
40 import apt_pkg, apt_inst
41 import examine_package
42 import daklib.database
59 ################################################################################
60 ################################################################################
61 ################################################################################
63 def reject (str, prefix="Rejected: "):
66 reject_message += prefix + str + "\n"
70 files = Upload.pkg.files
73 for file in files.keys():
74 # The .orig.tar.gz can disappear out from under us is it's a
75 # duplicate of one in the archive.
76 if not files.has_key(file):
78 # Check that the source still exists
79 if files[file]["type"] == "deb":
80 source_version = files[file]["source version"]
81 source_package = files[file]["source package"]
82 if not Upload.pkg.changes["architecture"].has_key("source") \
83 and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
84 source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
85 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
86 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
87 reject("no source found for %s %s (%s)." % (source_package, source_version, file))
89 # Version and file overwrite checks
90 if files[file]["type"] == "deb":
91 reject(Upload.check_binary_against_db(file))
92 elif files[file]["type"] == "dsc":
93 reject(Upload.check_source_against_db(file))
94 (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
97 if reject_message.find("Rejected") != -1:
99 if Options["No-Action"] or Options["Automatic"]:
102 print "REJECT\n" + reject_message,
103 prompt = "[R]eject, Skip, Quit ?"
105 while prompt.find(answer) == -1:
106 answer = daklib.utils.our_raw_input(prompt)
107 m = daklib.queue.re_default_answer.match(prompt)
110 answer = answer[:1].upper()
113 Upload.do_reject(0, reject_message)
114 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
123 ################################################################################
125 def indiv_sg_compare (a, b):
126 """Sort by source name, source, version, 'have source', and
127 finally by filename."""
128 # Sort by source version
129 q = apt_pkg.VersionCompare(a["version"], b["version"])
133 # Sort by 'have source'
134 a_has_source = a["architecture"].get("source")
135 b_has_source = b["architecture"].get("source")
136 if a_has_source and not b_has_source:
138 elif b_has_source and not a_has_source:
141 return cmp(a["filename"], b["filename"])
143 ############################################################
145 def sg_compare (a, b):
148 """Sort by have note, time of oldest upload."""
150 a_note_state = a["note_state"]
151 b_note_state = b["note_state"]
152 if a_note_state < b_note_state:
154 elif a_note_state > b_note_state:
157 # Sort by time of oldest upload
158 return cmp(a["oldest"], b["oldest"])
160 def sort_changes(changes_files):
161 """Sort into source groups, then sort each source group by version,
162 have source, filename. Finally, sort the source groups by have
163 note, time of oldest upload of each source upload."""
164 if len(changes_files) == 1:
169 # Read in all the .changes files
170 for filename in changes_files:
172 Upload.pkg.changes_file = filename
175 cache[filename] = copy.copy(Upload.pkg.changes)
176 cache[filename]["filename"] = filename
178 sorted_list.append(filename)
180 # Divide the .changes into per-source groups
182 for filename in cache.keys():
183 source = cache[filename]["source"]
184 if not per_source.has_key(source):
185 per_source[source] = {}
186 per_source[source]["list"] = []
187 per_source[source]["list"].append(cache[filename])
188 # Determine oldest time and have note status for each source group
189 for source in per_source.keys():
190 source_list = per_source[source]["list"]
191 first = source_list[0]
192 oldest = os.stat(first["filename"])[stat.ST_MTIME]
194 for d in per_source[source]["list"]:
195 mtime = os.stat(d["filename"])[stat.ST_MTIME]
198 have_note += (d.has_key("process-new note"))
199 per_source[source]["oldest"] = oldest
201 per_source[source]["note_state"] = 0; # none
202 elif have_note < len(source_list):
203 per_source[source]["note_state"] = 1; # some
205 per_source[source]["note_state"] = 2; # all
206 per_source[source]["list"].sort(indiv_sg_compare)
207 per_source_items = per_source.items()
208 per_source_items.sort(sg_compare)
209 for i in per_source_items:
210 for j in i[1]["list"]:
211 sorted_list.append(j["filename"])
214 ################################################################################
216 class Section_Completer:
219 q = projectB.query("SELECT section FROM section")
220 for i in q.getresult():
221 self.sections.append(i[0])
223 def complete(self, text, state):
227 for word in self.sections:
229 self.matches.append(word)
231 return self.matches[state]
235 ############################################################
237 class Priority_Completer:
240 q = projectB.query("SELECT priority FROM priority")
241 for i in q.getresult():
242 self.priorities.append(i[0])
244 def complete(self, text, state):
248 for word in self.priorities:
250 self.matches.append(word)
252 return self.matches[state]
256 ################################################################################
258 def print_new (new, indexed, file=sys.stdout):
259 daklib.queue.check_valid(new)
262 for pkg in new.keys():
264 section = new[pkg]["section"]
265 priority = new[pkg]["priority"]
266 if new[pkg]["section id"] == -1:
269 if new[pkg]["priority id"] == -1:
273 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
275 line = "%-20s %-20s %-20s" % (pkg, priority, section)
276 line = line.strip()+'\n'
278 note = Upload.pkg.changes.get("process-new note")
285 ################################################################################
287 def index_range (index):
291 return "1-%s" % (index)
293 ################################################################################
294 ################################################################################
297 # Write the current data to a temporary file
298 temp_filename = daklib.utils.temp_filename()
299 temp_file = daklib.utils.open_file(temp_filename, 'w')
300 print_new (new, 0, temp_file)
302 # Spawn an editor on that file
303 editor = os.environ.get("EDITOR","vi")
304 result = os.system("%s %s" % (editor, temp_filename))
306 daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
307 # Read the edited data back in
308 temp_file = daklib.utils.open_file(temp_filename)
309 lines = temp_file.readlines()
311 os.unlink(temp_filename)
318 # Pad the list if necessary
319 s[len(s):3] = [None] * (3-len(s))
320 (pkg, priority, section) = s[:3]
321 if not new.has_key(pkg):
322 daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
324 # Strip off any invalid markers, print_new will readd them.
325 if section.endswith("[!]"):
326 section = section[:-3]
327 if priority.endswith("[!]"):
328 priority = priority[:-3]
329 for file in new[pkg]["files"]:
330 Upload.pkg.files[file]["section"] = section
331 Upload.pkg.files[file]["priority"] = priority
332 new[pkg]["section"] = section
333 new[pkg]["priority"] = priority
335 ################################################################################
337 def edit_index (new, index):
338 priority = new[index]["priority"]
339 section = new[index]["section"]
340 type = new[index]["type"]
343 print "\t".join([index, priority, section])
347 prompt = "[B]oth, Priority, Section, Done ? "
349 prompt = "[S]ection, Done ? "
350 edit_priority = edit_section = 0
352 while prompt.find(answer) == -1:
353 answer = daklib.utils.our_raw_input(prompt)
354 m = daklib.queue.re_default_answer.match(prompt)
357 answer = answer[:1].upper()
364 edit_priority = edit_section = 1
370 readline.set_completer(Priorities.complete)
372 while not got_priority:
373 new_priority = daklib.utils.our_raw_input("New priority: ").strip()
374 if new_priority not in Priorities.priorities:
375 print "E: '%s' is not a valid priority, try again." % (new_priority)
378 priority = new_priority
382 readline.set_completer(Sections.complete)
384 while not got_section:
385 new_section = daklib.utils.our_raw_input("New section: ").strip()
386 if new_section not in Sections.sections:
387 print "E: '%s' is not a valid section, try again." % (new_section)
390 section = new_section
392 # Reset the readline completer
393 readline.set_completer(None)
395 for file in new[index]["files"]:
396 Upload.pkg.files[file]["section"] = section
397 Upload.pkg.files[file]["priority"] = priority
398 new[index]["priority"] = priority
399 new[index]["section"] = section
402 ################################################################################
404 def edit_overrides (new):
415 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
418 while not got_answer:
419 answer = daklib.utils.our_raw_input(prompt)
420 if not answer.isdigit():
421 answer = answer[:1].upper()
422 if answer == "E" or answer == "D":
424 elif daklib.queue.re_isanum.match (answer):
426 if (answer < 1) or (answer > index):
427 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
436 edit_index (new, new_index[answer])
440 ################################################################################
443 # Write the current data to a temporary file
444 temp_filename = daklib.utils.temp_filename()
445 temp_file = daklib.utils.open_file(temp_filename, 'w')
446 temp_file.write(note)
448 editor = os.environ.get("EDITOR","vi")
451 os.system("%s %s" % (editor, temp_filename))
452 temp_file = daklib.utils.open_file(temp_filename)
453 note = temp_file.read().rstrip()
456 print daklib.utils.prefix_multi_line_string(note," ")
457 prompt = "[D]one, Edit, Abandon, Quit ?"
459 while prompt.find(answer) == -1:
460 answer = daklib.utils.our_raw_input(prompt)
461 m = daklib.queue.re_default_answer.search(prompt)
464 answer = answer[:1].upper()
465 os.unlink(temp_filename)
470 Upload.pkg.changes["process-new note"] = note
471 Upload.dump_vars(Cnf["Dir::Queue::New"])
473 ################################################################################
477 less_fd = os.popen("less -R -", 'w', 0)
478 stdout_fd = sys.stdout
481 examine_package.display_changes(Upload.pkg.changes_file)
482 files = Upload.pkg.files
483 for file in files.keys():
484 if files[file].has_key("new"):
485 type = files[file]["type"]
487 examine_package.check_deb(file)
489 examine_package.check_dsc(file)
491 sys.stdout = stdout_fd
493 if errno.errorcode[e.errno] == 'EPIPE':
494 daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
498 except KeyboardInterrupt:
499 daklib.utils.warn("[examine_package] Caught C-c; skipping.")
502 ################################################################################
504 ## FIXME: horribly Debian specific
506 def do_bxa_notification():
507 files = Upload.pkg.files
509 for file in files.keys():
510 if files[file]["type"] == "deb":
511 control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
513 summary += "Package: %s\n" % (control.Find("Package"))
514 summary += "Description: %s\n" % (control.Find("Description"))
515 Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
516 bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
517 daklib.utils.send_mail(bxa_mail)
519 ################################################################################
521 def add_overrides (new):
522 changes = Upload.pkg.changes
523 files = Upload.pkg.files
525 projectB.query("BEGIN WORK")
526 for suite in changes["suite"].keys():
527 suite_id = daklib.database.get_suite_id(suite)
528 for pkg in new.keys():
529 component_id = daklib.database.get_component_id(new[pkg]["component"])
530 type_id = daklib.database.get_override_type_id(new[pkg]["type"])
531 priority_id = new[pkg]["priority id"]
532 section_id = new[pkg]["section id"]
533 projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
534 for file in new[pkg]["files"]:
535 if files[file].has_key("new"):
536 del files[file]["new"]
539 projectB.query("COMMIT WORK")
541 if Cnf.FindB("Dinstall::BXANotify"):
542 do_bxa_notification()
544 ################################################################################
546 def prod_maintainer ():
547 # Here we prepare an editor and get them ready to prod...
548 temp_filename = daklib.utils.temp_filename()
549 editor = os.environ.get("EDITOR","vi")
552 os.system("%s %s" % (editor, temp_filename))
553 file = daklib.utils.open_file(temp_filename)
554 prod_message = "".join(file.readlines())
556 print "Prod message:"
557 print daklib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
558 prompt = "[P]rod, Edit, Abandon, Quit ?"
560 while prompt.find(answer) == -1:
561 answer = daklib.utils.our_raw_input(prompt)
562 m = daklib.queue.re_default_answer.search(prompt)
565 answer = answer[:1].upper()
566 os.unlink(temp_filename)
571 # Otherwise, do the proding...
572 user_email_address = daklib.utils.whoami() + " <%s>" % (
573 Cnf["Dinstall::MyAdminAddress"])
577 Subst["__FROM_ADDRESS__"] = user_email_address
578 Subst["__PROD_MESSAGE__"] = prod_message
579 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
581 prod_mail_message = daklib.utils.TemplateSubst(
582 Subst,Cnf["Dir::Templates"]+"/process-new.prod")
584 # Send the prod mail if appropriate
585 if not Cnf["Dinstall::Options::No-Mail"]:
586 daklib.utils.send_mail(prod_mail_message)
588 print "Sent proding message"
590 ################################################################################
594 files = Upload.pkg.files
595 changes = Upload.pkg.changes
597 # Make a copy of distribution we can happily trample on
598 changes["suite"] = copy.copy(changes["distribution"])
600 # Fix up the list of target suites
601 for suite in changes["suite"].keys():
602 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
604 (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1,
605 daklib.database.get_suite_id(override) == -1)
607 (oinv, newinv) = ("", "")
608 if olderr: oinv = "invalid "
609 if newerr: ninv = "invalid "
610 print "warning: overriding %ssuite %s to %ssuite %s" % (
611 oinv, suite, ninv, override)
612 del changes["suite"][suite]
613 changes["suite"][override] = 1
615 for suite in changes["suite"].keys():
616 suite_id = daklib.database.get_suite_id(suite)
618 daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
620 # The main NEW processing loop
623 # Find out what's new
624 new = daklib.queue.determine_new(changes, files, projectB)
630 if Options["No-Action"] or Options["Automatic"]:
633 (broken, note) = print_new(new, 0)
636 if not broken and not note:
637 prompt = "Add overrides, "
639 print "W: [!] marked entries must be fixed before package can be processed."
641 print "W: note must be removed before package can be processed."
642 prompt += "Remove note, "
644 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
646 while prompt.find(answer) == -1:
647 answer = daklib.utils.our_raw_input(prompt)
648 m = daklib.queue.re_default_answer.search(prompt)
651 answer = answer[:1].upper()
654 done = add_overrides (new)
658 new = edit_overrides (new)
660 aborted = Upload.do_reject(1, Options["Manual-Reject"])
662 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
665 edit_note(changes.get("process-new note", ""))
669 confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
671 del changes["process-new note"]
677 ################################################################################
678 ################################################################################
679 ################################################################################
681 def usage (exit_code=0):
682 print """Usage: dak process-new [OPTION]... [CHANGES]...
683 -a, --automatic automatic run
684 -h, --help show this help and exit.
685 -m, --manual-reject=MSG manual reject with `msg'
686 -n, --no-action don't do anything
687 -V, --version display the version number and exit"""
690 ################################################################################
693 global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
695 Cnf = daklib.utils.get_conf()
697 Arguments = [('a',"automatic","Process-New::Options::Automatic"),
698 ('h',"help","Process-New::Options::Help"),
699 ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
700 ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
701 ('n',"no-action","Process-New::Options::No-Action")]
703 for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
704 if not Cnf.has_key("Process-New::Options::%s" % (i)):
705 Cnf["Process-New::Options::%s" % (i)] = ""
707 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
708 Options = Cnf.SubTree("Process-New::Options")
713 Upload = daklib.queue.Upload(Cnf)
715 if not Options["No-Action"]:
716 Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
718 projectB = Upload.projectB
720 Sections = Section_Completer()
721 Priorities = Priority_Completer()
722 readline.parse_and_bind("tab: complete")
726 ################################################################################
731 files = Upload.pkg.files
735 for file in files.keys():
736 if files[file]["type"] == "byhand":
737 if os.path.exists(file):
738 print "W: %s still present; please process byhand components and try again." % (file)
744 if Options["No-Action"]:
747 if Options["Automatic"] and not Options["No-Action"]:
749 prompt = "[A]ccept, Manual reject, Skip, Quit ?"
751 prompt = "Manual reject, [S]kip, Quit ?"
753 while prompt.find(answer) == -1:
754 answer = daklib.utils.our_raw_input(prompt)
755 m = daklib.queue.re_default_answer.search(prompt)
758 answer = answer[:1].upper()
765 Upload.do_reject(1, Options["Manual-Reject"])
766 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
773 ################################################################################
777 if not Options["No-Action"]:
781 lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
784 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
787 daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
789 print("Unable to get accepted lock (try %d of 10)" % retry)
793 (summary, short_summary) = Upload.build_summaries()
794 Upload.accept(summary, short_summary)
795 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
796 os.unlink(Cnf["Process-New::AcceptedLockFile"])
798 def check_status(files):
800 for file in files.keys():
801 if files[file]["type"] == "byhand":
803 elif files[file].has_key("new"):
807 def do_pkg(changes_file):
808 Upload.pkg.changes_file = changes_file
811 Upload.update_subst()
812 files = Upload.pkg.files
817 (new, byhand) = check_status(files)
823 (new, byhand) = check_status(files)
825 if not new and not byhand:
828 ################################################################################
831 accept_count = Upload.accept_count
832 accept_bytes = Upload.accept_bytes
838 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
839 Logger.log(["total",accept_count,accept_bytes])
841 if not Options["No-Action"]:
844 ################################################################################
846 def do_comments(dir, opref, npref, line, fn):
847 for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
848 lines = open("%s/%s" % (dir, comm)).readlines()
849 if len(lines) == 0 or lines[0] != line + "\n": continue
850 changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
851 and x.endswith(".changes") ]
852 changes_files = sort_changes(changes_files)
853 for f in changes_files:
854 f = daklib.utils.validate_changes_file_arg(f, 0)
857 fn(f, "".join(lines[1:]))
859 if opref != npref and not Options["No-Action"]:
860 newcomm = npref + comm[len(opref):]
861 os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
863 ################################################################################
865 def comment_accept(changes_file, comments):
866 Upload.pkg.changes_file = changes_file
869 Upload.update_subst()
870 files = Upload.pkg.files
873 return # dak wants to REJECT, crap
875 (new, byhand) = check_status(files)
876 if not new and not byhand:
879 ################################################################################
881 def comment_reject(changes_file, comments):
882 Upload.pkg.changes_file = changes_file
885 Upload.update_subst()
886 files = Upload.pkg.files
889 pass # dak has its own reasons to reject as well, which is fine
892 print "REJECT\n" + reject_message,
893 if not Options["No-Action"]:
894 Upload.do_reject(0, reject_message)
895 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
897 ################################################################################
900 changes_files = init()
901 if len(changes_files) > 50:
902 sys.stderr.write("Sorting changes...\n")
903 changes_files = sort_changes(changes_files)
905 # Kill me now? **FIXME**
906 Cnf["Dinstall::Options::No-Mail"] = ""
907 bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
908 if Cnf.has_key("Dinstall::Bcc"):
909 Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
911 Upload.Subst["__BCC__"] = bcc
913 commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
915 if changes_files != []:
916 sys.stderr.write("Can't specify any changes files if working with comments-dir")
918 do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
919 do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
921 for changes_file in changes_files:
922 changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
925 print "\n" + changes_file
926 do_pkg (changes_file)
930 ################################################################################
932 if __name__ == '__main__':