3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup <james@nocrew.org>
5 # $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # 23:12|<aj> I will not hush!
25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
26 # 23:13|<aj> I shall not be silenced!
27 # 23:13|<aj> The world shall know!
28 # 23:13|<aj> The world *must* know!
29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
30 # 23:13|<aj> yay powerpuff girls!!
31 # 23:13|<aj> buttercup's my favourite, who's yours?
32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
33 # 23:14|<aj> *AREN'T YOU*?!
34 # 23:15|<aj> I will not be treated like this.
35 # 23:15|<aj> I shall have my revenge.
36 # 23:15|<aj> I SHALL!!!
38 ################################################################################
40 import copy, errno, os, readline, stat, sys, time;
41 import apt_pkg, apt_inst;
42 import db_access, fernanda, katie, logging, utils;
45 lisa_version = "$Revision: 1.31 $";
58 ################################################################################
59 ################################################################################
60 ################################################################################
62 def reject (str, prefix="Rejected: "):
63 global reject_message;
65 reject_message += prefix + str + "\n";
68 global reject_message;
69 files = Katie.pkg.files;
72 for file in files.keys():
73 # The .orig.tar.gz can disappear out from under us is it's a
74 # duplicate of one in the archive.
75 if not files.has_key(file):
77 # Check that the source still exists
78 if files[file]["type"] == "deb":
79 source_version = files[file]["source version"];
80 source_package = files[file]["source package"];
81 if not Katie.pkg.changes["architecture"].has_key("source") \
82 and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()):
83 source_epochless_version = utils.re_no_epoch.sub('', source_version);
84 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
85 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
86 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
88 # Version and file overwrite checks
89 if files[file]["type"] == "deb":
90 reject(Katie.check_binary_against_db(file));
91 elif files[file]["type"] == "dsc":
92 reject(Katie.check_source_against_db(file));
93 (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
98 if Options["No-Action"] or Options["Automatic"]:
101 print "REJECT\n" + reject_message,;
102 prompt = "[R]eject, Skip, Quit ?";
104 while prompt.find(answer) == -1:
105 answer = utils.our_raw_input(prompt);
106 m = katie.re_default_answer.match(prompt);
109 answer = answer[:1].upper();
112 Katie.do_reject(0, reject_message);
113 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
122 ################################################################################
124 def determine_new (changes, files):
127 # Build up a list of potentially new things
128 for file in files.keys():
130 # Skip byhand elements
131 if f["type"] == "byhand":
134 priority = f["priority"];
135 section = f["section"];
137 if section == "non-US/main":
140 component = f["component"];
144 if not new.has_key(pkg):
146 new[pkg]["priority"] = priority;
147 new[pkg]["section"] = section;
148 new[pkg]["type"] = type;
149 new[pkg]["component"] = component;
150 new[pkg]["files"] = [];
152 old_type = new[pkg]["type"];
154 # source gets trumped by deb or udeb
155 if old_type == "dsc":
156 new[pkg]["priority"] = priority;
157 new[pkg]["section"] = section;
158 new[pkg]["type"] = type;
159 new[pkg]["component"] = component;
160 new[pkg]["files"].append(file);
161 if f.has_key("othercomponents"):
162 new[pkg]["othercomponents"] = f["othercomponents"];
164 for suite in changes["suite"].keys():
165 suite_id = db_access.get_suite_id(suite);
166 for pkg in new.keys():
167 component_id = db_access.get_component_id(new[pkg]["component"]);
168 type_id = db_access.get_override_type_id(new[pkg]["type"]);
169 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
172 for file in new[pkg]["files"]:
173 if files[file].has_key("new"):
174 del files[file]["new"];
177 if changes["suite"].has_key("stable"):
178 print "WARNING: overrides will be added for stable!";
179 if changes["suite"].has_key("oldstable"):
180 print "WARNING: overrides will be added for OLDstable!";
181 for pkg in new.keys():
182 if new[pkg].has_key("othercomponents"):
183 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
187 ################################################################################
189 def indiv_sg_compare (a, b):
190 """Sort by source name, source, version, 'have source', and
191 finally by filename."""
192 # Sort by source version
193 q = apt_pkg.VersionCompare(a["version"], b["version"]);
197 # Sort by 'have source'
198 a_has_source = a["architecture"].get("source");
199 b_has_source = b["architecture"].get("source");
200 if a_has_source and not b_has_source:
202 elif b_has_source and not a_has_source:
205 return cmp(a["filename"], b["filename"]);
207 ############################################################
209 def sg_compare (a, b):
212 """Sort by have note, time of oldest upload."""
214 a_note_state = a["note_state"];
215 b_note_state = b["note_state"];
216 if a_note_state < b_note_state:
218 elif a_note_state > b_note_state:
221 # Sort by time of oldest upload
222 return cmp(a["oldest"], b["oldest"]);
224 def sort_changes(changes_files):
225 """Sort into source groups, then sort each source group by version,
226 have source, filename. Finally, sort the source groups by have
227 note, time of oldest upload of each source upload."""
228 if len(changes_files) == 1:
229 return changes_files;
233 # Read in all the .changes files
234 for filename in changes_files:
236 Katie.pkg.changes_file = filename;
239 cache[filename] = copy.copy(Katie.pkg.changes);
240 cache[filename]["filename"] = filename;
242 sorted_list.append(filename);
244 # Divide the .changes into per-source groups
246 for filename in cache.keys():
247 source = cache[filename]["source"];
248 if not per_source.has_key(source):
249 per_source[source] = {};
250 per_source[source]["list"] = [];
251 per_source[source]["list"].append(cache[filename]);
252 # Determine oldest time and have note status for each source group
253 for source in per_source.keys():
254 source_list = per_source[source]["list"];
255 first = source_list[0];
256 oldest = os.stat(first["filename"])[stat.ST_MTIME];
258 for d in per_source[source]["list"]:
259 mtime = os.stat(d["filename"])[stat.ST_MTIME];
262 have_note += (d.has_key("lisa note"));
263 per_source[source]["oldest"] = oldest;
265 per_source[source]["note_state"] = 0; # none
266 elif have_note < len(source_list):
267 per_source[source]["note_state"] = 1; # some
269 per_source[source]["note_state"] = 2; # all
270 per_source[source]["list"].sort(indiv_sg_compare);
271 per_source_items = per_source.items();
272 per_source_items.sort(sg_compare);
273 for i in per_source_items:
274 for j in i[1]["list"]:
275 sorted_list.append(j["filename"]);
278 ################################################################################
280 class Section_Completer:
283 q = projectB.query("SELECT section FROM section");
284 for i in q.getresult():
285 self.sections.append(i[0]);
287 def complete(self, text, state):
291 for word in self.sections:
293 self.matches.append(word);
295 return self.matches[state]
299 ############################################################
301 class Priority_Completer:
303 self.priorities = [];
304 q = projectB.query("SELECT priority FROM priority");
305 for i in q.getresult():
306 self.priorities.append(i[0]);
308 def complete(self, text, state):
312 for word in self.priorities:
314 self.matches.append(word);
316 return self.matches[state]
320 ################################################################################
322 def check_valid (new):
323 for pkg in new.keys():
324 section = new[pkg]["section"];
325 priority = new[pkg]["priority"];
326 type = new[pkg]["type"];
327 new[pkg]["section id"] = db_access.get_section_id(section);
328 new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
330 if (section == "debian-installer" and type != "udeb") or \
331 (section != "debian-installer" and type == "udeb"):
332 new[pkg]["section id"] = -1;
333 if (priority == "source" and type != "dsc") or \
334 (priority != "source" and type == "dsc"):
335 new[pkg]["priority id"] = -1;
337 ################################################################################
339 def print_new (new, indexed, file=sys.stdout):
343 for pkg in new.keys():
345 section = new[pkg]["section"];
346 priority = new[pkg]["priority"];
347 if new[pkg]["section id"] == -1:
350 if new[pkg]["priority id"] == -1:
354 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
356 line = "%-20s %-20s %-20s" % (pkg, priority, section);
357 line = line.strip()+'\n';
359 note = Katie.pkg.changes.get("lisa note");
366 ################################################################################
370 if f.has_key("dbtype"):
372 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
375 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
377 # Validate the override type
378 type_id = db_access.get_override_type_id(type);
380 utils.fubar("invalid type (%s) for new. Say wha?" % (type));
384 ################################################################################
386 def index_range (index):
390 return "1-%s" % (index);
392 ################################################################################
393 ################################################################################
396 # Write the current data to a temporary file
397 temp_filename = utils.temp_filename();
398 temp_file = utils.open_file(temp_filename, 'w');
399 print_new (new, 0, temp_file);
401 # Spawn an editor on that file
402 editor = os.environ.get("EDITOR","vi")
403 result = os.system("%s %s" % (editor, temp_filename))
405 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
406 # Read the edited data back in
407 temp_file = utils.open_file(temp_filename);
408 lines = temp_file.readlines();
410 os.unlink(temp_filename);
417 # Pad the list if necessary
418 s[len(s):3] = [None] * (3-len(s));
419 (pkg, priority, section) = s[:3];
420 if not new.has_key(pkg):
421 utils.warn("Ignoring unknown package '%s'" % (pkg));
423 # Strip off any invalid markers, print_new will readd them.
424 if section.endswith("[!]"):
425 section = section[:-3];
426 if priority.endswith("[!]"):
427 priority = priority[:-3];
428 for file in new[pkg]["files"]:
429 Katie.pkg.files[file]["section"] = section;
430 Katie.pkg.files[file]["priority"] = priority;
431 new[pkg]["section"] = section;
432 new[pkg]["priority"] = priority;
434 ################################################################################
436 def edit_index (new, index):
437 priority = new[index]["priority"]
438 section = new[index]["section"]
439 type = new[index]["type"];
442 print "\t".join([index, priority, section]);
446 prompt = "[B]oth, Priority, Section, Done ? ";
448 prompt = "[S]ection, Done ? ";
449 edit_priority = edit_section = 0;
451 while prompt.find(answer) == -1:
452 answer = utils.our_raw_input(prompt);
453 m = katie.re_default_answer.match(prompt)
456 answer = answer[:1].upper()
463 edit_priority = edit_section = 1;
469 readline.set_completer(Priorities.complete);
471 while not got_priority:
472 new_priority = utils.our_raw_input("New priority: ").strip();
473 if new_priority not in Priorities.priorities:
474 print "E: '%s' is not a valid priority, try again." % (new_priority);
477 priority = new_priority;
481 readline.set_completer(Sections.complete);
483 while not got_section:
484 new_section = utils.our_raw_input("New section: ").strip();
485 if new_section not in Sections.sections:
486 print "E: '%s' is not a valid section, try again." % (new_section);
489 section = new_section;
491 # Reset the readline completer
492 readline.set_completer(None);
494 for file in new[index]["files"]:
495 Katie.pkg.files[file]["section"] = section;
496 Katie.pkg.files[file]["priority"] = priority;
497 new[index]["priority"] = priority;
498 new[index]["section"] = section;
501 ################################################################################
503 def edit_overrides (new):
512 new_index[index] = i;
514 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
517 while not got_answer:
518 answer = utils.our_raw_input(prompt);
519 if not utils.str_isnum(answer):
520 answer = answer[:1].upper();
521 if answer == "E" or answer == "D":
523 elif katie.re_isanum.match (answer):
524 answer = int(answer);
525 if (answer < 1) or (answer > index):
526 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index));
535 edit_index (new, new_index[answer]);
539 ################################################################################
542 # Write the current data to a temporary file
543 temp_filename = utils.temp_filename();
544 temp_file = utils.open_file(temp_filename, 'w');
545 temp_file.write(note);
547 editor = os.environ.get("EDITOR","vi")
550 os.system("%s %s" % (editor, temp_filename))
551 temp_file = utils.open_file(temp_filename);
552 note = temp_file.read().rstrip();
555 print utils.prefix_multi_line_string(note," ");
556 prompt = "[D]one, Edit, Abandon, Quit ?"
558 while prompt.find(answer) == -1:
559 answer = utils.our_raw_input(prompt);
560 m = katie.re_default_answer.search(prompt);
563 answer = answer[:1].upper();
564 os.unlink(temp_filename);
569 Katie.pkg.changes["lisa note"] = note;
570 Katie.dump_vars(Cnf["Dir::Queue::New"]);
572 ################################################################################
576 less_fd = os.popen("less -R -", 'w', 0);
577 stdout_fd = sys.stdout;
579 sys.stdout = less_fd;
580 fernanda.display_changes(Katie.pkg.changes_file);
581 files = Katie.pkg.files;
582 for file in files.keys():
583 if files[file].has_key("new"):
584 type = files[file]["type"];
586 fernanda.check_deb(file);
588 fernanda.check_dsc(file);
590 sys.stdout = stdout_fd;
592 if errno.errorcode[e.errno] == 'EPIPE':
593 utils.warn("[fernanda] Caught EPIPE; skipping.");
597 except KeyboardInterrupt:
598 utils.warn("[fernanda] Caught C-c; skipping.");
601 ################################################################################
603 ## FIXME: horribly Debian specific
605 def do_bxa_notification():
606 files = Katie.pkg.files;
608 for file in files.keys():
609 if files[file]["type"] == "deb":
610 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
612 summary += "Package: %s\n" % (control.Find("Package"));
613 summary += "Description: %s\n" % (control.Find("Description"));
614 Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
615 bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
616 utils.send_mail(bxa_mail);
618 ################################################################################
620 def add_overrides (new):
621 changes = Katie.pkg.changes;
622 files = Katie.pkg.files;
624 projectB.query("BEGIN WORK");
625 for suite in changes["suite"].keys():
626 suite_id = db_access.get_suite_id(suite);
627 for pkg in new.keys():
628 component_id = db_access.get_component_id(new[pkg]["component"]);
629 type_id = db_access.get_override_type_id(new[pkg]["type"]);
630 priority_id = new[pkg]["priority id"];
631 section_id = new[pkg]["section id"];
632 projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
633 for file in new[pkg]["files"]:
634 if files[file].has_key("new"):
635 del files[file]["new"];
638 projectB.query("COMMIT WORK");
640 if Cnf.FindB("Dinstall::BXANotify"):
641 do_bxa_notification();
643 ################################################################################
645 def prod_maintainer ():
646 # Here we prepare an editor and get them ready to prod...
647 temp_filename = utils.temp_filename();
648 editor = os.environ.get("EDITOR","vi")
651 os.system("%s %s" % (editor, temp_filename))
652 file = utils.open_file(temp_filename);
653 prod_message = "".join(file.readlines());
655 print "Prod message:";
656 print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1);
657 prompt = "[P]rod, Edit, Abandon, Quit ?"
659 while prompt.find(answer) == -1:
660 answer = utils.our_raw_input(prompt);
661 m = katie.re_default_answer.search(prompt);
664 answer = answer[:1].upper();
665 os.unlink(temp_filename);
670 # Otherwise, do the proding...
671 user_email_address = utils.whoami() + " <%s>" % (
672 Cnf["Dinstall::MyAdminAddress"]);
676 Subst["__FROM_ADDRESS__"] = user_email_address;
677 Subst["__PROD_MESSAGE__"] = prod_message;
678 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
680 prod_mail_message = utils.TemplateSubst(
681 Subst,Cnf["Dir::Templates"]+"/lisa.prod");
683 # Send the prod mail if appropriate
684 if not Cnf["Dinstall::Options::No-Mail"]:
685 utils.send_mail(prod_mail_message);
687 print "Sent proding message";
689 ################################################################################
693 files = Katie.pkg.files;
694 changes = Katie.pkg.changes;
696 # Make a copy of distribution we can happily trample on
697 changes["suite"] = copy.copy(changes["distribution"]);
699 # Fix up the list of target suites
700 for suite in changes["suite"].keys():
701 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
703 del changes["suite"][suite];
704 changes["suite"][override] = 1;
706 for suite in changes["suite"].keys():
707 suite_id = db_access.get_suite_id(suite);
709 utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
711 # The main NEW processing loop
714 # Find out what's new
715 new = determine_new(changes, files);
721 if Options["No-Action"] or Options["Automatic"]:
724 (broken, note) = print_new(new, 0);
727 if not broken and not note:
728 prompt = "Add overrides, ";
730 print "W: [!] marked entries must be fixed before package can be processed.";
732 print "W: note must be removed before package can be processed.";
733 prompt += "Remove note, ";
735 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?";
737 while prompt.find(answer) == -1:
738 answer = utils.our_raw_input(prompt);
739 m = katie.re_default_answer.search(prompt);
742 answer = answer[:1].upper()
745 done = add_overrides (new);
749 new = edit_overrides (new);
751 aborted = Katie.do_reject(1, Options["Manual-Reject"]);
753 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
756 edit_note(changes.get("lisa note", ""));
760 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower();
762 del changes["lisa note"];
768 ################################################################################
769 ################################################################################
770 ################################################################################
772 def usage (exit_code=0):
773 print """Usage: lisa [OPTION]... [CHANGES]...
774 -a, --automatic automatic run
775 -h, --help show this help and exit.
776 -m, --manual-reject=MSG manual reject with `msg'
777 -n, --no-action don't do anything
778 -V, --version display the version number and exit"""
781 ################################################################################
784 global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
786 Cnf = utils.get_conf();
788 Arguments = [('a',"automatic","Lisa::Options::Automatic"),
789 ('h',"help","Lisa::Options::Help"),
790 ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
791 ('n',"no-action","Lisa::Options::No-Action"),
792 ('V',"version","Lisa::Options::Version")];
794 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
795 if not Cnf.has_key("Lisa::Options::%s" % (i)):
796 Cnf["Lisa::Options::%s" % (i)] = "";
798 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
799 Options = Cnf.SubTree("Lisa::Options")
804 if Options["Version"]:
805 print "lisa %s" % (lisa_version);
808 Katie = katie.Katie(Cnf);
810 if not Options["No-Action"]:
811 Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
813 projectB = Katie.projectB;
815 Sections = Section_Completer();
816 Priorities = Priority_Completer();
817 readline.parse_and_bind("tab: complete");
819 return changes_files;
821 ################################################################################
826 files = Katie.pkg.files;
830 for file in files.keys():
831 if files[file]["type"] == "byhand":
832 if os.path.exists(file):
833 print "W: %s still present; please process byhand components and try again." % (file);
839 if Options["No-Action"]:
842 if Options["Automatic"] and not Options["No-Action"]:
844 prompt = "[A]ccept, Manual reject, Skip, Quit ?";
846 prompt = "Manual reject, [S]kip, Quit ?";
848 while prompt.find(answer) == -1:
849 answer = utils.our_raw_input(prompt);
850 m = katie.re_default_answer.search(prompt);
853 answer = answer[:1].upper();
860 Katie.do_reject(1, Options["Manual-Reject"]);
861 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
868 ################################################################################
872 if not Options["No-Action"]:
876 lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL);
879 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
882 utils.fubar("Couldn't obtain lock; assuming jennifer is already running.");
884 print("Unable to get accepted lock (try %d of 10)" % retry);
888 (summary, short_summary) = Katie.build_summaries();
889 Katie.accept(summary, short_summary);
890 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
891 os.unlink(Cnf["Lisa::AcceptedLockFile"]);
893 def check_status(files):
895 for file in files.keys():
896 if files[file]["type"] == "byhand":
898 elif files[file].has_key("new"):
900 return (new, byhand);
902 def do_pkg(changes_file):
903 Katie.pkg.changes_file = changes_file;
906 Katie.update_subst();
907 files = Katie.pkg.files;
912 (new, byhand) = check_status(files);
918 (new, byhand) = check_status(files);
920 if not new and not byhand:
923 ################################################################################
926 accept_count = Katie.accept_count;
927 accept_bytes = Katie.accept_bytes;
933 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
934 Logger.log(["total",accept_count,accept_bytes]);
936 if not Options["No-Action"]:
939 ################################################################################
942 changes_files = init();
943 if len(changes_files) > 50:
944 sys.stderr.write("Sorting changes...\n");
945 changes_files = sort_changes(changes_files);
947 # Kill me now? **FIXME**
948 Cnf["Dinstall::Options::No-Mail"] = "";
949 bcc = "X-Katie: lisa %s" % (lisa_version);
950 if Cnf.has_key("Dinstall::Bcc"):
951 Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
953 Katie.Subst["__BCC__"] = bcc;
955 for changes_file in changes_files:
956 changes_file = utils.validate_changes_file_arg(changes_file, 0);
959 print "\n" + changes_file;
960 do_pkg (changes_file);
964 ################################################################################
966 if __name__ == '__main__':