3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003 James Troup <james@nocrew.org>
5 # $Id: lisa,v 1.21 2003-02-11 19:39:28 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # 23:12|<aj> I will not hush!
25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
26 # 23:13|<aj> I shall not be silenced!
27 # 23:13|<aj> The world shall know!
28 # 23:13|<aj> The world *must* know!
29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
30 # 23:13|<aj> yay powerpuff girls!!
31 # 23:13|<aj> buttercup's my favourite, who's yours?
32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
33 # 23:14|<aj> *AREN'T YOU*?!
34 # 23:15|<aj> I will not be treated like this.
35 # 23:15|<aj> I shall have my revenge.
36 # 23:15|<aj> I SHALL!!!
38 ################################################################################
40 import copy, errno, os, readline, stat, sys, tempfile;
41 import apt_pkg, apt_inst;
42 import db_access, fernanda, katie, logging, utils;
45 lisa_version = "$Revision: 1.21 $";
58 ################################################################################
59 ################################################################################
60 ################################################################################
62 def reject (str, prefix="Rejected: "):
63 global reject_message;
65 reject_message += prefix + str + "\n";
68 global reject_message;
69 files = Katie.pkg.files;
72 for file in files.keys():
73 # Check that the source still exists
74 if files[file]["type"] == "deb":
75 source_version = files[file]["source version"];
76 source_package = files[file]["source package"];
77 if not Katie.pkg.changes["architecture"].has_key("source") \
78 and not Katie.source_exists(source_package, source_version):
79 source_epochless_version = utils.re_no_epoch.sub('', source_version);
80 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
81 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
82 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
84 # Version and file overwrite checks
85 if files[file]["type"] == "deb":
86 reject(Katie.check_binary_against_db(file));
87 elif files[file]["type"] == "dsc":
88 reject(Katie.check_source_against_db(file));
89 (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
94 if Options["No-Action"] or Options["Automatic"]:
97 print "REJECT\n" + reject_message,;
98 prompt = "[R]eject, Skip, Quit ?";
100 while prompt.find(answer) == -1:
101 answer = utils.our_raw_input(prompt);
102 m = katie.re_default_answer.match(prompt);
105 answer = answer[:1].upper();
108 Katie.do_reject(0, reject_message);
109 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
118 ################################################################################
120 def determine_new (changes, files):
123 # Build up a list of potentially new things
124 for file in files.keys():
126 # Skip byhand elements
127 if f["type"] == "byhand":
130 priority = f["priority"];
131 section = f["section"];
133 if section == "non-US/main":
136 component = f["component"];
140 if not new.has_key(pkg):
142 new[pkg]["priority"] = priority;
143 new[pkg]["section"] = section;
144 new[pkg]["type"] = type;
145 new[pkg]["component"] = component;
146 new[pkg]["files"] = [];
148 old_type = new[pkg]["type"];
150 # source gets trumped by deb or udeb
151 if old_type == "dsc":
152 new[pkg]["priority"] = priority;
153 new[pkg]["section"] = section;
154 new[pkg]["type"] = type;
155 new[pkg]["component"] = component;
156 new[pkg]["files"].append(file);
157 if f.has_key("othercomponents"):
158 new[pkg]["othercomponents"] = f["othercomponents"];
160 for suite in changes["suite"].keys():
161 suite_id = db_access.get_suite_id(suite);
162 for pkg in new.keys():
163 component_id = db_access.get_component_id(new[pkg]["component"]);
164 type_id = db_access.get_override_type_id(new[pkg]["type"]);
165 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
168 for file in new[pkg]["files"]:
169 if files[file].has_key("new"):
170 del files[file]["new"];
173 if changes["suite"].has_key("stable"):
174 print "WARNING: overrides will be added for stable!";
175 for pkg in new.keys():
176 if new[pkg].has_key("othercomponents"):
177 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
181 ################################################################################
183 def indiv_sg_compare (a, b):
184 """Sort by source name, source, version, 'have source', and
185 finally by filename."""
186 # Sort by source version
187 q = apt_pkg.VersionCompare(a["version"], b["version"]);
191 # Sort by 'have source'
192 a_has_source = a["architecture"].get("source");
193 b_has_source = b["architecture"].get("source");
194 if a_has_source and not b_has_source:
196 elif b_has_source and not a_has_source:
199 return cmp(a["filename"], b["filename"]);
201 ############################################################
203 def sg_compare (a, b):
206 """Sort by have note, time of oldest upload."""
208 a_note_state = a["note_state"];
209 b_note_state = b["note_state"];
210 if a_note_state < b_note_state:
212 elif a_note_state > b_note_state:
215 # Sort by time of oldest upload
216 return cmp(a["oldest"], b["oldest"]);
218 def sort_changes(changes_files):
219 """Sort into source groups, then sort each source group by version,
220 have source, filename. Finally, sort the source groups by have
221 note, time of oldest upload of each source upload."""
222 if len(changes_files) == 1:
223 return changes_files;
227 # Read in all the .changes files
228 for filename in changes_files:
230 Katie.pkg.changes_file = filename;
233 cache[filename] = copy.copy(Katie.pkg.changes);
234 cache[filename]["filename"] = filename;
236 sorted_list.append(filename);
238 # Divide the .changes into per-source groups
240 for filename in cache.keys():
241 source = cache[filename]["source"];
242 if not per_source.has_key(source):
243 per_source[source] = {};
244 per_source[source]["list"] = [];
245 per_source[source]["list"].append(cache[filename]);
246 # Determine oldest time and have note status for each source group
247 for source in per_source.keys():
248 source_list = per_source[source]["list"];
249 first = source_list[0];
250 oldest = os.stat(first["filename"])[stat.ST_CTIME];
252 for d in per_source[source]["list"]:
253 ctime = os.stat(d["filename"])[stat.ST_CTIME];
256 have_note += (d.has_key("lisa note"));
257 per_source[source]["oldest"] = oldest;
259 per_source[source]["note_state"] = 0; # none
260 elif have_note < len(source_list):
261 per_source[source]["note_state"] = 1; # some
263 per_source[source]["note_state"] = 2; # all
264 per_source[source]["list"].sort(indiv_sg_compare);
265 per_source_items = per_source.items();
266 per_source_items.sort(sg_compare);
267 for i in per_source_items:
268 for j in i[1]["list"]:
269 sorted_list.append(j["filename"]);
272 ################################################################################
274 class Section_Completer:
277 q = projectB.query("SELECT section FROM section");
278 for i in q.getresult():
279 self.sections.append(i[0]);
281 def complete(self, text, state):
285 for word in self.sections:
287 self.matches.append(word);
289 return self.matches[state]
293 ############################################################
295 class Priority_Completer:
297 self.priorities = [];
298 q = projectB.query("SELECT priority FROM priority");
299 for i in q.getresult():
300 self.priorities.append(i[0]);
302 def complete(self, text, state):
306 for word in self.priorities:
308 self.matches.append(word);
310 return self.matches[state]
314 ################################################################################
316 def check_valid (new):
317 for pkg in new.keys():
318 section = new[pkg]["section"];
319 priority = new[pkg]["priority"];
320 type = new[pkg]["type"];
321 new[pkg]["section id"] = db_access.get_section_id(section);
322 new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
324 if (section == "debian-installer" and type != "udeb") or \
325 (section != "debian-installer" and type == "udeb"):
326 new[pkg]["section id"] = -1;
327 if (priority == "source" and type != "dsc") or \
328 (priority != "source" and type == "dsc"):
329 new[pkg]["priority id"] = -1;
331 ################################################################################
333 def print_new (new, indexed, file=sys.stdout):
337 for pkg in new.keys():
339 section = new[pkg]["section"];
340 priority = new[pkg]["priority"];
341 if new[pkg]["section id"] == -1:
344 if new[pkg]["priority id"] == -1:
348 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
350 line = "%-20s %-20s %-20s" % (pkg, priority, section);
351 line = line.strip()+'\n';
353 note = Katie.pkg.changes.get("lisa note");
360 ################################################################################
364 if f.has_key("dbtype"):
366 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
369 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
371 # Validate the override type
372 type_id = db_access.get_override_type_id(type);
374 utils.fubar("invalid type (%s) for new. Say wha?" % (type));
378 ################################################################################
380 def index_range (index):
384 return "1-%s" % (index);
386 ################################################################################
387 ################################################################################
390 # Write the current data to a temporary file
391 temp_filename = tempfile.mktemp();
392 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
394 temp_file = utils.open_file(temp_filename, 'w');
395 print_new (new, 0, temp_file);
397 # Spawn an editor on that file
398 editor = os.environ.get("EDITOR","vi")
399 result = os.system("%s %s" % (editor, temp_filename))
401 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
402 # Read the edited data back in
403 temp_file = utils.open_file(temp_filename);
404 lines = temp_file.readlines();
406 os.unlink(temp_filename);
413 # Pad the list if necessary
414 s[len(s):3] = [None] * (3-len(s));
415 (pkg, priority, section) = s[:3];
416 if not new.has_key(pkg):
417 utils.warn("Ignoring unknown package '%s'" % (pkg));
419 # Strip off any invalid markers, print_new will readd them.
420 if section.endswith("[!]"):
421 section = section[:-3];
422 if priority.endswith("[!]"):
423 priority = priority[:-3];
424 for file in new[pkg]["files"]:
425 Katie.pkg.files[file]["section"] = section;
426 Katie.pkg.files[file]["priority"] = priority;
427 new[pkg]["section"] = section;
428 new[pkg]["priority"] = priority;
430 ################################################################################
432 def edit_index (new, index):
433 priority = new[index]["priority"]
434 section = new[index]["section"]
435 type = new[index]["type"];
438 print "\t".join([index, priority, section]);
442 prompt = "[B]oth, Priority, Section, Done ? ";
444 prompt = "[S]ection, Done ? ";
445 edit_priority = edit_section = 0;
447 while prompt.find(answer) == -1:
448 answer = utils.our_raw_input(prompt);
449 m = katie.re_default_answer.match(prompt)
452 answer = answer[:1].upper()
459 edit_priority = edit_section = 1;
465 readline.set_completer(Priorities.complete);
467 while not got_priority:
468 new_priority = utils.our_raw_input("New priority: ").strip();
469 if Priorities.priorities.count(new_priority) == 0:
470 print "E: '%s' is not a valid priority, try again." % (new_priority);
473 priority = new_priority;
477 readline.set_completer(Sections.complete);
479 while not got_section:
480 new_section = utils.our_raw_input("New section: ").strip();
481 if Sections.sections.count(new_section) == 0:
482 print "E: '%s' is not a valid section, try again." % (new_section);
485 section = new_section;
487 # Reset the readline completer
488 readline.set_completer(None);
490 for file in new[index]["files"]:
491 Katie.pkg.files[file]["section"] = section;
492 Katie.pkg.files[file]["priority"] = priority;
493 new[index]["priority"] = priority;
494 new[index]["section"] = section;
497 ################################################################################
499 def edit_overrides (new):
508 new_index[index] = i;
510 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
513 while not got_answer:
514 answer = utils.our_raw_input(prompt);
515 if not utils.str_isnum(answer):
516 answer = answer[:1].upper();
517 if answer == "E" or answer == "D":
519 elif katie.re_isanum.match (answer):
520 answer = int(answer);
521 if (answer < 1) or (answer > index):
522 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index));
531 edit_index (new, new_index[answer]);
535 ################################################################################
538 # Write the current data to a temporary file
539 temp_filename = tempfile.mktemp();
540 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
542 temp_file = utils.open_file(temp_filename, 'w');
543 temp_file.write(note);
545 editor = os.environ.get("EDITOR","vi")
548 os.system("%s %s" % (editor, temp_filename))
549 temp_file = utils.open_file(temp_filename);
550 note = temp_file.read().rstrip();
553 print utils.prefix_multi_line_string(note," ");
554 prompt = "[D]one, Edit, Abandon, Quit ?"
556 while prompt.find(answer) == -1:
557 answer = utils.our_raw_input(prompt);
558 m = katie.re_default_answer.search(prompt);
561 answer = answer[:1].upper();
562 os.unlink(temp_filename);
567 Katie.pkg.changes["lisa note"] = note;
568 Katie.dump_vars(Cnf["Dir::Queue::New"]);
570 ################################################################################
574 less_fd = os.popen("less -R -", 'w', 0);
575 stdout_fd = sys.stdout;
577 sys.stdout = less_fd;
578 fernanda.display_changes(Katie.pkg.changes_file);
579 files = Katie.pkg.files;
580 for file in files.keys():
581 if files[file].has_key("new"):
582 type = files[file]["type"];
584 fernanda.check_deb(file);
586 fernanda.check_dsc(file);
588 sys.stdout = stdout_fd;
590 if errno.errorcode[e.errno] == 'EPIPE':
591 utils.warn("[fernanda] Caught EPIPE; skipping.");
595 except KeyboardInterrupt:
596 utils.warn("[fernanda] Caught C-c; skipping.");
599 ################################################################################
601 ## FIXME: horribly Debian specific
603 def do_bxa_notification():
604 files = Katie.pkg.files;
606 for file in files.keys():
607 if files[file]["type"] == "deb":
608 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
610 summary += "Package: %s\n" % (control.Find("Package"));
611 summary += "Description: %s\n" % (control.Find("Description"));
612 Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
613 bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
614 utils.send_mail(bxa_mail,"");
616 ################################################################################
618 def add_overrides (new):
619 changes = Katie.pkg.changes;
620 files = Katie.pkg.files;
622 projectB.query("BEGIN WORK");
623 for suite in changes["suite"].keys():
624 suite_id = db_access.get_suite_id(suite);
625 for pkg in new.keys():
626 component_id = db_access.get_component_id(new[pkg]["component"]);
627 type_id = db_access.get_override_type_id(new[pkg]["type"]);
628 priority_id = new[pkg]["priority id"];
629 section_id = new[pkg]["section id"];
630 projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
631 for file in new[pkg]["files"]:
632 if files[file].has_key("new"):
633 del files[file]["new"];
636 projectB.query("COMMIT WORK");
638 if Cnf.FindB("Dinstall::BXANotify"):
639 do_bxa_notification();
641 ################################################################################
645 files = Katie.pkg.files;
646 changes = Katie.pkg.changes;
648 # Make a copy of distribution we can happily trample on
649 changes["suite"] = copy.copy(changes["distribution"]);
651 # Fix up the list of target suites
652 for suite in changes["suite"].keys():
653 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
655 del changes["suite"][suite];
656 changes["suite"][override] = 1;
658 for suite in changes["suite"].keys():
659 suite_id = db_access.get_suite_id(suite);
661 utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
663 # The main NEW processing loop
666 # Find out what's new
667 new = determine_new(changes, files);
673 if Options["No-Action"] or Options["Automatic"]:
676 (broken, note) = print_new(new, 0);
679 if not broken and not note:
680 prompt = "Add overrides, ";
682 print "W: [!] marked entries must be fixed before package can be processed.";
684 print "W: note must be removed before package can be processed.";
685 prompt += "Remove note, ";
687 prompt += "Edit overrides, Check, Manual reject, Note edit, [S]kip, Quit ?";
689 while prompt.find(answer) == -1:
690 answer = utils.our_raw_input(prompt);
691 m = katie.re_default_answer.search(prompt);
694 answer = answer[:1].upper()
697 done = add_overrides (new);
701 new = edit_overrides (new);
703 aborted = Katie.do_reject(1, Options["Manual-Reject"]);
705 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
708 edit_note(changes.get("lisa note", ""));
710 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower();
712 del changes["lisa note"];
718 ################################################################################
719 ################################################################################
720 ################################################################################
722 def usage (exit_code=0):
723 print """Usage: lisa [OPTION]... [CHANGES]...
724 -a, --automatic automatic run
725 -h, --help show this help and exit.
726 -m, --manual-reject=MSG manual reject with `msg'
727 -n, --no-action don't do anything
728 -V, --version display the version number and exit"""
731 ################################################################################
734 global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
736 Cnf = utils.get_conf();
738 Arguments = [('a',"automatic","Lisa::Options::Automatic"),
739 ('h',"help","Lisa::Options::Help"),
740 ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
741 ('n',"no-action","Lisa::Options::No-Action"),
742 ('V',"version","Lisa::Options::Version")];
744 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
745 if not Cnf.has_key("Lisa::Options::%s" % (i)):
746 Cnf["Lisa::Options::%s" % (i)] = "";
748 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
749 Options = Cnf.SubTree("Lisa::Options")
754 if Options["Version"]:
755 print "lisa %s" % (lisa_version);
758 Katie = katie.Katie(Cnf);
760 if not Options["No-Action"]:
761 Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
763 projectB = Katie.projectB;
765 Sections = Section_Completer();
766 Priorities = Priority_Completer();
767 readline.parse_and_bind("tab: complete");
769 return changes_files;
771 ################################################################################
776 files = Katie.pkg.files;
780 for file in files.keys():
781 if files[file]["type"] == "byhand":
782 if os.path.exists(file):
783 print "W: %s still present; please process byhand components and try again." % (file);
789 if Options["No-Action"]:
792 if Options["Automatic"] and not Options["No-Action"]:
794 prompt = "[A]ccept, Manual reject, Skip, Quit ?";
796 prompt = "Manual reject, [S]kip, Quit ?";
798 while prompt.find(answer) == -1:
799 answer = utils.our_raw_input(prompt);
800 m = katie.re_default_answer.search(prompt);
803 answer = answer[:1].upper();
810 Katie.do_reject(1, Options["Manual-Reject"]);
811 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
818 ################################################################################
822 if not Options["No-Action"]:
823 (summary, short_summary) = Katie.build_summaries();
824 Katie.accept(summary, short_summary);
825 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
827 def check_status(files):
829 for file in files.keys():
830 if files[file]["type"] == "byhand":
832 elif files[file].has_key("new"):
834 return (new, byhand);
836 def do_pkg(changes_file):
837 Katie.pkg.changes_file = changes_file;
840 Katie.update_subst();
841 files = Katie.pkg.files;
846 (new, byhand) = check_status(files);
852 (new, byhand) = check_status(files);
854 if not new and not byhand:
857 ################################################################################
860 accept_count = Katie.accept_count;
861 accept_bytes = Katie.accept_bytes;
867 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
868 Logger.log(["total",accept_count,accept_bytes]);
870 if not Options["No-Action"]:
873 ################################################################################
876 changes_files = init();
877 if len(changes_files) > 50:
878 sys.stderr.write("Sorting changes...\n");
879 changes_files = sort_changes(changes_files);
881 # Kill me now? **FIXME**
882 Cnf["Dinstall::Options::No-Mail"] = "";
883 bcc = "X-Katie: lisa %s" % (lisa_version);
884 if Cnf.has_key("Dinstall::Bcc"):
885 Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
887 Katie.Subst["__BCC__"] = bcc;
889 for changes_file in changes_files:
890 changes_file = utils.validate_changes_file_arg(changes_file, 0);
893 print "\n" + changes_file;
894 do_pkg (changes_file);
898 ################################################################################
900 if __name__ == '__main__':