3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002 James Troup <james@nocrew.org>
5 # $Id: lisa,v 1.14 2002-05-19 22:33:56 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # 23:12|<aj> I will not hush!
25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
26 # 23:13|<aj> I shall not be silenced!
27 # 23:13|<aj> The world shall know!
28 # 23:13|<aj> The world *must* know!
29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
30 # 23:13|<aj> yay powerpuff girls!!
31 # 23:13|<aj> buttercup's my favourite, who's yours?
32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
33 # 23:14|<aj> *AREN'T YOU*?!
34 # 23:15|<aj> I will not be treated like this.
35 # 23:15|<aj> I shall have my revenge.
36 # 23:15|<aj> I SHALL!!!
38 ################################################################################
43 # We don't check error codes very thoroughly; the old 'trust jennifer'
44 # chess nut... db_access calls in particular
49 # Handle multiple different section/priorities (?)
50 # hardcoded debianness (debian-installer, source priority etc.) (?)
51 # Slang/ncurses interface (?)
52 # write changed sections/priority back to katie for later processing (?)
54 ################################################################################
56 import copy, errno, os, readline, string, stat, sys, tempfile;
57 import apt_pkg, apt_inst;
58 import db_access, fernanda, katie, logging, utils;
61 lisa_version = "$Revision: 1.14 $";
74 ################################################################################
75 ################################################################################
76 ################################################################################
78 def reject (str, prefix="Rejected: "):
79 global reject_message;
81 reject_message = reject_message + prefix + str + "\n";
84 global reject_message;
85 files = Katie.pkg.files;
88 for file in files.keys():
89 # Check that the source still exists
90 if files[file]["type"] == "deb":
91 source_version = files[file]["source version"];
92 source_package = files[file]["source package"];
93 if not Katie.pkg.changes["architecture"].has_key("source") \
94 and not Katie.source_exists(source_package, source_version):
95 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
97 # Version and file overwrite checks
98 if files[file]["type"] == "deb":
99 reject(Katie.check_binary_against_db(file));
100 elif files[file]["type"] == "dsc":
101 reject(Katie.check_source_against_db(file));
102 (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
107 if Options["No-Action"] or Options["Automatic"]:
110 print "REJECT\n" + reject_message,;
111 prompt = "[R]eject, Skip, Quit ?";
113 while string.find(prompt, answer) == -1:
114 answer = utils.our_raw_input(prompt);
115 m = katie.re_default_answer.match(prompt);
118 answer = string.upper(answer[:1]);
121 Katie.do_reject(0, reject_message);
129 ################################################################################
132 def determine_new (changes, files):
135 # Build up a list of potentially new things
136 for file in files.keys():
138 # Skip byhand elements
139 if f["type"] == "byhand":
142 priority = f["priority"];
143 section = f["section"];
145 if section == "non-US/main":
148 component = f["component"];
152 if not new.has_key(pkg):
154 new[pkg]["priority"] = priority;
155 new[pkg]["section"] = section;
156 new[pkg]["type"] = type;
157 new[pkg]["component"] = component;
158 new[pkg]["files"] = [];
160 old_type = new[pkg]["type"];
162 # source gets trumped by deb or udeb
163 if old_type == "dsc":
164 new[pkg]["priority"] = priority;
165 new[pkg]["section"] = section;
166 new[pkg]["type"] = type;
167 new[pkg]["component"] = component;
168 new[pkg]["files"].append(file);
169 if f.has_key("othercomponents"):
170 new[pkg]["othercomponents"] = f["othercomponents"];
172 for suite in changes["suite"].keys():
173 suite_id = db_access.get_suite_id(suite);
174 for pkg in new.keys():
175 component_id = db_access.get_component_id(new[pkg]["component"]);
176 type_id = db_access.get_override_type_id(new[pkg]["type"]);
177 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
180 for file in new[pkg]["files"]:
181 if files[file].has_key("new"):
182 del files[file]["new"];
185 if changes["suite"].has_key("stable"):
186 print "WARNING: overrides will be added for stable!";
187 for pkg in new.keys():
188 if new[pkg].has_key("othercomponents"):
189 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
193 ################################################################################
195 # Sort by 'have note', 'have source', by ctime, by source name, by
196 # source version number, and finally by filename
198 def changes_compare (a, b):
200 Katie.pkg.changes_file = a;
203 a_changes = copy.copy(Katie.pkg.changes);
208 Katie.pkg.changes_file = b;
211 b_changes = copy.copy(Katie.pkg.changes);
215 # Sort by 'have note';
216 a_has_note = a_changes.get("lisa note");
217 b_has_note = b_changes.get("lisa note");
218 if a_has_note and not b_has_note:
220 elif b_has_note and not a_has_note:
223 # Sort by 'have source'
224 a_has_source = a_changes["architecture"].get("source");
225 b_has_source = b_changes["architecture"].get("source");
226 if a_has_source and not b_has_source:
228 elif b_has_source and not a_has_source:
231 # Sort by ctime-per-source
232 a_source = a_changes.get("source");
233 b_source = b_changes.get("source");
234 if a_source != b_source:
235 a_ctime = os.stat(a)[stat.ST_CTIME];
236 b_ctime = os.stat(b)[stat.ST_CTIME];
237 q = cmp (a_ctime, b_ctime);
241 # Sort by source name
242 q = cmp (a_source, b_source);
246 # Sort by source version
247 a_version = a_changes.get("version");
248 b_version = b_changes.get("version");
249 q = apt_pkg.VersionCompare(a_version, b_version);
253 # Fall back to sort by filename
256 ################################################################################
258 class Section_Completer:
261 q = projectB.query("SELECT section FROM section");
262 for i in q.getresult():
263 self.sections.append(i[0]);
265 def complete(self, text, state):
269 for word in self.sections:
271 self.matches.append(word);
273 return self.matches[state]
277 ############################################################
279 class Priority_Completer:
281 self.priorities = [];
282 q = projectB.query("SELECT priority FROM priority");
283 for i in q.getresult():
284 self.priorities.append(i[0]);
286 def complete(self, text, state):
290 for word in self.priorities:
292 self.matches.append(word);
294 return self.matches[state]
298 ################################################################################
300 def check_valid (new):
301 for pkg in new.keys():
302 section = new[pkg]["section"];
303 priority = new[pkg]["priority"];
304 type = new[pkg]["type"];
305 new[pkg]["section id"] = db_access.get_section_id(section);
306 new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
308 if (section == "debian-installer" and type != "udeb") or \
309 (section != "debian-installer" and type == "udeb"):
310 new[pkg]["section id"] = -1;
311 if (priority == "source" and type != "dsc") or \
312 (priority != "source" and type == "dsc"):
313 new[pkg]["priority id"] = -1;
315 ################################################################################
317 def print_new (new, indexed, file=sys.stdout):
321 for pkg in new.keys():
323 section = new[pkg]["section"];
324 priority = new[pkg]["priority"];
325 if new[pkg]["section id"] == -1:
326 section = section + "[!]";
328 if new[pkg]["priority id"] == -1:
329 priority = priority + "[!]";
332 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
334 line = "%-20s %-20s %-20s" % (pkg, priority, section);
335 line = string.strip(line)+'\n';
337 note = Katie.pkg.changes.get("lisa note");
344 ################################################################################
348 if f.has_key("dbtype"):
350 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
353 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
355 # Validate the override type
356 type_id = db_access.get_override_type_id(type);
358 utils.fubar("invalid type (%s) for new. Say wha?" % (type));
362 ################################################################################
364 def index_range (index):
368 return "1-%s" % (index);
370 ################################################################################
371 ################################################################################
374 # Write the current data to a temporary file
375 temp_filename = tempfile.mktemp();
376 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
378 temp_file = utils.open_file(temp_filename, 'w');
379 print_new (new, 0, temp_file);
381 # Spawn an editor on that file
382 editor = os.environ.get("EDITOR","vi")
383 result = os.system("%s %s" % (editor, temp_filename))
385 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
386 # Read the edited data back in
387 temp_file = utils.open_file(temp_filename);
388 lines = temp_file.readlines();
390 os.unlink(temp_filename);
393 line = string.strip(line[:-1]);
396 s = string.split(line);
397 # Pad the list if necessary
398 s[len(s):3] = [None] * (3-len(s));
399 (pkg, priority, section) = s[:3];
400 if not new.has_key(pkg):
401 utils.warn("Ignoring unknown package '%s'" % (pkg));
403 # Strip off any invalid markers, print_new will readd them.
404 if section[-3:] == "[!]":
405 section = section[:-3];
406 if priority[-3:] == "[!]":
407 priority = priority[:-3];
408 for file in new[pkg]["files"]:
409 Katie.pkg.files[file]["section"] = section;
410 Katie.pkg.files[file]["priority"] = priority;
411 new[pkg]["section"] = section;
412 new[pkg]["priority"] = priority;
414 ################################################################################
416 def edit_index (new, index):
417 priority = new[index]["priority"]
418 section = new[index]["section"]
419 type = new[index]["type"];
422 print string.join([index, priority, section], '\t');
426 prompt = "[B]oth, Priority, Section, Done ? ";
428 prompt = "[S]ection, Done ? ";
429 edit_priority = edit_section = 0;
431 while string.find(prompt, answer) == -1:
432 answer = utils.our_raw_input(prompt);
433 m = katie.re_default_answer.match(prompt)
436 answer = string.upper(answer[:1])
443 edit_priority = edit_section = 1;
449 readline.set_completer(Priorities.complete);
451 while not got_priority:
452 new_priority = string.strip(utils.our_raw_input("New priority: "));
453 if Priorities.priorities.count(new_priority) == 0:
454 print "E: '%s' is not a valid priority, try again." % (new_priority);
457 priority = new_priority;
461 readline.set_completer(Sections.complete);
463 while not got_section:
464 new_section = string.strip(utils.our_raw_input("New section: "));
465 if Sections.sections.count(new_section) == 0:
466 print "E: '%s' is not a valid section, try again." % (new_section);
469 section = new_section;
471 # Reset the readline completer
472 readline.set_completer(None);
474 for file in new[index]["files"]:
475 Katie.pkg.files[file]["section"] = section;
476 Katie.pkg.files[file]["priority"] = priority;
477 new[index]["priority"] = priority;
478 new[index]["section"] = section;
481 ################################################################################
483 def edit_overrides (new):
492 new_index[index] = i;
494 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
497 while not got_answer:
498 answer = utils.our_raw_input(prompt);
499 answer = string.upper(answer[:1]);
500 if answer == "E" or answer == "D":
502 elif katie.re_isanum.match (answer):
503 answer = int(answer);
504 if (answer < 1) or (answer > index):
505 print "%s is not a valid index (%s). Please retry." % (index_range(index), answer);
514 edit_index (new, new_index[answer]);
518 ################################################################################
521 # Write the current data to a temporary file
522 temp_filename = tempfile.mktemp();
523 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
525 temp_file = utils.open_file(temp_filename, 'w');
526 temp_file.write(note);
528 editor = os.environ.get("EDITOR","vi")
531 os.system("%s %s" % (editor, temp_filename))
532 temp_file = utils.open_file(temp_filename);
533 note = string.rstrip(temp_file.read());
536 print utils.prefix_multi_line_string(note," ");
537 prompt = "[D]one, Edit, Abandon, Quit ?"
539 while string.find(prompt, answer) == -1:
540 answer = utils.our_raw_input(prompt);
541 m = katie.re_default_answer.search(prompt);
544 answer = string.upper(answer[:1]);
545 os.unlink(temp_filename);
550 Katie.pkg.changes["lisa note"] = note;
551 Katie.dump_vars(Cnf["Dir::Queue::New"]);
553 ################################################################################
557 less_fd = os.popen("less -", 'w', 0);
558 stdout_fd = sys.stdout;
560 sys.stdout = less_fd;
561 fernanda.display_changes(Katie.pkg.changes_file);
562 files = Katie.pkg.files;
563 for file in files.keys():
564 if files[file].has_key("new"):
565 type = files[file]["type"];
567 fernanda.check_deb(file);
569 fernanda.check_dsc(file);
571 sys.stdout = stdout_fd;
573 if errno.errorcode[e.errno] == 'EPIPE':
574 utils.warn("[fernanda] Caught EPIPE; skipping.");
578 except KeyboardInterrupt:
579 utils.warn("[fernanda] Caught C-c; skipping.");
582 ################################################################################
584 ## FIXME: horribly Debian specific
586 def do_bxa_notification():
587 files = Katie.pkg.files;
589 for file in files.keys():
590 if files[file]["type"] == "deb":
591 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
592 summary = summary + "\n";
593 summary = summary + "Package: %s\n" % (control.Find("Package"));
594 summary = summary + "Description: %s\n" % (control.Find("Description"));
595 Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
596 bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
597 utils.send_mail(bxa_mail,"");
599 ################################################################################
601 def add_overrides (new):
602 changes = Katie.pkg.changes;
603 files = Katie.pkg.files;
605 projectB.query("BEGIN WORK");
606 for suite in changes["suite"].keys():
607 suite_id = db_access.get_suite_id(suite);
608 for pkg in new.keys():
609 component_id = db_access.get_component_id(new[pkg]["component"]);
610 type_id = db_access.get_override_type_id(new[pkg]["type"]);
611 priority_id = new[pkg]["priority id"];
612 section_id = new[pkg]["section id"];
613 projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
614 for file in new[pkg]["files"]:
615 if files[file].has_key("new"):
616 del files[file]["new"];
619 projectB.query("COMMIT WORK");
621 if Cnf.FindB("Dinstall::BXANotify"):
622 do_bxa_notification();
624 ################################################################################
628 files = Katie.pkg.files;
629 changes = Katie.pkg.changes;
631 # Make a copy of distribution we can happily trample on
632 changes["suite"] = copy.copy(changes["distribution"]);
634 # Fix up the list of target suites
635 for suite in changes["suite"].keys():
636 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
638 del changes["suite"][suite];
639 changes["suite"][override] = 1;
641 for suite in changes["suite"].keys():
642 suite_id = db_access.get_suite_id(suite);
644 utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
646 # The main NEW processing loop
649 # Find out what's new
650 new = determine_new(changes, files);
656 if Options["No-Action"] or Options["Automatic"]:
659 (broken, note) = print_new(new, 0);
662 if not broken and not note:
663 prompt = "Add overrides, ";
665 print "W: [!] marked entries must be fixed before package can be processed.";
667 print "W: note must be removed before package can be processed.";
668 prompt = prompt + "Remove note, ";
670 prompt = prompt + "Edit overrides, Check, Manual reject, Note edit, [S]kip, Quit ?";
672 while string.find(prompt, answer) == -1:
673 answer = utils.our_raw_input(prompt);
674 m = katie.re_default_answer.search(prompt);
677 answer = string.upper(answer[:1])
680 done = add_overrides (new);
684 new = edit_overrides (new);
686 aborted = Katie.do_reject(1, Options["Manual-Reject"]);
688 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
691 edit_note(changes.get("lisa note", ""));
693 confirm = string.lower(utils.our_raw_input("Really clear note (y/N)? "));
695 del changes["lisa note"];
701 ################################################################################
702 ################################################################################
703 ################################################################################
705 def usage (exit_code=0):
706 print """Usage: lisa [OPTION]... [CHANGES]...
707 -a, --automatic automatic run
708 -h, --help show this help and exit.
709 -m, --manual-reject=MSG manual reject with `msg'
710 -n, --no-action don't do anything
711 -V, --version display the version number and exit"""
714 ################################################################################
717 global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
719 Cnf = utils.get_conf();
721 Arguments = [('a',"automatic","Lisa::Options::Automatic"),
722 ('h',"help","Lisa::Options::Help"),
723 ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
724 ('n',"no-action","Lisa::Options::No-Action"),
725 ('V',"version","Lisa::Options::Version")];
727 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
728 if not Cnf.has_key("Lisa::Options::%s" % (i)):
729 Cnf["Lisa::Options::%s" % (i)] = "";
731 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
732 Options = Cnf.SubTree("Lisa::Options")
737 if Options["Version"]:
738 print "lisa %s" % (lisa_version);
741 Katie = katie.Katie(Cnf);
743 if not Options["No-Action"]:
744 Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
746 projectB = Katie.projectB;
748 Sections = Section_Completer();
749 Priorities = Priority_Completer();
750 readline.parse_and_bind("tab: complete");
752 return changes_files;
754 ################################################################################
759 files = Katie.pkg.files;
763 for file in files.keys():
764 if files[file]["type"] == "byhand":
765 if os.path.exists(file):
766 print "W: %s still present; please process byhand components and try again." % (file);
772 if Options["No-Action"]:
775 if Options["Automatic"] and not Options["No-Action"]:
777 prompt = "[A]ccept, Manual reject, Skip, Quit ?";
779 prompt = "Manual reject, [S]kip, Quit ?";
781 while string.find(prompt, answer) == -1:
782 answer = utils.our_raw_input(prompt);
783 m = katie.re_default_answer.search(prompt);
786 answer = string.upper(answer[:1]);
793 Katie.do_reject(1, Options["Manual-Reject"]);
794 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
801 ################################################################################
805 if not Options["No-Action"]:
806 (summary, short_summary) = Katie.build_summaries();
807 Katie.accept(summary, short_summary);
808 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
810 def check_status(files):
812 for file in files.keys():
813 if files[file]["type"] == "byhand":
815 elif files[file].has_key("new"):
817 return (new, byhand);
819 def do_pkg(changes_file):
820 Katie.pkg.changes_file = changes_file;
823 Katie.update_subst();
824 files = Katie.pkg.files;
829 (new, byhand) = check_status(files);
835 (new, byhand) = check_status(files);
837 if not new and not byhand:
840 ################################################################################
843 accept_count = Katie.accept_count;
844 accept_bytes = Katie.accept_bytes;
850 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
851 Logger.log(["total",accept_count,accept_bytes]);
853 if not Options["No-Action"]:
856 ################################################################################
859 changes_files = init();
860 if len(changes_files) > 50:
861 sys.stderr.write("Sorting changes...\n");
862 changes_files.sort(changes_compare);
864 # Kill me now? **FIXME**
865 Cnf["Dinstall::Options::No-Mail"] = "";
866 bcc = "X-Katie: %s" % (lisa_version);
867 if Cnf.has_key("Dinstall::Bcc"):
868 Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
870 Katie.Subst["__BCC__"] = bcc;
872 for changes_file in changes_files:
873 changes_file = utils.validate_changes_file_arg(changes_file, 0);
876 print "\n" + changes_file;
877 do_pkg (changes_file);
881 ################################################################################
883 if __name__ == '__main__':