3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002 James Troup <james@nocrew.org>
5 # $Id: lisa,v 1.16 2002-05-23 09:54:23 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # 23:12|<aj> I will not hush!
25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
26 # 23:13|<aj> I shall not be silenced!
27 # 23:13|<aj> The world shall know!
28 # 23:13|<aj> The world *must* know!
29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
30 # 23:13|<aj> yay powerpuff girls!!
31 # 23:13|<aj> buttercup's my favourite, who's yours?
32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
33 # 23:14|<aj> *AREN'T YOU*?!
34 # 23:15|<aj> I will not be treated like this.
35 # 23:15|<aj> I shall have my revenge.
36 # 23:15|<aj> I SHALL!!!
38 ################################################################################
40 import copy, errno, os, readline, string, stat, sys, tempfile;
41 import apt_pkg, apt_inst;
42 import db_access, fernanda, katie, logging, utils;
45 lisa_version = "$Revision: 1.16 $";
58 ################################################################################
59 ################################################################################
60 ################################################################################
62 def reject (str, prefix="Rejected: "):
63 global reject_message;
65 reject_message = reject_message + prefix + str + "\n";
68 global reject_message;
69 files = Katie.pkg.files;
72 for file in files.keys():
73 # Check that the source still exists
74 if files[file]["type"] == "deb":
75 source_version = files[file]["source version"];
76 source_package = files[file]["source package"];
77 if not Katie.pkg.changes["architecture"].has_key("source") \
78 and not Katie.source_exists(source_package, source_version):
79 source_epochless_version = utils.re_no_epoch.sub('', source_version);
80 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
81 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
82 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
84 # Version and file overwrite checks
85 if files[file]["type"] == "deb":
86 reject(Katie.check_binary_against_db(file));
87 elif files[file]["type"] == "dsc":
88 reject(Katie.check_source_against_db(file));
89 (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
94 if Options["No-Action"] or Options["Automatic"]:
97 print "REJECT\n" + reject_message,;
98 prompt = "[R]eject, Skip, Quit ?";
100 while string.find(prompt, answer) == -1:
101 answer = utils.our_raw_input(prompt);
102 m = katie.re_default_answer.match(prompt);
105 answer = string.upper(answer[:1]);
108 Katie.do_reject(0, reject_message);
116 ################################################################################
118 def determine_new (changes, files):
121 # Build up a list of potentially new things
122 for file in files.keys():
124 # Skip byhand elements
125 if f["type"] == "byhand":
128 priority = f["priority"];
129 section = f["section"];
131 if section == "non-US/main":
134 component = f["component"];
138 if not new.has_key(pkg):
140 new[pkg]["priority"] = priority;
141 new[pkg]["section"] = section;
142 new[pkg]["type"] = type;
143 new[pkg]["component"] = component;
144 new[pkg]["files"] = [];
146 old_type = new[pkg]["type"];
148 # source gets trumped by deb or udeb
149 if old_type == "dsc":
150 new[pkg]["priority"] = priority;
151 new[pkg]["section"] = section;
152 new[pkg]["type"] = type;
153 new[pkg]["component"] = component;
154 new[pkg]["files"].append(file);
155 if f.has_key("othercomponents"):
156 new[pkg]["othercomponents"] = f["othercomponents"];
158 for suite in changes["suite"].keys():
159 suite_id = db_access.get_suite_id(suite);
160 for pkg in new.keys():
161 component_id = db_access.get_component_id(new[pkg]["component"]);
162 type_id = db_access.get_override_type_id(new[pkg]["type"]);
163 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
166 for file in new[pkg]["files"]:
167 if files[file].has_key("new"):
168 del files[file]["new"];
171 if changes["suite"].has_key("stable"):
172 print "WARNING: overrides will be added for stable!";
173 for pkg in new.keys():
174 if new[pkg].has_key("othercomponents"):
175 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
179 ################################################################################
181 def indiv_sg_compare (a, b):
182 """Sort by source name, source, version, 'have source', and
183 finally by filename."""
184 # Sort by source version
185 q = apt_pkg.VersionCompare(a["version"], b["version"]);
189 # Sort by 'have source'
190 a_has_source = a["architecture"].get("source");
191 b_has_source = b["architecture"].get("source");
192 if a_has_source and not b_has_source:
194 elif b_has_source and not a_has_source:
197 return cmp(a["filename"], b["filename"]);
199 ############################################################
201 def sg_compare (a, b):
204 """Sort by have note, time of oldest upload."""
206 a_note_state = a["note_state"];
207 b_note_state = b["note_state"];
208 if a_note_state != b_note_state:
211 # Sort by time of oldest upload
212 return cmp(a["oldest"], b["oldest"]);
214 def sort_changes(changes_files):
215 """Sort into source groups, then sort each source group by version,
216 have source, filename. Finally, sort the source groups by have
217 note, time of oldest upload of each source upload."""
218 if len(changes_files) == 1:
219 return changes_files;
223 # Read in all the .changes files
224 for filename in changes_files:
226 Katie.pkg.changes_file = filename;
229 cache[filename] = copy.copy(Katie.pkg.changes);
230 cache[filename]["filename"] = filename;
232 sorted_list.append(filename);
234 # Divide the .changes into per-source groups
236 for filename in cache.keys():
237 source = cache[filename]["source"];
238 if not per_source.has_key(source):
239 per_source[source] = {};
240 per_source[source]["list"] = [];
241 per_source[source]["list"].append(cache[filename]);
242 # Determine oldest time and have note status for each source group
243 for source in per_source.keys():
244 source_list = per_source[source]["list"];
245 first = source_list[0];
246 oldest = os.stat(first["filename"])[stat.ST_CTIME];
248 for d in per_source[source]["list"]:
249 ctime = os.stat(d["filename"])[stat.ST_CTIME];
252 have_note = have_note + (d.has_key("lisa note"));
253 per_source[source]["oldest"] = oldest;
255 per_source[source]["note_state"] = 0; # none
256 elif have_note < len(source_list):
257 per_source[source]["note_state"] = 1; # some
259 per_source[source]["note_state"] = 2; # all
260 per_source[source]["list"].sort(indiv_sg_compare);
261 per_source_items = per_source.items();
262 per_source_items.sort(sg_compare);
263 for i in per_source_items:
264 for j in i[1]["list"]:
265 sorted_list.append(j["filename"]);
268 ################################################################################
270 class Section_Completer:
273 q = projectB.query("SELECT section FROM section");
274 for i in q.getresult():
275 self.sections.append(i[0]);
277 def complete(self, text, state):
281 for word in self.sections:
283 self.matches.append(word);
285 return self.matches[state]
289 ############################################################
291 class Priority_Completer:
293 self.priorities = [];
294 q = projectB.query("SELECT priority FROM priority");
295 for i in q.getresult():
296 self.priorities.append(i[0]);
298 def complete(self, text, state):
302 for word in self.priorities:
304 self.matches.append(word);
306 return self.matches[state]
310 ################################################################################
312 def check_valid (new):
313 for pkg in new.keys():
314 section = new[pkg]["section"];
315 priority = new[pkg]["priority"];
316 type = new[pkg]["type"];
317 new[pkg]["section id"] = db_access.get_section_id(section);
318 new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
320 if (section == "debian-installer" and type != "udeb") or \
321 (section != "debian-installer" and type == "udeb"):
322 new[pkg]["section id"] = -1;
323 if (priority == "source" and type != "dsc") or \
324 (priority != "source" and type == "dsc"):
325 new[pkg]["priority id"] = -1;
327 ################################################################################
329 def print_new (new, indexed, file=sys.stdout):
333 for pkg in new.keys():
335 section = new[pkg]["section"];
336 priority = new[pkg]["priority"];
337 if new[pkg]["section id"] == -1:
338 section = section + "[!]";
340 if new[pkg]["priority id"] == -1:
341 priority = priority + "[!]";
344 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
346 line = "%-20s %-20s %-20s" % (pkg, priority, section);
347 line = string.strip(line)+'\n';
349 note = Katie.pkg.changes.get("lisa note");
356 ################################################################################
360 if f.has_key("dbtype"):
362 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
365 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
367 # Validate the override type
368 type_id = db_access.get_override_type_id(type);
370 utils.fubar("invalid type (%s) for new. Say wha?" % (type));
374 ################################################################################
376 def index_range (index):
380 return "1-%s" % (index);
382 ################################################################################
383 ################################################################################
386 # Write the current data to a temporary file
387 temp_filename = tempfile.mktemp();
388 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
390 temp_file = utils.open_file(temp_filename, 'w');
391 print_new (new, 0, temp_file);
393 # Spawn an editor on that file
394 editor = os.environ.get("EDITOR","vi")
395 result = os.system("%s %s" % (editor, temp_filename))
397 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
398 # Read the edited data back in
399 temp_file = utils.open_file(temp_filename);
400 lines = temp_file.readlines();
402 os.unlink(temp_filename);
405 line = string.strip(line[:-1]);
408 s = string.split(line);
409 # Pad the list if necessary
410 s[len(s):3] = [None] * (3-len(s));
411 (pkg, priority, section) = s[:3];
412 if not new.has_key(pkg):
413 utils.warn("Ignoring unknown package '%s'" % (pkg));
415 # Strip off any invalid markers, print_new will readd them.
416 if section[-3:] == "[!]":
417 section = section[:-3];
418 if priority[-3:] == "[!]":
419 priority = priority[:-3];
420 for file in new[pkg]["files"]:
421 Katie.pkg.files[file]["section"] = section;
422 Katie.pkg.files[file]["priority"] = priority;
423 new[pkg]["section"] = section;
424 new[pkg]["priority"] = priority;
426 ################################################################################
428 def edit_index (new, index):
429 priority = new[index]["priority"]
430 section = new[index]["section"]
431 type = new[index]["type"];
434 print string.join([index, priority, section], '\t');
438 prompt = "[B]oth, Priority, Section, Done ? ";
440 prompt = "[S]ection, Done ? ";
441 edit_priority = edit_section = 0;
443 while string.find(prompt, answer) == -1:
444 answer = utils.our_raw_input(prompt);
445 m = katie.re_default_answer.match(prompt)
448 answer = string.upper(answer[:1])
455 edit_priority = edit_section = 1;
461 readline.set_completer(Priorities.complete);
463 while not got_priority:
464 new_priority = string.strip(utils.our_raw_input("New priority: "));
465 if Priorities.priorities.count(new_priority) == 0:
466 print "E: '%s' is not a valid priority, try again." % (new_priority);
469 priority = new_priority;
473 readline.set_completer(Sections.complete);
475 while not got_section:
476 new_section = string.strip(utils.our_raw_input("New section: "));
477 if Sections.sections.count(new_section) == 0:
478 print "E: '%s' is not a valid section, try again." % (new_section);
481 section = new_section;
483 # Reset the readline completer
484 readline.set_completer(None);
486 for file in new[index]["files"]:
487 Katie.pkg.files[file]["section"] = section;
488 Katie.pkg.files[file]["priority"] = priority;
489 new[index]["priority"] = priority;
490 new[index]["section"] = section;
493 ################################################################################
495 def edit_overrides (new):
504 new_index[index] = i;
506 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
509 while not got_answer:
510 answer = utils.our_raw_input(prompt);
511 answer = string.upper(answer[:1]);
512 if answer == "E" or answer == "D":
514 elif katie.re_isanum.match (answer):
515 answer = int(answer);
516 if (answer < 1) or (answer > index):
517 print "%s is not a valid index (%s). Please retry." % (index_range(index), answer);
526 edit_index (new, new_index[answer]);
530 ################################################################################
533 # Write the current data to a temporary file
534 temp_filename = tempfile.mktemp();
535 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
537 temp_file = utils.open_file(temp_filename, 'w');
538 temp_file.write(note);
540 editor = os.environ.get("EDITOR","vi")
543 os.system("%s %s" % (editor, temp_filename))
544 temp_file = utils.open_file(temp_filename);
545 note = string.rstrip(temp_file.read());
548 print utils.prefix_multi_line_string(note," ");
549 prompt = "[D]one, Edit, Abandon, Quit ?"
551 while string.find(prompt, answer) == -1:
552 answer = utils.our_raw_input(prompt);
553 m = katie.re_default_answer.search(prompt);
556 answer = string.upper(answer[:1]);
557 os.unlink(temp_filename);
562 Katie.pkg.changes["lisa note"] = note;
563 Katie.dump_vars(Cnf["Dir::Queue::New"]);
565 ################################################################################
569 less_fd = os.popen("less -", 'w', 0);
570 stdout_fd = sys.stdout;
572 sys.stdout = less_fd;
573 fernanda.display_changes(Katie.pkg.changes_file);
574 files = Katie.pkg.files;
575 for file in files.keys():
576 if files[file].has_key("new"):
577 type = files[file]["type"];
579 fernanda.check_deb(file);
581 fernanda.check_dsc(file);
583 sys.stdout = stdout_fd;
585 if errno.errorcode[e.errno] == 'EPIPE':
586 utils.warn("[fernanda] Caught EPIPE; skipping.");
590 except KeyboardInterrupt:
591 utils.warn("[fernanda] Caught C-c; skipping.");
594 ################################################################################
596 ## FIXME: horribly Debian specific
598 def do_bxa_notification():
599 files = Katie.pkg.files;
601 for file in files.keys():
602 if files[file]["type"] == "deb":
603 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
604 summary = summary + "\n";
605 summary = summary + "Package: %s\n" % (control.Find("Package"));
606 summary = summary + "Description: %s\n" % (control.Find("Description"));
607 Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
608 bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
609 utils.send_mail(bxa_mail,"");
611 ################################################################################
613 def add_overrides (new):
614 changes = Katie.pkg.changes;
615 files = Katie.pkg.files;
617 projectB.query("BEGIN WORK");
618 for suite in changes["suite"].keys():
619 suite_id = db_access.get_suite_id(suite);
620 for pkg in new.keys():
621 component_id = db_access.get_component_id(new[pkg]["component"]);
622 type_id = db_access.get_override_type_id(new[pkg]["type"]);
623 priority_id = new[pkg]["priority id"];
624 section_id = new[pkg]["section id"];
625 projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
626 for file in new[pkg]["files"]:
627 if files[file].has_key("new"):
628 del files[file]["new"];
631 projectB.query("COMMIT WORK");
633 if Cnf.FindB("Dinstall::BXANotify"):
634 do_bxa_notification();
636 ################################################################################
640 files = Katie.pkg.files;
641 changes = Katie.pkg.changes;
643 # Make a copy of distribution we can happily trample on
644 changes["suite"] = copy.copy(changes["distribution"]);
646 # Fix up the list of target suites
647 for suite in changes["suite"].keys():
648 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
650 del changes["suite"][suite];
651 changes["suite"][override] = 1;
653 for suite in changes["suite"].keys():
654 suite_id = db_access.get_suite_id(suite);
656 utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
658 # The main NEW processing loop
661 # Find out what's new
662 new = determine_new(changes, files);
668 if Options["No-Action"] or Options["Automatic"]:
671 (broken, note) = print_new(new, 0);
674 if not broken and not note:
675 prompt = "Add overrides, ";
677 print "W: [!] marked entries must be fixed before package can be processed.";
679 print "W: note must be removed before package can be processed.";
680 prompt = prompt + "Remove note, ";
682 prompt = prompt + "Edit overrides, Check, Manual reject, Note edit, [S]kip, Quit ?";
684 while string.find(prompt, answer) == -1:
685 answer = utils.our_raw_input(prompt);
686 m = katie.re_default_answer.search(prompt);
689 answer = string.upper(answer[:1])
692 done = add_overrides (new);
696 new = edit_overrides (new);
698 aborted = Katie.do_reject(1, Options["Manual-Reject"]);
700 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
703 edit_note(changes.get("lisa note", ""));
705 confirm = string.lower(utils.our_raw_input("Really clear note (y/N)? "));
707 del changes["lisa note"];
713 ################################################################################
714 ################################################################################
715 ################################################################################
717 def usage (exit_code=0):
718 print """Usage: lisa [OPTION]... [CHANGES]...
719 -a, --automatic automatic run
720 -h, --help show this help and exit.
721 -m, --manual-reject=MSG manual reject with `msg'
722 -n, --no-action don't do anything
723 -V, --version display the version number and exit"""
726 ################################################################################
729 global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
731 Cnf = utils.get_conf();
733 Arguments = [('a',"automatic","Lisa::Options::Automatic"),
734 ('h',"help","Lisa::Options::Help"),
735 ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
736 ('n',"no-action","Lisa::Options::No-Action"),
737 ('V',"version","Lisa::Options::Version")];
739 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
740 if not Cnf.has_key("Lisa::Options::%s" % (i)):
741 Cnf["Lisa::Options::%s" % (i)] = "";
743 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
744 Options = Cnf.SubTree("Lisa::Options")
749 if Options["Version"]:
750 print "lisa %s" % (lisa_version);
753 Katie = katie.Katie(Cnf);
755 if not Options["No-Action"]:
756 Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
758 projectB = Katie.projectB;
760 Sections = Section_Completer();
761 Priorities = Priority_Completer();
762 readline.parse_and_bind("tab: complete");
764 return changes_files;
766 ################################################################################
771 files = Katie.pkg.files;
775 for file in files.keys():
776 if files[file]["type"] == "byhand":
777 if os.path.exists(file):
778 print "W: %s still present; please process byhand components and try again." % (file);
784 if Options["No-Action"]:
787 if Options["Automatic"] and not Options["No-Action"]:
789 prompt = "[A]ccept, Manual reject, Skip, Quit ?";
791 prompt = "Manual reject, [S]kip, Quit ?";
793 while string.find(prompt, answer) == -1:
794 answer = utils.our_raw_input(prompt);
795 m = katie.re_default_answer.search(prompt);
798 answer = string.upper(answer[:1]);
805 Katie.do_reject(1, Options["Manual-Reject"]);
806 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
813 ################################################################################
817 if not Options["No-Action"]:
818 (summary, short_summary) = Katie.build_summaries();
819 Katie.accept(summary, short_summary);
820 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
822 def check_status(files):
824 for file in files.keys():
825 if files[file]["type"] == "byhand":
827 elif files[file].has_key("new"):
829 return (new, byhand);
831 def do_pkg(changes_file):
832 Katie.pkg.changes_file = changes_file;
835 Katie.update_subst();
836 files = Katie.pkg.files;
841 (new, byhand) = check_status(files);
847 (new, byhand) = check_status(files);
849 if not new and not byhand:
852 ################################################################################
855 accept_count = Katie.accept_count;
856 accept_bytes = Katie.accept_bytes;
862 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
863 Logger.log(["total",accept_count,accept_bytes]);
865 if not Options["No-Action"]:
868 ################################################################################
871 changes_files = init();
872 if len(changes_files) > 50:
873 sys.stderr.write("Sorting changes...\n");
874 changes_files = sort_changes(changes_files);
876 # Kill me now? **FIXME**
877 Cnf["Dinstall::Options::No-Mail"] = "";
878 bcc = "X-Katie: %s" % (lisa_version);
879 if Cnf.has_key("Dinstall::Bcc"):
880 Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
882 Katie.Subst["__BCC__"] = bcc;
884 for changes_file in changes_files:
885 changes_file = utils.validate_changes_file_arg(changes_file, 0);
888 print "\n" + changes_file;
889 do_pkg (changes_file);
893 ################################################################################
895 if __name__ == '__main__':