3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003 James Troup <james@nocrew.org>
5 # $Id: lisa,v 1.25 2003-08-09 09:59:14 rdonald Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # 23:12|<aj> I will not hush!
25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
26 # 23:13|<aj> I shall not be silenced!
27 # 23:13|<aj> The world shall know!
28 # 23:13|<aj> The world *must* know!
29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
30 # 23:13|<aj> yay powerpuff girls!!
31 # 23:13|<aj> buttercup's my favourite, who's yours?
32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
33 # 23:14|<aj> *AREN'T YOU*?!
34 # 23:15|<aj> I will not be treated like this.
35 # 23:15|<aj> I shall have my revenge.
36 # 23:15|<aj> I SHALL!!!
38 ################################################################################
40 import copy, errno, os, readline, stat, sys, tempfile;
41 import apt_pkg, apt_inst;
42 import db_access, fernanda, katie, logging, utils;
45 lisa_version = "$Revision: 1.25 $";
58 ################################################################################
59 ################################################################################
60 ################################################################################
62 def reject (str, prefix="Rejected: "):
63 global reject_message;
65 reject_message += prefix + str + "\n";
68 global reject_message;
69 files = Katie.pkg.files;
72 for file in files.keys():
73 # Check that the source still exists
74 if files[file]["type"] == "deb":
75 source_version = files[file]["source version"];
76 source_package = files[file]["source package"];
77 if not Katie.pkg.changes["architecture"].has_key("source") \
78 and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()):
79 source_epochless_version = utils.re_no_epoch.sub('', source_version);
80 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
81 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
82 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
84 # Version and file overwrite checks
85 if files[file]["type"] == "deb":
86 reject(Katie.check_binary_against_db(file));
87 elif files[file]["type"] == "dsc":
88 reject(Katie.check_source_against_db(file));
89 (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
94 if Options["No-Action"] or Options["Automatic"]:
97 print "REJECT\n" + reject_message,;
98 prompt = "[R]eject, Skip, Quit ?";
100 while prompt.find(answer) == -1:
101 answer = utils.our_raw_input(prompt);
102 m = katie.re_default_answer.match(prompt);
105 answer = answer[:1].upper();
108 Katie.do_reject(0, reject_message);
109 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
118 ################################################################################
120 def determine_new (changes, files):
123 # Build up a list of potentially new things
124 for file in files.keys():
126 # Skip byhand elements
127 if f["type"] == "byhand":
130 priority = f["priority"];
131 section = f["section"];
133 if section == "non-US/main":
136 component = f["component"];
140 if not new.has_key(pkg):
142 new[pkg]["priority"] = priority;
143 new[pkg]["section"] = section;
144 new[pkg]["type"] = type;
145 new[pkg]["component"] = component;
146 new[pkg]["files"] = [];
148 old_type = new[pkg]["type"];
150 # source gets trumped by deb or udeb
151 if old_type == "dsc":
152 new[pkg]["priority"] = priority;
153 new[pkg]["section"] = section;
154 new[pkg]["type"] = type;
155 new[pkg]["component"] = component;
156 new[pkg]["files"].append(file);
157 if f.has_key("othercomponents"):
158 new[pkg]["othercomponents"] = f["othercomponents"];
160 for suite in changes["suite"].keys():
161 suite_id = db_access.get_suite_id(suite);
162 for pkg in new.keys():
163 component_id = db_access.get_component_id(new[pkg]["component"]);
164 type_id = db_access.get_override_type_id(new[pkg]["type"]);
165 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
168 for file in new[pkg]["files"]:
169 if files[file].has_key("new"):
170 del files[file]["new"];
173 if changes["suite"].has_key("stable"):
174 print "WARNING: overrides will be added for stable!";
175 if changes["suite"].has_key("oldstable"):
176 print "WARNING: overrides will be added for OLDstable!";
177 for pkg in new.keys():
178 if new[pkg].has_key("othercomponents"):
179 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
183 ################################################################################
185 def indiv_sg_compare (a, b):
186 """Sort by source name, source, version, 'have source', and
187 finally by filename."""
188 # Sort by source version
189 q = apt_pkg.VersionCompare(a["version"], b["version"]);
193 # Sort by 'have source'
194 a_has_source = a["architecture"].get("source");
195 b_has_source = b["architecture"].get("source");
196 if a_has_source and not b_has_source:
198 elif b_has_source and not a_has_source:
201 return cmp(a["filename"], b["filename"]);
203 ############################################################
205 def sg_compare (a, b):
208 """Sort by have note, time of oldest upload."""
210 a_note_state = a["note_state"];
211 b_note_state = b["note_state"];
212 if a_note_state < b_note_state:
214 elif a_note_state > b_note_state:
217 # Sort by time of oldest upload
218 return cmp(a["oldest"], b["oldest"]);
220 def sort_changes(changes_files):
221 """Sort into source groups, then sort each source group by version,
222 have source, filename. Finally, sort the source groups by have
223 note, time of oldest upload of each source upload."""
224 if len(changes_files) == 1:
225 return changes_files;
229 # Read in all the .changes files
230 for filename in changes_files:
232 Katie.pkg.changes_file = filename;
235 cache[filename] = copy.copy(Katie.pkg.changes);
236 cache[filename]["filename"] = filename;
238 sorted_list.append(filename);
240 # Divide the .changes into per-source groups
242 for filename in cache.keys():
243 source = cache[filename]["source"];
244 if not per_source.has_key(source):
245 per_source[source] = {};
246 per_source[source]["list"] = [];
247 per_source[source]["list"].append(cache[filename]);
248 # Determine oldest time and have note status for each source group
249 for source in per_source.keys():
250 source_list = per_source[source]["list"];
251 first = source_list[0];
252 oldest = os.stat(first["filename"])[stat.ST_CTIME];
254 for d in per_source[source]["list"]:
255 ctime = os.stat(d["filename"])[stat.ST_CTIME];
258 have_note += (d.has_key("lisa note"));
259 per_source[source]["oldest"] = oldest;
261 per_source[source]["note_state"] = 0; # none
262 elif have_note < len(source_list):
263 per_source[source]["note_state"] = 1; # some
265 per_source[source]["note_state"] = 2; # all
266 per_source[source]["list"].sort(indiv_sg_compare);
267 per_source_items = per_source.items();
268 per_source_items.sort(sg_compare);
269 for i in per_source_items:
270 for j in i[1]["list"]:
271 sorted_list.append(j["filename"]);
274 ################################################################################
276 class Section_Completer:
279 q = projectB.query("SELECT section FROM section");
280 for i in q.getresult():
281 self.sections.append(i[0]);
283 def complete(self, text, state):
287 for word in self.sections:
289 self.matches.append(word);
291 return self.matches[state]
295 ############################################################
297 class Priority_Completer:
299 self.priorities = [];
300 q = projectB.query("SELECT priority FROM priority");
301 for i in q.getresult():
302 self.priorities.append(i[0]);
304 def complete(self, text, state):
308 for word in self.priorities:
310 self.matches.append(word);
312 return self.matches[state]
316 ################################################################################
318 def check_valid (new):
319 for pkg in new.keys():
320 section = new[pkg]["section"];
321 priority = new[pkg]["priority"];
322 type = new[pkg]["type"];
323 new[pkg]["section id"] = db_access.get_section_id(section);
324 new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
326 if (section == "debian-installer" and type != "udeb") or \
327 (section != "debian-installer" and type == "udeb"):
328 new[pkg]["section id"] = -1;
329 if (priority == "source" and type != "dsc") or \
330 (priority != "source" and type == "dsc"):
331 new[pkg]["priority id"] = -1;
333 ################################################################################
335 def print_new (new, indexed, file=sys.stdout):
339 for pkg in new.keys():
341 section = new[pkg]["section"];
342 priority = new[pkg]["priority"];
343 if new[pkg]["section id"] == -1:
346 if new[pkg]["priority id"] == -1:
350 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
352 line = "%-20s %-20s %-20s" % (pkg, priority, section);
353 line = line.strip()+'\n';
355 note = Katie.pkg.changes.get("lisa note");
362 ################################################################################
366 if f.has_key("dbtype"):
368 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
371 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
373 # Validate the override type
374 type_id = db_access.get_override_type_id(type);
376 utils.fubar("invalid type (%s) for new. Say wha?" % (type));
380 ################################################################################
382 def index_range (index):
386 return "1-%s" % (index);
388 ################################################################################
389 ################################################################################
392 # Write the current data to a temporary file
393 temp_filename = tempfile.mktemp();
394 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
396 temp_file = utils.open_file(temp_filename, 'w');
397 print_new (new, 0, temp_file);
399 # Spawn an editor on that file
400 editor = os.environ.get("EDITOR","vi")
401 result = os.system("%s %s" % (editor, temp_filename))
403 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
404 # Read the edited data back in
405 temp_file = utils.open_file(temp_filename);
406 lines = temp_file.readlines();
408 os.unlink(temp_filename);
415 # Pad the list if necessary
416 s[len(s):3] = [None] * (3-len(s));
417 (pkg, priority, section) = s[:3];
418 if not new.has_key(pkg):
419 utils.warn("Ignoring unknown package '%s'" % (pkg));
421 # Strip off any invalid markers, print_new will readd them.
422 if section.endswith("[!]"):
423 section = section[:-3];
424 if priority.endswith("[!]"):
425 priority = priority[:-3];
426 for file in new[pkg]["files"]:
427 Katie.pkg.files[file]["section"] = section;
428 Katie.pkg.files[file]["priority"] = priority;
429 new[pkg]["section"] = section;
430 new[pkg]["priority"] = priority;
432 ################################################################################
434 def edit_index (new, index):
435 priority = new[index]["priority"]
436 section = new[index]["section"]
437 type = new[index]["type"];
440 print "\t".join([index, priority, section]);
444 prompt = "[B]oth, Priority, Section, Done ? ";
446 prompt = "[S]ection, Done ? ";
447 edit_priority = edit_section = 0;
449 while prompt.find(answer) == -1:
450 answer = utils.our_raw_input(prompt);
451 m = katie.re_default_answer.match(prompt)
454 answer = answer[:1].upper()
461 edit_priority = edit_section = 1;
467 readline.set_completer(Priorities.complete);
469 while not got_priority:
470 new_priority = utils.our_raw_input("New priority: ").strip();
471 if Priorities.priorities.count(new_priority) == 0:
472 print "E: '%s' is not a valid priority, try again." % (new_priority);
475 priority = new_priority;
479 readline.set_completer(Sections.complete);
481 while not got_section:
482 new_section = utils.our_raw_input("New section: ").strip();
483 if Sections.sections.count(new_section) == 0:
484 print "E: '%s' is not a valid section, try again." % (new_section);
487 section = new_section;
489 # Reset the readline completer
490 readline.set_completer(None);
492 for file in new[index]["files"]:
493 Katie.pkg.files[file]["section"] = section;
494 Katie.pkg.files[file]["priority"] = priority;
495 new[index]["priority"] = priority;
496 new[index]["section"] = section;
499 ################################################################################
501 def edit_overrides (new):
510 new_index[index] = i;
512 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
515 while not got_answer:
516 answer = utils.our_raw_input(prompt);
517 if not utils.str_isnum(answer):
518 answer = answer[:1].upper();
519 if answer == "E" or answer == "D":
521 elif katie.re_isanum.match (answer):
522 answer = int(answer);
523 if (answer < 1) or (answer > index):
524 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index));
533 edit_index (new, new_index[answer]);
537 ################################################################################
540 # Write the current data to a temporary file
541 temp_filename = tempfile.mktemp();
542 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
544 temp_file = utils.open_file(temp_filename, 'w');
545 temp_file.write(note);
547 editor = os.environ.get("EDITOR","vi")
550 os.system("%s %s" % (editor, temp_filename))
551 temp_file = utils.open_file(temp_filename);
552 note = temp_file.read().rstrip();
555 print utils.prefix_multi_line_string(note," ");
556 prompt = "[D]one, Edit, Abandon, Quit ?"
558 while prompt.find(answer) == -1:
559 answer = utils.our_raw_input(prompt);
560 m = katie.re_default_answer.search(prompt);
563 answer = answer[:1].upper();
564 os.unlink(temp_filename);
569 Katie.pkg.changes["lisa note"] = note;
570 Katie.dump_vars(Cnf["Dir::Queue::New"]);
572 ################################################################################
576 less_fd = os.popen("less -R -", 'w', 0);
577 stdout_fd = sys.stdout;
579 sys.stdout = less_fd;
580 fernanda.display_changes(Katie.pkg.changes_file);
581 files = Katie.pkg.files;
582 for file in files.keys():
583 if files[file].has_key("new"):
584 type = files[file]["type"];
586 fernanda.check_deb(file);
588 fernanda.check_dsc(file);
590 sys.stdout = stdout_fd;
592 if errno.errorcode[e.errno] == 'EPIPE':
593 utils.warn("[fernanda] Caught EPIPE; skipping.");
597 except KeyboardInterrupt:
598 utils.warn("[fernanda] Caught C-c; skipping.");
601 ################################################################################
603 ## FIXME: horribly Debian specific
605 def do_bxa_notification():
606 files = Katie.pkg.files;
608 for file in files.keys():
609 if files[file]["type"] == "deb":
610 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
612 summary += "Package: %s\n" % (control.Find("Package"));
613 summary += "Description: %s\n" % (control.Find("Description"));
614 Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
615 bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
616 utils.send_mail(bxa_mail);
618 ################################################################################
620 def add_overrides (new):
621 changes = Katie.pkg.changes;
622 files = Katie.pkg.files;
624 projectB.query("BEGIN WORK");
625 for suite in changes["suite"].keys():
626 suite_id = db_access.get_suite_id(suite);
627 for pkg in new.keys():
628 component_id = db_access.get_component_id(new[pkg]["component"]);
629 type_id = db_access.get_override_type_id(new[pkg]["type"]);
630 priority_id = new[pkg]["priority id"];
631 section_id = new[pkg]["section id"];
632 projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
633 for file in new[pkg]["files"]:
634 if files[file].has_key("new"):
635 del files[file]["new"];
638 projectB.query("COMMIT WORK");
640 if Cnf.FindB("Dinstall::BXANotify"):
641 do_bxa_notification();
643 ################################################################################
647 files = Katie.pkg.files;
648 changes = Katie.pkg.changes;
650 # Make a copy of distribution we can happily trample on
651 changes["suite"] = copy.copy(changes["distribution"]);
653 # Fix up the list of target suites
654 for suite in changes["suite"].keys():
655 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
657 del changes["suite"][suite];
658 changes["suite"][override] = 1;
660 for suite in changes["suite"].keys():
661 suite_id = db_access.get_suite_id(suite);
663 utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
665 # The main NEW processing loop
668 # Find out what's new
669 new = determine_new(changes, files);
675 if Options["No-Action"] or Options["Automatic"]:
678 (broken, note) = print_new(new, 0);
681 if not broken and not note:
682 prompt = "Add overrides, ";
684 print "W: [!] marked entries must be fixed before package can be processed.";
686 print "W: note must be removed before package can be processed.";
687 prompt += "Remove note, ";
689 prompt += "Edit overrides, Check, Manual reject, Note edit, [S]kip, Quit ?";
691 while prompt.find(answer) == -1:
692 answer = utils.our_raw_input(prompt);
693 m = katie.re_default_answer.search(prompt);
696 answer = answer[:1].upper()
699 done = add_overrides (new);
703 new = edit_overrides (new);
705 aborted = Katie.do_reject(1, Options["Manual-Reject"]);
707 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
710 edit_note(changes.get("lisa note", ""));
712 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower();
714 del changes["lisa note"];
720 ################################################################################
721 ################################################################################
722 ################################################################################
724 def usage (exit_code=0):
725 print """Usage: lisa [OPTION]... [CHANGES]...
726 -a, --automatic automatic run
727 -h, --help show this help and exit.
728 -m, --manual-reject=MSG manual reject with `msg'
729 -n, --no-action don't do anything
730 -V, --version display the version number and exit"""
733 ################################################################################
736 global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
738 Cnf = utils.get_conf();
740 Arguments = [('a',"automatic","Lisa::Options::Automatic"),
741 ('h',"help","Lisa::Options::Help"),
742 ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
743 ('n',"no-action","Lisa::Options::No-Action"),
744 ('V',"version","Lisa::Options::Version")];
746 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
747 if not Cnf.has_key("Lisa::Options::%s" % (i)):
748 Cnf["Lisa::Options::%s" % (i)] = "";
750 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
751 Options = Cnf.SubTree("Lisa::Options")
756 if Options["Version"]:
757 print "lisa %s" % (lisa_version);
760 Katie = katie.Katie(Cnf);
762 if not Options["No-Action"]:
763 Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
765 projectB = Katie.projectB;
767 Sections = Section_Completer();
768 Priorities = Priority_Completer();
769 readline.parse_and_bind("tab: complete");
771 return changes_files;
773 ################################################################################
778 files = Katie.pkg.files;
782 for file in files.keys():
783 if files[file]["type"] == "byhand":
784 if os.path.exists(file):
785 print "W: %s still present; please process byhand components and try again." % (file);
791 if Options["No-Action"]:
794 if Options["Automatic"] and not Options["No-Action"]:
796 prompt = "[A]ccept, Manual reject, Skip, Quit ?";
798 prompt = "Manual reject, [S]kip, Quit ?";
800 while prompt.find(answer) == -1:
801 answer = utils.our_raw_input(prompt);
802 m = katie.re_default_answer.search(prompt);
805 answer = answer[:1].upper();
812 Katie.do_reject(1, Options["Manual-Reject"]);
813 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
820 ################################################################################
824 if not Options["No-Action"]:
825 (summary, short_summary) = Katie.build_summaries();
826 Katie.accept(summary, short_summary);
827 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
829 def check_status(files):
831 for file in files.keys():
832 if files[file]["type"] == "byhand":
834 elif files[file].has_key("new"):
836 return (new, byhand);
838 def do_pkg(changes_file):
839 Katie.pkg.changes_file = changes_file;
842 Katie.update_subst();
843 files = Katie.pkg.files;
848 (new, byhand) = check_status(files);
854 (new, byhand) = check_status(files);
856 if not new and not byhand:
859 ################################################################################
862 accept_count = Katie.accept_count;
863 accept_bytes = Katie.accept_bytes;
869 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
870 Logger.log(["total",accept_count,accept_bytes]);
872 if not Options["No-Action"]:
875 ################################################################################
878 changes_files = init();
879 if len(changes_files) > 50:
880 sys.stderr.write("Sorting changes...\n");
881 changes_files = sort_changes(changes_files);
883 # Kill me now? **FIXME**
884 Cnf["Dinstall::Options::No-Mail"] = "";
885 bcc = "X-Katie: lisa %s" % (lisa_version);
886 if Cnf.has_key("Dinstall::Bcc"):
887 Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
889 Katie.Subst["__BCC__"] = bcc;
891 for changes_file in changes_files:
892 changes_file = utils.validate_changes_file_arg(changes_file, 0);
895 print "\n" + changes_file;
896 do_pkg (changes_file);
900 ################################################################################
902 if __name__ == '__main__':