3 # Handles NEW and BYHAND packages
4 # Copyright (C) 2001, 2002, 2003 James Troup <james@nocrew.org>
5 # $Id: lisa,v 1.27 2004-01-29 23:18:47 dsilvers Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # 23:12|<aj> I will not hush!
25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
26 # 23:13|<aj> I shall not be silenced!
27 # 23:13|<aj> The world shall know!
28 # 23:13|<aj> The world *must* know!
29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
30 # 23:13|<aj> yay powerpuff girls!!
31 # 23:13|<aj> buttercup's my favourite, who's yours?
32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
33 # 23:14|<aj> *AREN'T YOU*?!
34 # 23:15|<aj> I will not be treated like this.
35 # 23:15|<aj> I shall have my revenge.
36 # 23:15|<aj> I SHALL!!!
38 ################################################################################
40 import copy, errno, os, readline, stat, sys, tempfile;
41 import apt_pkg, apt_inst;
42 import db_access, fernanda, katie, logging, utils;
45 lisa_version = "$Revision: 1.27 $";
58 ################################################################################
59 ################################################################################
60 ################################################################################
62 def reject (str, prefix="Rejected: "):
63 global reject_message;
65 reject_message += prefix + str + "\n";
68 global reject_message;
69 files = Katie.pkg.files;
72 for file in files.keys():
73 # The .orig.tar.gz can disappear out from under us is it's a
74 # duplicate of one in the archive.
75 if not files.has_key(file):
77 # Check that the source still exists
78 if files[file]["type"] == "deb":
79 source_version = files[file]["source version"];
80 source_package = files[file]["source package"];
81 if not Katie.pkg.changes["architecture"].has_key("source") \
82 and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()):
83 source_epochless_version = utils.re_no_epoch.sub('', source_version);
84 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version);
85 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
86 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
88 # Version and file overwrite checks
89 if files[file]["type"] == "deb":
90 reject(Katie.check_binary_against_db(file));
91 elif files[file]["type"] == "dsc":
92 reject(Katie.check_source_against_db(file));
93 (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
98 if Options["No-Action"] or Options["Automatic"]:
101 print "REJECT\n" + reject_message,;
102 prompt = "[R]eject, Skip, Quit ?";
104 while prompt.find(answer) == -1:
105 answer = utils.our_raw_input(prompt);
106 m = katie.re_default_answer.match(prompt);
109 answer = answer[:1].upper();
112 Katie.do_reject(0, reject_message);
113 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
122 ################################################################################
124 def determine_new (changes, files):
127 # Build up a list of potentially new things
128 for file in files.keys():
130 # Skip byhand elements
131 if f["type"] == "byhand":
134 priority = f["priority"];
135 section = f["section"];
137 if section == "non-US/main":
140 component = f["component"];
144 if not new.has_key(pkg):
146 new[pkg]["priority"] = priority;
147 new[pkg]["section"] = section;
148 new[pkg]["type"] = type;
149 new[pkg]["component"] = component;
150 new[pkg]["files"] = [];
152 old_type = new[pkg]["type"];
154 # source gets trumped by deb or udeb
155 if old_type == "dsc":
156 new[pkg]["priority"] = priority;
157 new[pkg]["section"] = section;
158 new[pkg]["type"] = type;
159 new[pkg]["component"] = component;
160 new[pkg]["files"].append(file);
161 if f.has_key("othercomponents"):
162 new[pkg]["othercomponents"] = f["othercomponents"];
164 for suite in changes["suite"].keys():
165 suite_id = db_access.get_suite_id(suite);
166 for pkg in new.keys():
167 component_id = db_access.get_component_id(new[pkg]["component"]);
168 type_id = db_access.get_override_type_id(new[pkg]["type"]);
169 q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id));
172 for file in new[pkg]["files"]:
173 if files[file].has_key("new"):
174 del files[file]["new"];
177 if changes["suite"].has_key("stable"):
178 print "WARNING: overrides will be added for stable!";
179 if changes["suite"].has_key("oldstable"):
180 print "WARNING: overrides will be added for OLDstable!";
181 for pkg in new.keys():
182 if new[pkg].has_key("othercomponents"):
183 print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]);
187 ################################################################################
189 def indiv_sg_compare (a, b):
190 """Sort by source name, source, version, 'have source', and
191 finally by filename."""
192 # Sort by source version
193 q = apt_pkg.VersionCompare(a["version"], b["version"]);
197 # Sort by 'have source'
198 a_has_source = a["architecture"].get("source");
199 b_has_source = b["architecture"].get("source");
200 if a_has_source and not b_has_source:
202 elif b_has_source and not a_has_source:
205 return cmp(a["filename"], b["filename"]);
207 ############################################################
209 def sg_compare (a, b):
212 """Sort by have note, time of oldest upload."""
214 a_note_state = a["note_state"];
215 b_note_state = b["note_state"];
216 if a_note_state < b_note_state:
218 elif a_note_state > b_note_state:
221 # Sort by time of oldest upload
222 return cmp(a["oldest"], b["oldest"]);
224 def sort_changes(changes_files):
225 """Sort into source groups, then sort each source group by version,
226 have source, filename. Finally, sort the source groups by have
227 note, time of oldest upload of each source upload."""
228 if len(changes_files) == 1:
229 return changes_files;
233 # Read in all the .changes files
234 for filename in changes_files:
236 Katie.pkg.changes_file = filename;
239 cache[filename] = copy.copy(Katie.pkg.changes);
240 cache[filename]["filename"] = filename;
242 sorted_list.append(filename);
244 # Divide the .changes into per-source groups
246 for filename in cache.keys():
247 source = cache[filename]["source"];
248 if not per_source.has_key(source):
249 per_source[source] = {};
250 per_source[source]["list"] = [];
251 per_source[source]["list"].append(cache[filename]);
252 # Determine oldest time and have note status for each source group
253 for source in per_source.keys():
254 source_list = per_source[source]["list"];
255 first = source_list[0];
256 oldest = os.stat(first["filename"])[stat.ST_CTIME];
258 for d in per_source[source]["list"]:
259 ctime = os.stat(d["filename"])[stat.ST_CTIME];
262 have_note += (d.has_key("lisa note"));
263 per_source[source]["oldest"] = oldest;
265 per_source[source]["note_state"] = 0; # none
266 elif have_note < len(source_list):
267 per_source[source]["note_state"] = 1; # some
269 per_source[source]["note_state"] = 2; # all
270 per_source[source]["list"].sort(indiv_sg_compare);
271 per_source_items = per_source.items();
272 per_source_items.sort(sg_compare);
273 for i in per_source_items:
274 for j in i[1]["list"]:
275 sorted_list.append(j["filename"]);
278 ################################################################################
280 class Section_Completer:
283 q = projectB.query("SELECT section FROM section");
284 for i in q.getresult():
285 self.sections.append(i[0]);
287 def complete(self, text, state):
291 for word in self.sections:
293 self.matches.append(word);
295 return self.matches[state]
299 ############################################################
301 class Priority_Completer:
303 self.priorities = [];
304 q = projectB.query("SELECT priority FROM priority");
305 for i in q.getresult():
306 self.priorities.append(i[0]);
308 def complete(self, text, state):
312 for word in self.priorities:
314 self.matches.append(word);
316 return self.matches[state]
320 ################################################################################
322 def check_valid (new):
323 for pkg in new.keys():
324 section = new[pkg]["section"];
325 priority = new[pkg]["priority"];
326 type = new[pkg]["type"];
327 new[pkg]["section id"] = db_access.get_section_id(section);
328 new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]);
330 if (section == "debian-installer" and type != "udeb") or \
331 (section != "debian-installer" and type == "udeb"):
332 new[pkg]["section id"] = -1;
333 if (priority == "source" and type != "dsc") or \
334 (priority != "source" and type == "dsc"):
335 new[pkg]["priority id"] = -1;
337 ################################################################################
339 def print_new (new, indexed, file=sys.stdout):
343 for pkg in new.keys():
345 section = new[pkg]["section"];
346 priority = new[pkg]["priority"];
347 if new[pkg]["section id"] == -1:
350 if new[pkg]["priority id"] == -1:
354 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section);
356 line = "%-20s %-20s %-20s" % (pkg, priority, section);
357 line = line.strip()+'\n';
359 note = Katie.pkg.changes.get("lisa note");
366 ################################################################################
370 if f.has_key("dbtype"):
372 elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
375 utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type));
377 # Validate the override type
378 type_id = db_access.get_override_type_id(type);
380 utils.fubar("invalid type (%s) for new. Say wha?" % (type));
384 ################################################################################
386 def index_range (index):
390 return "1-%s" % (index);
392 ################################################################################
393 ################################################################################
396 # Write the current data to a temporary file
397 temp_filename = tempfile.mktemp();
398 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
400 temp_file = utils.open_file(temp_filename, 'w');
401 print_new (new, 0, temp_file);
403 # Spawn an editor on that file
404 editor = os.environ.get("EDITOR","vi")
405 result = os.system("%s %s" % (editor, temp_filename))
407 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
408 # Read the edited data back in
409 temp_file = utils.open_file(temp_filename);
410 lines = temp_file.readlines();
412 os.unlink(temp_filename);
419 # Pad the list if necessary
420 s[len(s):3] = [None] * (3-len(s));
421 (pkg, priority, section) = s[:3];
422 if not new.has_key(pkg):
423 utils.warn("Ignoring unknown package '%s'" % (pkg));
425 # Strip off any invalid markers, print_new will readd them.
426 if section.endswith("[!]"):
427 section = section[:-3];
428 if priority.endswith("[!]"):
429 priority = priority[:-3];
430 for file in new[pkg]["files"]:
431 Katie.pkg.files[file]["section"] = section;
432 Katie.pkg.files[file]["priority"] = priority;
433 new[pkg]["section"] = section;
434 new[pkg]["priority"] = priority;
436 ################################################################################
438 def edit_index (new, index):
439 priority = new[index]["priority"]
440 section = new[index]["section"]
441 type = new[index]["type"];
444 print "\t".join([index, priority, section]);
448 prompt = "[B]oth, Priority, Section, Done ? ";
450 prompt = "[S]ection, Done ? ";
451 edit_priority = edit_section = 0;
453 while prompt.find(answer) == -1:
454 answer = utils.our_raw_input(prompt);
455 m = katie.re_default_answer.match(prompt)
458 answer = answer[:1].upper()
465 edit_priority = edit_section = 1;
471 readline.set_completer(Priorities.complete);
473 while not got_priority:
474 new_priority = utils.our_raw_input("New priority: ").strip();
475 if Priorities.priorities.count(new_priority) == 0:
476 print "E: '%s' is not a valid priority, try again." % (new_priority);
479 priority = new_priority;
483 readline.set_completer(Sections.complete);
485 while not got_section:
486 new_section = utils.our_raw_input("New section: ").strip();
487 if Sections.sections.count(new_section) == 0:
488 print "E: '%s' is not a valid section, try again." % (new_section);
491 section = new_section;
493 # Reset the readline completer
494 readline.set_completer(None);
496 for file in new[index]["files"]:
497 Katie.pkg.files[file]["section"] = section;
498 Katie.pkg.files[file]["priority"] = priority;
499 new[index]["priority"] = priority;
500 new[index]["section"] = section;
503 ################################################################################
505 def edit_overrides (new):
514 new_index[index] = i;
516 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index));
519 while not got_answer:
520 answer = utils.our_raw_input(prompt);
521 if not utils.str_isnum(answer):
522 answer = answer[:1].upper();
523 if answer == "E" or answer == "D":
525 elif katie.re_isanum.match (answer):
526 answer = int(answer);
527 if (answer < 1) or (answer > index):
528 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index));
537 edit_index (new, new_index[answer]);
541 ################################################################################
544 # Write the current data to a temporary file
545 temp_filename = tempfile.mktemp();
546 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
548 temp_file = utils.open_file(temp_filename, 'w');
549 temp_file.write(note);
551 editor = os.environ.get("EDITOR","vi")
554 os.system("%s %s" % (editor, temp_filename))
555 temp_file = utils.open_file(temp_filename);
556 note = temp_file.read().rstrip();
559 print utils.prefix_multi_line_string(note," ");
560 prompt = "[D]one, Edit, Abandon, Quit ?"
562 while prompt.find(answer) == -1:
563 answer = utils.our_raw_input(prompt);
564 m = katie.re_default_answer.search(prompt);
567 answer = answer[:1].upper();
568 os.unlink(temp_filename);
573 Katie.pkg.changes["lisa note"] = note;
574 Katie.dump_vars(Cnf["Dir::Queue::New"]);
576 ################################################################################
580 less_fd = os.popen("less -R -", 'w', 0);
581 stdout_fd = sys.stdout;
583 sys.stdout = less_fd;
584 fernanda.display_changes(Katie.pkg.changes_file);
585 files = Katie.pkg.files;
586 for file in files.keys():
587 if files[file].has_key("new"):
588 type = files[file]["type"];
590 fernanda.check_deb(file);
592 fernanda.check_dsc(file);
594 sys.stdout = stdout_fd;
596 if errno.errorcode[e.errno] == 'EPIPE':
597 utils.warn("[fernanda] Caught EPIPE; skipping.");
601 except KeyboardInterrupt:
602 utils.warn("[fernanda] Caught C-c; skipping.");
605 ################################################################################
607 ## FIXME: horribly Debian specific
609 def do_bxa_notification():
610 files = Katie.pkg.files;
612 for file in files.keys():
613 if files[file]["type"] == "deb":
614 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
616 summary += "Package: %s\n" % (control.Find("Package"));
617 summary += "Description: %s\n" % (control.Find("Description"));
618 Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary;
619 bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification");
620 utils.send_mail(bxa_mail);
622 ################################################################################
624 def add_overrides (new):
625 changes = Katie.pkg.changes;
626 files = Katie.pkg.files;
628 projectB.query("BEGIN WORK");
629 for suite in changes["suite"].keys():
630 suite_id = db_access.get_suite_id(suite);
631 for pkg in new.keys():
632 component_id = db_access.get_component_id(new[pkg]["component"]);
633 type_id = db_access.get_override_type_id(new[pkg]["type"]);
634 priority_id = new[pkg]["priority id"];
635 section_id = new[pkg]["section id"];
636 projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id));
637 for file in new[pkg]["files"]:
638 if files[file].has_key("new"):
639 del files[file]["new"];
642 projectB.query("COMMIT WORK");
644 if Cnf.FindB("Dinstall::BXANotify"):
645 do_bxa_notification();
647 ################################################################################
649 def prod_maintainer ():
650 # Here we prepare an editor and get them ready to prod...
651 temp_filename = tempfile.mktemp();
652 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
654 editor = os.environ.get("EDITOR","vi")
657 os.system("%s %s" % (editor, temp_filename))
658 file = utils.open_file(temp_filename);
659 prod_message = "".join(file.readlines());
661 print "Prod message:";
662 print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1);
663 prompt = "[P]rod, Edit, Abandon, Quit ?"
665 while prompt.find(answer) == -1:
666 answer = utils.our_raw_input(prompt);
667 m = katie.re_default_answer.search(prompt);
670 answer = answer[:1].upper();
671 os.unlink(temp_filename);
676 # Otherwise, do the proding...
677 user_email_address = utils.whoami() + " <%s>" % (
678 Cnf["Dinstall::MyAdminAddress"]);
682 Subst["__FROM_ADDRESS__"] = user_email_address;
683 Subst["__PROD_MESSAGE__"] = prod_message;
684 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
686 prod_mail_message = utils.TemplateSubst(
687 Subst,Cnf["Dir::Templates"]+"/lisa.prod");
689 # Send the prod mail if appropriate
690 if not Cnf["Dinstall::Options::No-Mail"]:
691 utils.send_mail(prod_mail_message);
693 print "Sent proding message";
695 ################################################################################
699 files = Katie.pkg.files;
700 changes = Katie.pkg.changes;
702 # Make a copy of distribution we can happily trample on
703 changes["suite"] = copy.copy(changes["distribution"]);
705 # Fix up the list of target suites
706 for suite in changes["suite"].keys():
707 override = Cnf.Find("Suite::%s::OverrideSuite" % (suite));
709 del changes["suite"][suite];
710 changes["suite"][override] = 1;
712 for suite in changes["suite"].keys():
713 suite_id = db_access.get_suite_id(suite);
715 utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite));
717 # The main NEW processing loop
720 # Find out what's new
721 new = determine_new(changes, files);
727 if Options["No-Action"] or Options["Automatic"]:
730 (broken, note) = print_new(new, 0);
733 if not broken and not note:
734 prompt = "Add overrides, ";
736 print "W: [!] marked entries must be fixed before package can be processed.";
738 print "W: note must be removed before package can be processed.";
739 prompt += "Remove note, ";
741 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?";
743 while prompt.find(answer) == -1:
744 answer = utils.our_raw_input(prompt);
745 m = katie.re_default_answer.search(prompt);
748 answer = answer[:1].upper()
751 done = add_overrides (new);
755 new = edit_overrides (new);
757 aborted = Katie.do_reject(1, Options["Manual-Reject"]);
759 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
762 edit_note(changes.get("lisa note", ""));
766 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower();
768 del changes["lisa note"];
774 ################################################################################
775 ################################################################################
776 ################################################################################
778 def usage (exit_code=0):
779 print """Usage: lisa [OPTION]... [CHANGES]...
780 -a, --automatic automatic run
781 -h, --help show this help and exit.
782 -m, --manual-reject=MSG manual reject with `msg'
783 -n, --no-action don't do anything
784 -V, --version display the version number and exit"""
787 ################################################################################
790 global Cnf, Options, Logger, Katie, projectB, Sections, Priorities;
792 Cnf = utils.get_conf();
794 Arguments = [('a',"automatic","Lisa::Options::Automatic"),
795 ('h',"help","Lisa::Options::Help"),
796 ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"),
797 ('n',"no-action","Lisa::Options::No-Action"),
798 ('V',"version","Lisa::Options::Version")];
800 for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
801 if not Cnf.has_key("Lisa::Options::%s" % (i)):
802 Cnf["Lisa::Options::%s" % (i)] = "";
804 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
805 Options = Cnf.SubTree("Lisa::Options")
810 if Options["Version"]:
811 print "lisa %s" % (lisa_version);
814 Katie = katie.Katie(Cnf);
816 if not Options["No-Action"]:
817 Logger = Katie.Logger = logging.Logger(Cnf, "lisa");
819 projectB = Katie.projectB;
821 Sections = Section_Completer();
822 Priorities = Priority_Completer();
823 readline.parse_and_bind("tab: complete");
825 return changes_files;
827 ################################################################################
832 files = Katie.pkg.files;
836 for file in files.keys():
837 if files[file]["type"] == "byhand":
838 if os.path.exists(file):
839 print "W: %s still present; please process byhand components and try again." % (file);
845 if Options["No-Action"]:
848 if Options["Automatic"] and not Options["No-Action"]:
850 prompt = "[A]ccept, Manual reject, Skip, Quit ?";
852 prompt = "Manual reject, [S]kip, Quit ?";
854 while prompt.find(answer) == -1:
855 answer = utils.our_raw_input(prompt);
856 m = katie.re_default_answer.search(prompt);
859 answer = answer[:1].upper();
866 Katie.do_reject(1, Options["Manual-Reject"]);
867 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
874 ################################################################################
878 if not Options["No-Action"]:
879 (summary, short_summary) = Katie.build_summaries();
880 Katie.accept(summary, short_summary);
881 os.unlink(Katie.pkg.changes_file[:-8]+".katie");
883 def check_status(files):
885 for file in files.keys():
886 if files[file]["type"] == "byhand":
888 elif files[file].has_key("new"):
890 return (new, byhand);
892 def do_pkg(changes_file):
893 Katie.pkg.changes_file = changes_file;
896 Katie.update_subst();
897 files = Katie.pkg.files;
902 (new, byhand) = check_status(files);
908 (new, byhand) = check_status(files);
910 if not new and not byhand:
913 ################################################################################
916 accept_count = Katie.accept_count;
917 accept_bytes = Katie.accept_bytes;
923 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))));
924 Logger.log(["total",accept_count,accept_bytes]);
926 if not Options["No-Action"]:
929 ################################################################################
932 changes_files = init();
933 if len(changes_files) > 50:
934 sys.stderr.write("Sorting changes...\n");
935 changes_files = sort_changes(changes_files);
937 # Kill me now? **FIXME**
938 Cnf["Dinstall::Options::No-Mail"] = "";
939 bcc = "X-Katie: lisa %s" % (lisa_version);
940 if Cnf.has_key("Dinstall::Bcc"):
941 Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
943 Katie.Subst["__BCC__"] = bcc;
945 for changes_file in changes_files:
946 changes_file = utils.validate_changes_file_arg(changes_file, 0);
949 print "\n" + changes_file;
950 do_pkg (changes_file);
954 ################################################################################
956 if __name__ == '__main__':