2 # vim:set et ts=4 sw=4:
4 """ Handles NEW and BYHAND packages
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
28 # 23:12|<aj> I will not hush!
30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
31 # 23:13|<aj> I shall not be silenced!
32 # 23:13|<aj> The world shall know!
33 # 23:13|<aj> The world *must* know!
34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
35 # 23:13|<aj> yay powerpuff girls!!
36 # 23:13|<aj> buttercup's my favourite, who's yours?
37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
38 # 23:14|<aj> *AREN'T YOU*?!
39 # 23:15|<aj> I will not be treated like this.
40 # 23:15|<aj> I shall have my revenge.
41 # 23:15|<aj> I SHALL!!!
43 ################################################################################
45 from __future__ import with_statement
56 import apt_pkg, apt_inst
57 import examine_package
59 from daklib.dbconn import *
60 from daklib.queue import *
61 from daklib import daklog
62 from daklib import utils
63 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
64 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
65 from daklib.summarystats import SummaryStats
66 from daklib.config import Config
67 from daklib.changesutils import *
76 ################################################################################
77 ################################################################################
78 ################################################################################
80 def recheck(upload, session):
81 # STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session)
82 if len(upload.rejects) > 0:
84 if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
87 print "REJECT\n%s" % '\n'.join(upload.rejects)
88 prompt = "[R]eject, Skip, Quit ?"
90 while prompt.find(answer) == -1:
91 answer = utils.our_raw_input(prompt)
92 m = re_default_answer.match(prompt)
95 answer = answer[:1].upper()
98 upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
99 upload.pkg.remove_known_changes(session=session)
110 ################################################################################
112 class Section_Completer:
113 def __init__ (self, session):
116 for s, in session.query(Section.section):
117 self.sections.append(s)
119 def complete(self, text, state):
123 for word in self.sections:
125 self.matches.append(word)
127 return self.matches[state]
131 ############################################################
133 class Priority_Completer:
134 def __init__ (self, session):
137 for p, in session.query(Priority.priority):
138 self.priorities.append(p)
140 def complete(self, text, state):
144 for word in self.priorities:
146 self.matches.append(word)
148 return self.matches[state]
152 ################################################################################
154 def print_new (new, upload, indexed, file=sys.stdout):
158 for pkg in new.keys():
160 section = new[pkg]["section"]
161 priority = new[pkg]["priority"]
162 if new[pkg]["section id"] == -1:
165 if new[pkg]["priority id"] == -1:
169 line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
171 line = "%-20s %-20s %-20s" % (pkg, priority, section)
172 line = line.strip()+'\n'
174 notes = get_new_comments(upload.pkg.changes.get("source"))
176 print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
177 % (note.author, note.version, note.notedate, note.comment)
179 return broken, len(notes) > 0
181 ################################################################################
183 def index_range (index):
187 return "1-%s" % (index)
189 ################################################################################
190 ################################################################################
192 def edit_new (new, upload):
193 # Write the current data to a temporary file
194 (fd, temp_filename) = utils.temp_filename()
195 temp_file = os.fdopen(fd, 'w')
196 print_new (new, upload, indexed=0, file=temp_file)
198 # Spawn an editor on that file
199 editor = os.environ.get("EDITOR","vi")
200 result = os.system("%s %s" % (editor, temp_filename))
202 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
203 # Read the edited data back in
204 temp_file = utils.open_file(temp_filename)
205 lines = temp_file.readlines()
207 os.unlink(temp_filename)
214 # Pad the list if necessary
215 s[len(s):3] = [None] * (3-len(s))
216 (pkg, priority, section) = s[:3]
217 if not new.has_key(pkg):
218 utils.warn("Ignoring unknown package '%s'" % (pkg))
220 # Strip off any invalid markers, print_new will readd them.
221 if section.endswith("[!]"):
222 section = section[:-3]
223 if priority.endswith("[!]"):
224 priority = priority[:-3]
225 for f in new[pkg]["files"]:
226 upload.pkg.files[f]["section"] = section
227 upload.pkg.files[f]["priority"] = priority
228 new[pkg]["section"] = section
229 new[pkg]["priority"] = priority
231 ################################################################################
233 def edit_index (new, upload, index):
234 priority = new[index]["priority"]
235 section = new[index]["section"]
236 ftype = new[index]["type"]
239 print "\t".join([index, priority, section])
243 prompt = "[B]oth, Priority, Section, Done ? "
245 prompt = "[S]ection, Done ? "
246 edit_priority = edit_section = 0
248 while prompt.find(answer) == -1:
249 answer = utils.our_raw_input(prompt)
250 m = re_default_answer.match(prompt)
253 answer = answer[:1].upper()
260 edit_priority = edit_section = 1
266 readline.set_completer(Priorities.complete)
268 while not got_priority:
269 new_priority = utils.our_raw_input("New priority: ").strip()
270 if new_priority not in Priorities.priorities:
271 print "E: '%s' is not a valid priority, try again." % (new_priority)
274 priority = new_priority
278 readline.set_completer(Sections.complete)
280 while not got_section:
281 new_section = utils.our_raw_input("New section: ").strip()
282 if new_section not in Sections.sections:
283 print "E: '%s' is not a valid section, try again." % (new_section)
286 section = new_section
288 # Reset the readline completer
289 readline.set_completer(None)
291 for f in new[index]["files"]:
292 upload.pkg.files[f]["section"] = section
293 upload.pkg.files[f]["priority"] = priority
294 new[index]["priority"] = priority
295 new[index]["section"] = section
298 ################################################################################
300 def edit_overrides (new, upload, session):
304 print_new (new, upload, indexed=1)
311 prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
314 while not got_answer:
315 answer = utils.our_raw_input(prompt)
316 if not answer.isdigit():
317 answer = answer[:1].upper()
318 if answer == "E" or answer == "D":
320 elif re_isanum.match (answer):
322 if (answer < 1) or (answer > index):
323 print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
328 edit_new(new, upload)
332 edit_index (new, upload, new_index[answer])
337 ################################################################################
339 def check_pkg (upload):
341 less_fd = os.popen("less -R -", 'w', 0)
342 stdout_fd = sys.stdout
345 changes = utils.parse_changes (upload.pkg.changes_file)
346 print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
347 files = upload.pkg.files
348 for f in files.keys():
349 if files[f].has_key("new"):
350 ftype = files[f]["type"]
352 print examine_package.check_deb(changes['distribution'], f)
354 print examine_package.check_dsc(changes['distribution'], f)
356 print examine_package.output_package_relations()
357 sys.stdout = stdout_fd
359 if e.errno == errno.EPIPE:
360 utils.warn("[examine_package] Caught EPIPE; skipping.")
364 except KeyboardInterrupt:
365 utils.warn("[examine_package] Caught C-c; skipping.")
368 ################################################################################
370 ## FIXME: horribly Debian specific
372 def do_bxa_notification(upload):
373 files = upload.pkg.files
375 for f in files.keys():
376 if files[f]["type"] == "deb":
377 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
379 summary += "Package: %s\n" % (control.Find("Package"))
380 summary += "Description: %s\n" % (control.Find("Description"))
381 upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
382 bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
383 utils.send_mail(bxa_mail)
385 ################################################################################
387 def add_overrides (new, upload, session):
388 changes = upload.pkg.changes
389 files = upload.pkg.files
390 srcpkg = changes.get("source")
392 for suite in changes["suite"].keys():
393 suite_id = get_suite(suite).suite_id
394 for pkg in new.keys():
395 component_id = get_component(new[pkg]["component"]).component_id
396 type_id = get_override_type(new[pkg]["type"]).overridetype_id
397 priority_id = new[pkg]["priority id"]
398 section_id = new[pkg]["section id"]
399 Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
400 session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
401 { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
402 for f in new[pkg]["files"]:
403 if files[f].has_key("new"):
409 if Config().FindB("Dinstall::BXANotify"):
410 do_bxa_notification(upload)
412 ################################################################################
414 def do_new(upload, session):
416 files = upload.pkg.files
417 upload.check_files(not Options["No-Action"])
418 changes = upload.pkg.changes
421 # Check for a valid distribution
422 upload.check_distributions()
424 # Make a copy of distribution we can happily trample on
425 changes["suite"] = copy.copy(changes["distribution"])
427 # Try to get an included dsc
429 (status, _) = upload.load_dsc()
433 # The main NEW processing loop
436 # Find out what's new
437 new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session)
443 if Options["No-Action"] or Options["Automatic"]:
446 (broken, note) = print_new(new, upload, indexed=0)
449 if not broken and not note:
450 prompt = "Add overrides, "
452 print "W: [!] marked entries must be fixed before package can be processed."
454 print "W: note must be removed before package can be processed."
455 prompt += "RemOve all notes, Remove note, "
457 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
459 while prompt.find(answer) == -1:
460 answer = utils.our_raw_input(prompt)
461 m = re_default_answer.search(prompt)
464 answer = answer[:1].upper()
466 if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
467 utils.warn("Trainees can't do that")
470 if answer == 'A' and not Options["Trainee"]:
473 done = add_overrides (new, upload, session)
474 new_accept(upload, Options["No-Action"], session)
475 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
476 except CantGetLockError:
477 print "Hello? Operator! Give me the number for 911!"
478 print "Dinstall in the locked area, cant process packages, come back later"
481 elif answer == 'E' and not Options["Trainee"]:
482 new = edit_overrides (new, upload, session)
483 elif answer == 'M' and not Options["Trainee"]:
484 aborted = upload.do_reject(manual=1,
485 reject_message=Options["Manual-Reject"],
486 notes=get_new_comments(changes.get("source", ""), session=session))
488 upload.pkg.remove_known_changes(session=session)
490 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
493 edit_note(get_new_comments(changes.get("source", ""), session=session),
494 upload, session, bool(Options["Trainee"]))
495 elif answer == 'P' and not Options["Trainee"]:
496 prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
498 Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
499 elif answer == 'R' and not Options["Trainee"]:
500 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
502 for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
505 elif answer == 'O' and not Options["Trainee"]:
506 confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
508 for c in get_new_comments(changes.get("source", ""), session=session):
518 ################################################################################
519 ################################################################################
520 ################################################################################
522 def usage (exit_code=0):
523 print """Usage: dak process-new [OPTION]... [CHANGES]...
524 -a, --automatic automatic run
525 -b, --no-binaries do not sort binary-NEW packages first
526 -c, --comments show NEW comments
527 -h, --help show this help and exit.
528 -m, --manual-reject=MSG manual reject with `msg'
529 -n, --no-action don't do anything
530 -t, --trainee FTP Trainee mode
531 -V, --version display the version number and exit"""
534 ################################################################################
536 def do_byhand(upload, session):
539 files = upload.pkg.files
543 for f in files.keys():
544 if files[f]["section"] == "byhand":
545 if os.path.exists(f):
546 print "W: %s still present; please process byhand components and try again." % (f)
552 if Options["No-Action"]:
555 if Options["Automatic"] and not Options["No-Action"]:
557 prompt = "[A]ccept, Manual reject, Skip, Quit ?"
559 prompt = "Manual reject, [S]kip, Quit ?"
561 while prompt.find(answer) == -1:
562 answer = utils.our_raw_input(prompt)
563 m = re_default_answer.search(prompt)
566 answer = answer[:1].upper()
569 dbchg = get_dbchange(upload.pkg.changes_file, session)
571 print "Warning: cannot find changes file in database; can't process BYHAND"
577 # Find the file entry in the database
579 for f in dbchg.files:
586 print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b
589 Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
590 except CantGetLockError:
591 print "Hello? Operator! Give me the number for 911!"
592 print "Dinstall in the locked area, cant process packages, come back later"
594 aborted = upload.do_reject(manual=1,
595 reject_message=Options["Manual-Reject"],
596 notes=get_new_comments(changes.get("source", ""), session=session))
598 upload.pkg.remove_known_changes(session=session)
600 Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
608 ################################################################################
610 def check_daily_lock():
612 Raises CantGetLockError if the dinstall daily.lock exists.
617 os.open(cnf["Process-New::DinstallLockFile"],
618 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
620 if e.errno == errno.EEXIST or e.errno == errno.EACCES:
621 raise CantGetLockError
623 os.unlink(cnf["Process-New::DinstallLockFile"])
626 @contextlib.contextmanager
627 def lock_package(package):
629 Lock C{package} so that noone else jumps in processing it.
631 @type package: string
632 @param package: source package name to lock
635 path = os.path.join(Config()["Process-New::LockDir"], package)
637 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
639 if e.errno == errno.EEXIST or e.errno == errno.EACCES:
640 user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
641 raise AlreadyLockedError, user
648 class clean_holding(object):
649 def __init__(self,pkg):
655 def __exit__(self, type, value, traceback):
658 for f in self.pkg.files.keys():
659 if os.path.exists(os.path.join(h.holding_dir, f)):
660 os.unlink(os.path.join(h.holding_dir, f))
663 def do_pkg(changes_full_path, session):
664 changes_dir = os.path.dirname(changes_full_path)
665 changes_file = os.path.basename(changes_full_path)
668 u.pkg.changes_file = changes_file
669 (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
670 u.load_changes(changes_file)
671 u.pkg.directory = changes_dir
674 origchanges = os.path.abspath(u.pkg.changes_file)
676 # Try to get an included dsc
678 (status, _) = u.load_dsc()
683 bcc = "X-DAK: dak process-new"
684 if cnf.has_key("Dinstall::Bcc"):
685 u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
687 u.Subst["__BCC__"] = bcc
690 for deb_filename, f in files.items():
691 if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
692 u.binary_file_checks(deb_filename, session)
693 u.check_binary_against_db(deb_filename, session)
695 u.source_file_checks(deb_filename, session)
696 u.check_source_against_db(deb_filename, session)
698 u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
701 with lock_package(u.pkg.changes["source"]):
702 with clean_holding(u.pkg):
703 if not recheck(u, session):
706 new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session)
708 do_byhand(u, session)
714 new_accept(u, Options["No-Action"], session)
715 except CantGetLockError:
716 print "Hello? Operator! Give me the number for 911!"
717 print "Dinstall in the locked area, cant process packages, come back later"
719 except AlreadyLockedError, e:
720 print "Seems to be locked by %s already, skipping..." % (e)
722 def show_new_comments(changes_files, session):
724 query = """SELECT package, version, comment, author
726 WHERE package IN ('"""
728 for changes in changes_files:
729 sources.add(os.path.basename(changes).split("_")[0])
731 query += "%s') ORDER BY package, version" % "', '".join(sources)
732 r = session.execute(query)
735 print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
739 ################################################################################
742 accept_count = SummaryStats().accept_count
743 accept_bytes = SummaryStats().accept_bytes
749 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
750 Logger.log(["total",accept_count,accept_bytes])
752 if not Options["No-Action"] and not Options["Trainee"]:
755 ################################################################################
758 global Options, Logger, Sections, Priorities
761 session = DBConn().session()
763 Arguments = [('a',"automatic","Process-New::Options::Automatic"),
764 ('b',"no-binaries","Process-New::Options::No-Binaries"),
765 ('c',"comments","Process-New::Options::Comments"),
766 ('h',"help","Process-New::Options::Help"),
767 ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
768 ('t',"trainee","Process-New::Options::Trainee"),
769 ('n',"no-action","Process-New::Options::No-Action")]
771 for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
772 if not cnf.has_key("Process-New::Options::%s" % (i)):
773 cnf["Process-New::Options::%s" % (i)] = ""
775 changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
776 if len(changes_files) == 0:
777 new_queue = get_policy_queue('new', session );
778 changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
780 changes_paths = [ os.path.abspath(j) for j in changes_files ]
782 Options = cnf.SubTree("Process-New::Options")
787 if not Options["No-Action"]:
789 Logger = daklog.Logger(cnf, "process-new")
790 except CantOpenError, e:
791 Options["Trainee"] = "True"
793 Sections = Section_Completer(session)
794 Priorities = Priority_Completer(session)
795 readline.parse_and_bind("tab: complete")
797 if len(changes_paths) > 1:
798 sys.stderr.write("Sorting changes...\n")
799 changes_files = sort_changes(changes_paths, session, Options["No-Binaries"])
801 if Options["Comments"]:
802 show_new_comments(changes_files, session)
804 for changes_file in changes_files:
805 changes_file = utils.validate_changes_file_arg(changes_file, 0)
808 print "\n" + os.path.basename(changes_file)
810 do_pkg (changes_file, session)
814 ################################################################################
816 if __name__ == '__main__':