2 # vim:set et ts=4 sw=4:
4 """ Handles NEW and BYHAND packages
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @copyright: 2009 Joerg Jaspert <joerg@debian.org>
9 @copyright: 2009 Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
28 # 23:12|<aj> I will not hush!
30 # 23:12|<aj> Where there is injustice in the world, I shall be there!
31 # 23:13|<aj> I shall not be silenced!
32 # 23:13|<aj> The world shall know!
33 # 23:13|<aj> The world *must* know!
34 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
35 # 23:13|<aj> yay powerpuff girls!!
36 # 23:13|<aj> buttercup's my favourite, who's yours?
37 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
38 # 23:14|<aj> *AREN'T YOU*?!
39 # 23:15|<aj> I will not be treated like this.
40 # 23:15|<aj> I shall have my revenge.
41 # 23:15|<aj> I SHALL!!!
43 ################################################################################
54 import apt_pkg, apt_inst
55 import examine_package
57 import daklib.daksubprocess
58 from sqlalchemy import or_
60 from daklib.dbconn import *
61 from daklib.queue import *
62 from daklib import daklog
63 from daklib import utils
64 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
65 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
66 from daklib.summarystats import SummaryStats
67 from daklib.config import Config
68 from daklib.policy import UploadCopy, PolicyQueueUploadHandler
77 ################################################################################
78 ################################################################################
79 ################################################################################
81 class Section_Completer:
82 def __init__ (self, session):
85 for s, in session.query(Section.section):
86 self.sections.append(s)
88 def complete(self, text, state):
92 for word in self.sections:
94 self.matches.append(word)
96 return self.matches[state]
100 ############################################################
102 class Priority_Completer:
103 def __init__ (self, session):
106 for p, in session.query(Priority.priority):
107 self.priorities.append(p)
109 def complete(self, text, state):
113 for word in self.priorities:
115 self.matches.append(word)
117 return self.matches[state]
121 ################################################################################
123 def takenover_binaries(upload, missing, session):
125 binaries = set([x.package for x in upload.binaries])
127 if m['type'] != 'dsc':
128 binaries.discard(m['package'])
130 source = upload.binaries[0].source.source
131 suite = upload.target_suite.overridesuite or \
132 upload.target_suite.suite_name
133 suites = [s[0] for s in session.query(Suite.suite_name).filter \
134 (or_(Suite.suite_name == suite,
135 Suite.overridesuite == suite)).all()]
136 rows = session.query(DBSource.source, DBBinary.package).distinct(). \
137 filter(DBBinary.package.in_(binaries)). \
138 join(DBBinary.source). \
139 filter(DBSource.source != source). \
140 join(DBBinary.suites). \
141 filter(Suite.suite_name.in_(suites)). \
142 order_by(DBSource.source, DBBinary.package).all()
145 ################################################################################
147 def print_new (upload, missing, indexed, session, file=sys.stdout):
148 check_valid(missing, session)
152 if m['type'] != 'deb':
153 package = '{0}:{1}'.format(m['type'], m['package'])
155 package = m['package']
156 section = m['section']
157 priority = m['priority']
158 included = "" if m['included'] else "NOT UPLOADED"
160 line = "(%s): %-20s %-20s %-20s %s" % (index, package, priority, section, included)
162 line = "%-20s %-20s %-20s %s" % (package, priority, section, included)
167 takenover = takenover_binaries(upload, missing, session)
169 print '\n\nBINARIES TAKEN OVER\n'
171 print '%s: %s' % (t[0], t[1])
172 notes = get_new_comments(upload.policy_queue, upload.changes.source)
174 print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
175 % (note.author, note.version, note.notedate, note.comment)
177 return len(notes) > 0
179 ################################################################################
181 def index_range (index):
185 return "1-%s" % (index)
187 ################################################################################
188 ################################################################################
190 def edit_new (overrides, upload, session):
191 # Write the current data to a temporary file
192 (fd, temp_filename) = utils.temp_filename()
193 temp_file = os.fdopen(fd, 'w')
194 print_new (upload, overrides, indexed=0, session=session, file=temp_file)
196 # Spawn an editor on that file
197 editor = os.environ.get("EDITOR","vi")
198 result = os.system("%s %s" % (editor, temp_filename))
200 utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
201 # Read the edited data back in
202 temp_file = utils.open_file(temp_filename)
203 lines = temp_file.readlines()
205 os.unlink(temp_filename)
207 overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ])
212 if line == "" or line[0] == '#':
215 # Pad the list if necessary
216 s[len(s):3] = [None] * (3-len(s))
217 (pkg, priority, section) = s[:3]
218 if pkg.find(':') != -1:
219 type, pkg = pkg.split(':', 1)
222 o = overrides_map.get((type, pkg), None)
224 utils.warn("Ignoring unknown package '%s'" % (pkg))
226 if section.find('/') != -1:
227 component = section.split('/', 1)[0]
230 new_overrides.append(dict(
236 included=o['included'],
240 ################################################################################
242 def edit_index (new, upload, index):
243 package = new[index]['package']
244 priority = new[index]["priority"]
245 section = new[index]["section"]
246 ftype = new[index]["type"]
249 print "\t".join([package, priority, section])
253 prompt = "[B]oth, Priority, Section, Done ? "
255 prompt = "[S]ection, Done ? "
256 edit_priority = edit_section = 0
258 while prompt.find(answer) == -1:
259 answer = utils.our_raw_input(prompt)
260 m = re_default_answer.match(prompt)
263 answer = answer[:1].upper()
270 edit_priority = edit_section = 1
276 readline.set_completer(Priorities.complete)
278 while not got_priority:
279 new_priority = utils.our_raw_input("New priority: ").strip()
280 if new_priority not in Priorities.priorities:
281 print "E: '%s' is not a valid priority, try again." % (new_priority)
284 priority = new_priority
288 readline.set_completer(Sections.complete)
290 while not got_section:
291 new_section = utils.our_raw_input("New section: ").strip()
292 if new_section not in Sections.sections:
293 print "E: '%s' is not a valid section, try again." % (new_section)
296 section = new_section
298 # Reset the readline completer
299 readline.set_completer(None)
301 new[index]["priority"] = priority
302 new[index]["section"] = section
303 if section.find('/') != -1:
304 component = section.split('/', 1)[0]
307 new[index]['component'] = component
311 ################################################################################
313 def edit_overrides (new, upload, session):
317 print_new (upload, new, indexed=1, session=session)
318 prompt = "edit override <n>, Editor, Done ? "
321 while not got_answer:
322 answer = utils.our_raw_input(prompt)
323 if not answer.isdigit():
324 answer = answer[:1].upper()
325 if answer == "E" or answer == "D":
327 elif re_isanum.match (answer):
329 if answer < 1 or answer > len(new):
330 print "{0} is not a valid index. Please retry.".format(answer)
335 new = edit_new(new, upload, session)
339 edit_index (new, upload, answer - 1)
344 ################################################################################
346 def check_pkg (upload, upload_copy, session):
348 save_stdout = sys.stdout
349 changes = os.path.join(upload_copy.directory, upload.changes.changesname)
350 suite_name = upload.target_suite.suite_name
351 handler = PolicyQueueUploadHandler(upload, session)
352 missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)]
354 less_cmd = ("less", "-R", "-")
355 less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE)
357 sys.stdout = less_process.stdin
358 print examine_package.display_changes(suite_name, changes)
360 source = upload.source
361 if source is not None:
362 source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename))
363 print examine_package.check_dsc(suite_name, source_file)
365 for binary in upload.binaries:
366 binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
367 examined = examine_package.check_deb(suite_name, binary_file)
368 # We always need to call check_deb to display package relations for every binary,
369 # but we print its output only if new overrides are being added.
370 if ("deb", binary.package) in missing:
373 print examine_package.output_package_relations()
374 less_process.stdin.close()
376 if e.errno == errno.EPIPE:
377 utils.warn("[examine_package] Caught EPIPE; skipping.")
380 except KeyboardInterrupt:
381 utils.warn("[examine_package] Caught C-c; skipping.")
384 sys.stdout = save_stdout
386 ################################################################################
388 ## FIXME: horribly Debian specific
390 def do_bxa_notification(new, upload, session):
393 new = set([ o['package'] for o in new if o['type'] == 'deb' ])
397 key = session.query(MetadataKey).filter_by(key='Description').one()
399 for binary in upload.binaries:
400 if binary.package not in new:
402 description = session.query(BinaryMetadata).filter_by(binary=binary, key=key).one().value
404 summary += "Package: {0}\n".format(binary.package)
405 summary += "Description: {0}\n".format(description)
408 '__DISTRO__': cnf['Dinstall::MyDistribution'],
409 '__BCC__': 'X-DAK: dak process-new',
410 '__BINARY_DESCRIPTIONS__': summary,
413 bxa_mail = utils.TemplateSubst(subst,os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification"))
414 utils.send_mail(bxa_mail)
416 ################################################################################
418 def add_overrides (new_overrides, suite, session):
419 if suite.overridesuite is not None:
420 suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
422 for override in new_overrides:
423 package = override['package']
424 priority = session.query(Priority).filter_by(priority=override['priority']).first()
425 section = session.query(Section).filter_by(section=override['section']).first()
426 component = get_mapped_component(override['component'], session)
427 overridetype = session.query(OverrideType).filter_by(overridetype=override['type']).one()
430 raise Exception('Invalid priority {0} for package {1}'.format(priority, package))
432 raise Exception('Invalid section {0} for package {1}'.format(section, package))
433 if component is None:
434 raise Exception('Invalid component {0} for package {1}'.format(component, package))
436 o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype)
441 ################################################################################
443 def run_user_inspect_command(upload, upload_copy):
444 command = os.environ.get('DAK_INSPECT_UPLOAD')
448 directory = upload_copy.directory
450 dsc = os.path.basename(upload.source.poolfile.filename)
453 changes = upload.changes.changesname
455 shell_command = command.format(
461 daklib.daksubprocess.check_call(shell_command, shell=True)
463 ################################################################################
465 def get_reject_reason(reason=''):
466 """get reason for rejection
469 @return: string giving the reason for the rejection or C{None} if the
470 rejection should be cancelled
473 if Options['Automatic']:
477 reason = utils.call_editor(reason)
478 print "Reject message:"
479 print utils.prefix_multi_line_string(reason, " ", include_blank_lines=1)
480 prompt = "[R]eject, Edit, Abandon, Quit ?"
482 while prompt.find(answer) == -1:
483 answer = utils.our_raw_input(prompt)
484 m = re_default_answer.search(prompt)
487 answer = answer[:1].upper()
496 ################################################################################
498 def do_new(upload, upload_copy, handler, session):
501 run_user_inspect_command(upload, upload_copy)
503 # The main NEW processing loop
507 queuedir = upload.policy_queue.path
508 byhand = upload.byhand
510 missing = handler.missing_overrides(hints=missing)
511 broken = not check_valid(missing, session)
513 changesname = os.path.basename(upload.changes.changesname)
517 print "-" * len(changesname)
519 print " Target: {0}".format(upload.target_suite.suite_name)
520 print " Changed-By: {0}".format(upload.changes.changedby)
523 #if len(byhand) == 0 and len(missing) == 0:
530 if Options["No-Action"] or Options["Automatic"]:
533 note = print_new(upload, missing, indexed=0, session=session)
536 has_unprocessed_byhand = False
538 path = os.path.join(queuedir, f.filename)
539 if not f.processed and os.path.exists(path):
540 print "W: {0} still present; please process byhand components and try again".format(f.filename)
541 has_unprocessed_byhand = True
543 if not has_unprocessed_byhand and not broken and not note:
544 if len(missing) == 0:
548 prompt = "Add overrides, "
550 print "W: [!] marked entries must be fixed before package can be processed."
552 print "W: note must be removed before package can be processed."
553 prompt += "RemOve all notes, Remove note, "
555 prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
557 while prompt.find(answer) == -1:
558 answer = utils.our_raw_input(prompt)
559 m = re_default_answer.search(prompt)
562 answer = answer[:1].upper()
564 if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
565 utils.warn("Trainees can't do that")
568 if answer == 'A' and not Options["Trainee"]:
569 add_overrides(missing, upload.target_suite, session)
570 if Config().find_b("Dinstall::BXANotify"):
571 do_bxa_notification(missing, upload, session)
574 Logger.log(["NEW ACCEPT", upload.changes.changesname])
576 check_pkg(upload, upload_copy, session)
577 elif answer == 'E' and not Options["Trainee"]:
578 missing = edit_overrides (missing, upload, session)
579 elif answer == 'M' and not Options["Trainee"]:
580 reason = Options.get('Manual-Reject', '') + "\n"
581 reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)])
582 reason = get_reject_reason(reason)
583 if reason is not None:
584 Logger.log(["NEW REJECT", upload.changes.changesname])
585 handler.reject(reason)
588 if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
589 upload, session, bool(Options["Trainee"])) == 0:
592 elif answer == 'P' and not Options["Trainee"]:
593 if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
597 Logger.log(["NEW PROD", upload.changes.changesname])
598 elif answer == 'R' and not Options["Trainee"]:
599 confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
601 for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session):
604 elif answer == 'O' and not Options["Trainee"]:
605 confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
607 for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session):
617 if handler.get_action():
618 print "PENDING %s\n" % handler.get_action()
620 ################################################################################
621 ################################################################################
622 ################################################################################
624 def usage (exit_code=0):
625 print """Usage: dak process-new [OPTION]... [CHANGES]...
626 -a, --automatic automatic run
627 -b, --no-binaries do not sort binary-NEW packages first
628 -c, --comments show NEW comments
629 -h, --help show this help and exit.
630 -m, --manual-reject=MSG manual reject with `msg'
631 -n, --no-action don't do anything
632 -q, --queue=QUEUE operate on a different queue
633 -t, --trainee FTP Trainee mode
634 -V, --version display the version number and exit
636 ENVIRONMENT VARIABLES
638 DAK_INSPECT_UPLOAD: shell command to run to inspect a package
639 The command is automatically run in a shell when an upload
640 is checked. The following substitutions are available:
642 {directory}: directory the upload is contained in
643 {dsc}: name of the included dsc or the empty string
644 {changes}: name of the changes file
646 Note that Python's 'format' method is used to format the command.
648 Example: run mc in a tmux session to inspect the upload
650 export DAK_INSPECT_UPLOAD='tmux new-session -d -s process-new 2>/dev/null; tmux new-window -n "{changes}" -t process-new:0 -k "cd {directory}; mc"'
654 tmux attach -t process-new
656 in a separate terminal session.
660 ################################################################################
662 @contextlib.contextmanager
663 def lock_package(package):
665 Lock C{package} so that noone else jumps in processing it.
667 @type package: string
668 @param package: source package name to lock
673 path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package)
676 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
678 if e.errno == errno.EEXIST or e.errno == errno.EACCES:
679 user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
680 raise AlreadyLockedError(user)
687 def do_pkg(upload, session):
688 # Try to get an included dsc
692 group = cnf.get('Dinstall::UnprivGroup') or None
694 #bcc = "X-DAK: dak process-new"
695 #if cnf.has_key("Dinstall::Bcc"):
696 # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
698 # u.Subst["__BCC__"] = bcc
701 with lock_package(upload.changes.source):
702 with UploadCopy(upload, group=group) as upload_copy:
703 handler = PolicyQueueUploadHandler(upload, session)
704 if handler.get_action() is not None:
705 print "PENDING %s\n" % handler.get_action()
708 do_new(upload, upload_copy, handler, session)
709 except AlreadyLockedError as e:
710 print "Seems to be locked by %s already, skipping..." % (e)
712 def show_new_comments(uploads, session):
713 sources = [ upload.changes.source for upload in uploads ]
714 if len(sources) == 0:
717 query = """SELECT package, version, comment, author
719 WHERE package IN :sources
720 ORDER BY package, version"""
722 r = session.execute(query, params=dict(sources=tuple(sources)))
725 print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
729 ################################################################################
731 def sort_uploads(new_queue, uploads, session, nobinaries=False):
734 suitesrc = [s.source for s in session.query(DBSource.source). \
735 filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))]
736 comments = [p.package for p in session.query(NewComment.package). \
737 filter_by(trainee=False, policy_queue=new_queue).distinct()]
738 for upload in uploads:
739 source = upload.changes.source
740 if not source in sources:
742 sources[source].append({'upload': upload,
743 'date': upload.changes.created,
745 'binary': True if source in suitesrc else False,
746 'comments': True if source in comments else False})
748 if len(sources[src]) > 1:
749 changes = sources[src]
750 firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date']
751 changes.sort(key=lambda item:item['date'])
752 for i in range (0, len(changes)):
753 changes[i]['date'] = firstseen
754 changes[i]['stack'] = i + 1
755 sorteduploads += sources[src]
757 sorteduploads = [u["upload"] for u in sorted(sorteduploads,
758 key=lambda k: (k["comments"], k["binary"],
759 k["date"], -k["stack"]))]
761 sorteduploads = [u["upload"] for u in sorted(sorteduploads,
762 key=lambda k: (k["comments"], -k["binary"],
763 k["date"], -k["stack"]))]
766 ################################################################################
769 accept_count = SummaryStats().accept_count
770 accept_bytes = SummaryStats().accept_bytes
776 sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
777 Logger.log(["total",accept_count,accept_bytes])
779 if not Options["No-Action"] and not Options["Trainee"]:
782 ################################################################################
785 global Options, Logger, Sections, Priorities
788 session = DBConn().session()
790 Arguments = [('a',"automatic","Process-New::Options::Automatic"),
791 ('b',"no-binaries","Process-New::Options::No-Binaries"),
792 ('c',"comments","Process-New::Options::Comments"),
793 ('h',"help","Process-New::Options::Help"),
794 ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
795 ('t',"trainee","Process-New::Options::Trainee"),
796 ('q','queue','Process-New::Options::Queue', 'HasArg'),
797 ('n',"no-action","Process-New::Options::No-Action")]
799 changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
801 for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
802 if not cnf.has_key("Process-New::Options::%s" % (i)):
803 cnf["Process-New::Options::%s" % (i)] = ""
805 queue_name = cnf.get('Process-New::Options::Queue', 'new')
806 new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
807 if len(changes_files) == 0:
808 uploads = new_queue.uploads
810 uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
811 .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
813 Options = cnf.subtree("Process-New::Options")
818 if not Options["No-Action"]:
820 Logger = daklog.Logger("process-new")
821 except CantOpenError as e:
822 Options["Trainee"] = "True"
824 Sections = Section_Completer(session)
825 Priorities = Priority_Completer(session)
826 readline.parse_and_bind("tab: complete")
829 sys.stderr.write("Sorting changes...\n")
830 uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"])
832 if Options["Comments"]:
833 show_new_comments(uploads, session)
835 for upload in uploads:
836 do_pkg (upload, session)
840 ################################################################################
842 if __name__ == '__main__':