import pwd
import apt_pkg, apt_inst
import examine_package
+import subprocess
+import daklib.daksubprocess
+from sqlalchemy import or_
from daklib.dbconn import *
from daklib.queue import *
from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
from daklib.summarystats import SummaryStats
from daklib.config import Config
-from daklib.changesutils import *
+from daklib.policy import UploadCopy, PolicyQueueUploadHandler
# Globals
Options = None
################################################################################
################################################################################
-def recheck(upload, session):
-# STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session)
- if len(upload.rejects) > 0:
- answer = "XXX"
- if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
- answer = 'S'
-
- print "REJECT\n%s" % '\n'.join(upload.rejects)
- prompt = "[R]eject, Skip, Quit ?"
-
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt)
- m = re_default_answer.match(prompt)
- if answer == "":
- answer = m.group(1)
- answer = answer[:1].upper()
-
- if answer == 'R':
- upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
- upload.pkg.remove_known_changes(session=session)
- session.commit()
- return 0
- elif answer == 'S':
- return 0
- elif answer == 'Q':
- end()
- sys.exit(0)
-
- return 1
-
-################################################################################
-
class Section_Completer:
def __init__ (self, session):
self.sections = []
################################################################################
-def print_new (new, upload, indexed, file=sys.stdout):
- check_valid(new)
- broken = False
+def takenover_binaries(upload, missing, session):
+ rows = []
+ binaries = set([x.package for x in upload.binaries])
+ for m in missing:
+ if m['type'] != 'dsc':
+ binaries.discard(m['package'])
+ if binaries:
+ source = upload.binaries[0].source.source
+ suite = upload.target_suite.overridesuite or \
+ upload.target_suite.suite_name
+ suites = [s[0] for s in session.query(Suite.suite_name).filter \
+ (or_(Suite.suite_name == suite,
+ Suite.overridesuite == suite)).all()]
+ rows = session.query(DBSource.source, DBBinary.package).distinct(). \
+ filter(DBBinary.package.in_(binaries)). \
+ join(DBBinary.source). \
+ filter(DBSource.source != source). \
+ join(DBBinary.suites). \
+ filter(Suite.suite_name.in_(suites)). \
+ order_by(DBSource.source, DBBinary.package).all()
+ return rows
+
+################################################################################
+
+def print_new (upload, missing, indexed, session, file=sys.stdout):
+ check_valid(missing, session)
index = 0
- for pkg in new.keys():
+ for m in missing:
index += 1
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
- if new[pkg]["section id"] == -1:
- section += "[!]"
- broken = True
- if new[pkg]["priority id"] == -1:
- priority += "[!]"
- broken = True
+ if m['type'] != 'deb':
+ package = '{0}:{1}'.format(m['type'], m['package'])
+ else:
+ package = m['package']
+ section = m['section']
+ priority = m['priority']
+ included = "" if m['included'] else "NOT UPLOADED"
if indexed:
- line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
+ line = "(%s): %-20s %-20s %-20s %s" % (index, package, priority, section, included)
else:
- line = "%-20s %-20s %-20s" % (pkg, priority, section)
- line = line.strip()+'\n'
- file.write(line)
- notes = get_new_comments(upload.pkg.changes.get("source"))
+ line = "%-20s %-20s %-20s %s" % (package, priority, section, included)
+ line = line.strip()
+ if not m['valid']:
+ line = line + ' [!]'
+ print >>file, line
+ takenover = takenover_binaries(upload, missing, session)
+ if takenover:
+ print '\n\nBINARIES TAKEN OVER\n'
+ for t in takenover:
+ print '%s: %s' % (t[0], t[1])
+ notes = get_new_comments(upload.policy_queue, upload.changes.source)
for note in notes:
print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
% (note.author, note.version, note.notedate, note.comment)
print "-" * 72
- return broken, len(notes) > 0
+ return len(notes) > 0
################################################################################
################################################################################
################################################################################
-def edit_new (new, upload):
+def edit_new (overrides, upload, session):
# Write the current data to a temporary file
(fd, temp_filename) = utils.temp_filename()
temp_file = os.fdopen(fd, 'w')
- print_new (new, upload, indexed=0, file=temp_file)
+ print_new (upload, overrides, indexed=0, session=session, file=temp_file)
temp_file.close()
# Spawn an editor on that file
editor = os.environ.get("EDITOR","vi")
lines = temp_file.readlines()
temp_file.close()
os.unlink(temp_filename)
+
+ overrides_map = dict([ ((o['type'], o['package']), o) for o in overrides ])
+ new_overrides = []
# Parse the new data
for line in lines:
line = line.strip()
- if line == "":
+ if line == "" or line[0] == '#':
continue
s = line.split()
# Pad the list if necessary
s[len(s):3] = [None] * (3-len(s))
(pkg, priority, section) = s[:3]
- if not new.has_key(pkg):
+ if pkg.find(':') != -1:
+ type, pkg = pkg.split(':', 1)
+ else:
+ type = 'deb'
+ o = overrides_map.get((type, pkg), None)
+ if o is None:
utils.warn("Ignoring unknown package '%s'" % (pkg))
else:
- # Strip off any invalid markers, print_new will readd them.
- if section.endswith("[!]"):
- section = section[:-3]
- if priority.endswith("[!]"):
- priority = priority[:-3]
- for f in new[pkg]["files"]:
- upload.pkg.files[f]["section"] = section
- upload.pkg.files[f]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["priority"] = priority
+ if section.find('/') != -1:
+ component = section.split('/', 1)[0]
+ else:
+ component = 'main'
+ new_overrides.append(dict(
+ package=pkg,
+ type=type,
+ section=section,
+ component=component,
+ priority=priority,
+ included=o['included'],
+ ))
+ return new_overrides
################################################################################
def edit_index (new, upload, index):
+ package = new[index]['package']
priority = new[index]["priority"]
section = new[index]["section"]
ftype = new[index]["type"]
done = 0
while not done:
- print "\t".join([index, priority, section])
+ print "\t".join([package, priority, section])
answer = "XXX"
if ftype != "dsc":
# Reset the readline completer
readline.set_completer(None)
- for f in new[index]["files"]:
- upload.pkg.files[f]["section"] = section
- upload.pkg.files[f]["priority"] = priority
new[index]["priority"] = priority
new[index]["section"] = section
+ if section.find('/') != -1:
+ component = section.split('/', 1)[0]
+ else:
+ component = 'main'
+ new[index]['component'] = component
+
return new
################################################################################
print
done = 0
while not done:
- print_new (new, upload, indexed=1)
- new_index = {}
- index = 0
- for i in new.keys():
- index += 1
- new_index[index] = i
-
- prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
+ print_new (upload, new, indexed=1, session=session)
+ prompt = "edit override <n>, Editor, Done ? "
got_answer = 0
while not got_answer:
got_answer = 1
elif re_isanum.match (answer):
answer = int(answer)
- if (answer < 1) or (answer > index):
- print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
+ if answer < 1 or answer > len(new):
+ print "{0} is not a valid index. Please retry.".format(answer)
else:
got_answer = 1
if answer == 'E':
- edit_new(new, upload)
+ new = edit_new(new, upload, session)
elif answer == 'D':
done = 1
else:
- edit_index (new, upload, new_index[answer])
+ edit_index (new, upload, answer - 1)
return new
################################################################################
-def check_pkg (upload):
+def check_pkg (upload, upload_copy, session):
+ missing = []
save_stdout = sys.stdout
+ changes = os.path.join(upload_copy.directory, upload.changes.changesname)
+ suite_name = upload.target_suite.suite_name
+ handler = PolicyQueueUploadHandler(upload, session)
+ missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)]
+
+ less_cmd = ("less", "-R", "-")
+ less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE)
try:
- sys.stdout = os.popen("less -R -", 'w', 0)
- changes = utils.parse_changes (upload.pkg.changes_file)
- print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
- files = upload.pkg.files
- for f in files.keys():
- if files[f].has_key("new"):
- ftype = files[f]["type"]
- if ftype == "deb":
- print examine_package.check_deb(changes['distribution'], f)
- elif ftype == "dsc":
- print examine_package.check_dsc(changes['distribution'], f)
+ sys.stdout = less_process.stdin
+ print examine_package.display_changes(suite_name, changes)
+
+ source = upload.source
+ if source is not None:
+ source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename))
+ print examine_package.check_dsc(suite_name, source_file)
+
+ for binary in upload.binaries:
+ binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
+ examined = examine_package.check_deb(suite_name, binary_file)
+ # We always need to call check_deb to display package relations for every binary,
+ # but we print its output only if new overrides are being added.
+ if ("deb", binary.package) in missing:
+ print examined
+
print examine_package.output_package_relations()
+ less_process.stdin.close()
except IOError as e:
if e.errno == errno.EPIPE:
utils.warn("[examine_package] Caught EPIPE; skipping.")
else:
- sys.stdout = save_stdout
raise
except KeyboardInterrupt:
utils.warn("[examine_package] Caught C-c; skipping.")
- sys.stdout = save_stdout
+ finally:
+ less_process.wait()
+ sys.stdout = save_stdout
################################################################################
## FIXME: horribly Debian specific
-def do_bxa_notification(upload):
- files = upload.pkg.files
+def do_bxa_notification(new, upload, session):
+ cnf = Config()
+
+ new = set([ o['package'] for o in new if o['type'] == 'deb' ])
+ if len(new) == 0:
+ return
+
+ key = session.query(MetadataKey).filter_by(key='Description').one()
summary = ""
- for f in files.keys():
- if files[f]["type"] == "deb":
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
- summary += "\n"
- summary += "Package: %s\n" % (control.Find("Package"))
- summary += "Description: %s\n" % (control.Find("Description"))
- upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
- bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
+ for binary in upload.binaries:
+ if binary.package not in new:
+ continue
+ description = session.query(BinaryMetadata).filter_by(binary=binary, key=key).one().value
+ summary += "\n"
+ summary += "Package: {0}\n".format(binary.package)
+ summary += "Description: {0}\n".format(description)
+
+ subst = {
+ '__DISTRO__': cnf['Dinstall::MyDistribution'],
+ '__BCC__': 'X-DAK: dak process-new',
+ '__BINARY_DESCRIPTIONS__': summary,
+ }
+
+ bxa_mail = utils.TemplateSubst(subst,os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification"))
utils.send_mail(bxa_mail)
################################################################################
-def add_overrides (new, upload, session):
- changes = upload.pkg.changes
- files = upload.pkg.files
- srcpkg = changes.get("source")
-
- for suite in changes["suite"].keys():
- suite_id = get_suite(suite).suite_id
- for pkg in new.keys():
- component_id = get_component(new[pkg]["component"]).component_id
- type_id = get_override_type(new[pkg]["type"]).overridetype_id
- priority_id = new[pkg]["priority id"]
- section_id = new[pkg]["section id"]
- Logger.log(["%s (%s) overrides" % (pkg, srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
- session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
- { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
- for f in new[pkg]["files"]:
- if files[f].has_key("new"):
- del files[f]["new"]
- del new[pkg]
+def add_overrides (new_overrides, suite, session):
+ if suite.overridesuite is not None:
+ suite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
+
+ for override in new_overrides:
+ package = override['package']
+ priority = session.query(Priority).filter_by(priority=override['priority']).first()
+ section = session.query(Section).filter_by(section=override['section']).first()
+ component = get_mapped_component(override['component'], session)
+ overridetype = session.query(OverrideType).filter_by(overridetype=override['type']).one()
+
+ if priority is None:
+ raise Exception('Invalid priority {0} for package {1}'.format(priority, package))
+ if section is None:
+ raise Exception('Invalid section {0} for package {1}'.format(section, package))
+ if component is None:
+ raise Exception('Invalid component {0} for package {1}'.format(component, package))
+
+ o = Override(package=package, suite=suite, component=component, priority=priority, section=section, overridetype=overridetype)
+ session.add(o)
session.commit()
- if Config().FindB("Dinstall::BXANotify"):
- do_bxa_notification(upload)
+################################################################################
+
+def run_user_inspect_command(upload, upload_copy):
+ command = os.environ.get('DAK_INSPECT_UPLOAD')
+ if command is None:
+ return
+
+ directory = upload_copy.directory
+ if upload.source:
+ dsc = os.path.basename(upload.source.poolfile.filename)
+ else:
+ dsc = ''
+ changes = upload.changes.changesname
+
+ shell_command = command.format(
+ directory=directory,
+ dsc=dsc,
+ changes=changes,
+ )
+
+ daklib.daksubprocess.check_call(shell_command, shell=True)
################################################################################
-def do_new(upload, session):
- print "NEW\n"
- files = upload.pkg.files
- upload.check_files(not Options["No-Action"])
- changes = upload.pkg.changes
- cnf = Config()
+def get_reject_reason(reason=''):
+ """get reason for rejection
- # Check for a valid distribution
- upload.check_distributions()
+ @rtype: str
+ @return: string giving the reason for the rejection or C{None} if the
+ rejection should be cancelled
+ """
+ answer = 'E'
+ if Options['Automatic']:
+ answer = 'R'
+
+ while answer == 'E':
+ reason = utils.call_editor(reason)
+ print "Reject message:"
+ print utils.prefix_multi_line_string(reason, " ", include_blank_lines=1)
+ prompt = "[R]eject, Edit, Abandon, Quit ?"
+ answer = "XXX"
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
+ if answer == "":
+ answer = m.group(1)
+ answer = answer[:1].upper()
- # Make a copy of distribution we can happily trample on
- changes["suite"] = copy.copy(changes["distribution"])
+ if answer == 'Q':
+ sys.exit(0)
- # Try to get an included dsc
- dsc = None
- (status, _) = upload.load_dsc()
- if status:
- dsc = upload.pkg.dsc
+ if answer == 'R':
+ return reason
+ return None
+
+################################################################################
+
+def do_new(upload, upload_copy, handler, session):
+ cnf = Config()
+
+ run_user_inspect_command(upload, upload_copy)
# The main NEW processing loop
- done = 0
- new = {}
+ done = False
+ missing = []
while not done:
- # Find out what's new
- new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session, new=new)
+ queuedir = upload.policy_queue.path
+ byhand = upload.byhand
- if not new:
- break
+ missing = handler.missing_overrides(hints=missing)
+ broken = not check_valid(missing, session)
+
+ changesname = os.path.basename(upload.changes.changesname)
+
+ print
+ print changesname
+ print "-" * len(changesname)
+ print
+ print " Target: {0}".format(upload.target_suite.suite_name)
+ print " Changed-By: {0}".format(upload.changes.changedby)
+ print
+
+ #if len(byhand) == 0 and len(missing) == 0:
+ # break
+
+ if missing:
+ print "NEW\n"
answer = "XXX"
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
- (broken, note) = print_new(new, upload, indexed=0)
+ note = print_new(upload, missing, indexed=0, session=session)
prompt = ""
- if not broken and not note:
- prompt = "Add overrides, "
+ has_unprocessed_byhand = False
+ for f in byhand:
+ path = os.path.join(queuedir, f.filename)
+ if not f.processed and os.path.exists(path):
+ print "W: {0} still present; please process byhand components and try again".format(f.filename)
+ has_unprocessed_byhand = True
+
+ if not has_unprocessed_byhand and not broken and not note:
+ if len(missing) == 0:
+ prompt = "Accept, "
+ answer = 'A'
+ else:
+ prompt = "Add overrides, "
if broken:
print "W: [!] marked entries must be fixed before package can be processed."
if note:
continue
if answer == 'A' and not Options["Trainee"]:
- try:
- check_daily_lock()
- done = add_overrides (new, upload, session)
- new_accept(upload, Options["No-Action"], session)
- Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
- except CantGetLockError:
- print "Hello? Operator! Give me the number for 911!"
- print "Dinstall in the locked area, cant process packages, come back later"
+ add_overrides(missing, upload.target_suite, session)
+ if Config().find_b("Dinstall::BXANotify"):
+ do_bxa_notification(missing, upload, session)
+ handler.accept()
+ done = True
+ Logger.log(["NEW ACCEPT", upload.changes.changesname])
elif answer == 'C':
- check_pkg(upload)
+ check_pkg(upload, upload_copy, session)
elif answer == 'E' and not Options["Trainee"]:
- new = edit_overrides (new, upload, session)
+ missing = edit_overrides (missing, upload, session)
elif answer == 'M' and not Options["Trainee"]:
- aborted = upload.do_reject(manual=1,
- reject_message=Options["Manual-Reject"],
- notes=get_new_comments(changes.get("source", ""), session=session))
- if not aborted:
- upload.pkg.remove_known_changes(session=session)
- session.commit()
- Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
- done = 1
+ reason = Options.get('Manual-Reject', '') + "\n"
+ reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)])
+ reason = get_reject_reason(reason)
+ if reason is not None:
+ Logger.log(["NEW REJECT", upload.changes.changesname])
+ handler.reject(reason)
+ done = True
elif answer == 'N':
- edit_note(get_new_comments(changes.get("source", ""), session=session),
- upload, session, bool(Options["Trainee"]))
+ if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
+ upload, session, bool(Options["Trainee"])) == 0:
+ end()
+ sys.exit(0)
elif answer == 'P' and not Options["Trainee"]:
- prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
- upload)
- Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
+ if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session),
+ upload) == 0:
+ end()
+ sys.exit(0)
+ Logger.log(["NEW PROD", upload.changes.changesname])
elif answer == 'R' and not Options["Trainee"]:
confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
if confirm == "y":
- for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
+ for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session):
session.delete(c)
session.commit()
elif answer == 'O' and not Options["Trainee"]:
confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
if confirm == "y":
- for c in get_new_comments(changes.get("source", ""), session=session):
+ for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session):
session.delete(c)
session.commit()
elif answer == 'S':
- done = 1
+ done = True
elif answer == 'Q':
end()
sys.exit(0)
+ if handler.get_action():
+ print "PENDING %s\n" % handler.get_action()
+
################################################################################
################################################################################
################################################################################
-h, --help show this help and exit.
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
+ -q, --queue=QUEUE operate on a different queue
-t, --trainee FTP Trainee mode
- -V, --version display the version number and exit"""
- sys.exit(exit_code)
+ -V, --version display the version number and exit
-################################################################################
+ENVIRONMENT VARIABLES
-def do_byhand(upload, session):
- done = 0
- while not done:
- files = upload.pkg.files
- will_install = True
- byhand = []
-
- for f in files.keys():
- if files[f]["section"] == "byhand":
- if os.path.exists(f):
- print "W: %s still present; please process byhand components and try again." % (f)
- will_install = False
- else:
- byhand.append(f)
+ DAK_INSPECT_UPLOAD: shell command to run to inspect a package
+ The command is automatically run in a shell when an upload
+ is checked. The following substitutions are available:
- answer = "XXXX"
- if Options["No-Action"]:
- answer = "S"
- if will_install:
- if Options["Automatic"] and not Options["No-Action"]:
- answer = 'A'
- prompt = "[A]ccept, Manual reject, Skip, Quit ?"
- else:
- prompt = "Manual reject, [S]kip, Quit ?"
+ {directory}: directory the upload is contained in
+ {dsc}: name of the included dsc or the empty string
+ {changes}: name of the changes file
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt)
- m = re_default_answer.search(prompt)
- if answer == "":
- answer = m.group(1)
- answer = answer[:1].upper()
+ Note that Python's 'format' method is used to format the command.
- if answer == 'A':
- dbchg = get_dbchange(upload.pkg.changes_file, session)
- if dbchg is None:
- print "Warning: cannot find changes file in database; can't process BYHAND"
- else:
- try:
- check_daily_lock()
- done = 1
- for b in byhand:
- # Find the file entry in the database
- found = False
- for f in dbchg.files:
- if f.filename == b:
- found = True
- f.processed = True
- break
-
- if not found:
- print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b
-
- session.commit()
- Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
- except CantGetLockError:
- print "Hello? Operator! Give me the number for 911!"
- print "Dinstall in the locked area, cant process packages, come back later"
- elif answer == 'M':
- aborted = upload.do_reject(manual=1,
- reject_message=Options["Manual-Reject"],
- notes=get_new_comments(changes.get("source", ""), session=session))
- if not aborted:
- upload.pkg.remove_known_changes(session=session)
- session.commit()
- Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
- done = 1
- elif answer == 'S':
- done = 1
- elif answer == 'Q':
- end()
- sys.exit(0)
+ Example: run mc in a tmux session to inspect the upload
-################################################################################
+ export DAK_INSPECT_UPLOAD='tmux new-session -d -s process-new 2>/dev/null; tmux new-window -n "{changes}" -t process-new:0 -k "cd {directory}; mc"'
-def check_daily_lock():
- """
- Raises CantGetLockError if the dinstall daily.lock exists.
- """
-
- cnf = Config()
- try:
- lockfile = cnf.get("Process-New::DinstallLockFile",
- os.path.join(cnf['Dir::Lock'], 'processnew.lock'))
+ and run
- os.open(lockfile,
- os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- except OSError as e:
- if e.errno == errno.EEXIST or e.errno == errno.EACCES:
- raise CantGetLockError
+ tmux attach -t process-new
- os.unlink(lockfile)
+ in a separate terminal session.
+"""
+ sys.exit(exit_code)
+################################################################################
@contextlib.contextmanager
def lock_package(package):
finally:
os.unlink(path)
-class clean_holding(object):
- def __init__(self,pkg):
- self.pkg = pkg
-
- def __enter__(self):
- pass
-
- def __exit__(self, type, value, traceback):
- h = Holding()
-
- for f in self.pkg.files.keys():
- if os.path.exists(os.path.join(h.holding_dir, f)):
- os.unlink(os.path.join(h.holding_dir, f))
-
-
-def do_pkg(changes_full_path, session):
- changes_dir = os.path.dirname(changes_full_path)
- changes_file = os.path.basename(changes_full_path)
-
- u = Upload()
- u.pkg.changes_file = changes_file
- (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
- u.load_changes(changes_file)
- u.pkg.directory = changes_dir
- u.update_subst()
- u.logger = Logger
- origchanges = os.path.abspath(u.pkg.changes_file)
-
+def do_pkg(upload, session):
# Try to get an included dsc
- dsc = None
- (status, _) = u.load_dsc()
- if status:
- dsc = u.pkg.dsc
+ dsc = upload.source
cnf = Config()
- bcc = "X-DAK: dak process-new"
- if cnf.has_key("Dinstall::Bcc"):
- u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
- else:
- u.Subst["__BCC__"] = bcc
-
- files = u.pkg.files
- u.check_distributions()
- for deb_filename, f in files.items():
- if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
- u.binary_file_checks(deb_filename, session)
- u.check_binary_against_db(deb_filename, session)
- else:
- u.source_file_checks(deb_filename, session)
- u.check_source_against_db(deb_filename, session)
+ group = cnf.get('Dinstall::UnprivGroup') or None
- u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
+ #bcc = "X-DAK: dak process-new"
+ #if cnf.has_key("Dinstall::Bcc"):
+ # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ #else:
+ # u.Subst["__BCC__"] = bcc
try:
- with lock_package(u.pkg.changes["source"]):
- with clean_holding(u.pkg):
- if not recheck(u, session):
- return
-
- new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session)
- if byhand:
- do_byhand(u, session)
- elif new:
- do_new(u, session)
- else:
- try:
- check_daily_lock()
- new_accept(u, Options["No-Action"], session)
- except CantGetLockError:
- print "Hello? Operator! Give me the number for 911!"
- print "Dinstall in the locked area, cant process packages, come back later"
-
+ with lock_package(upload.changes.source):
+ with UploadCopy(upload, group=group) as upload_copy:
+ handler = PolicyQueueUploadHandler(upload, session)
+ if handler.get_action() is not None:
+ print "PENDING %s\n" % handler.get_action()
+ return
+
+ do_new(upload, upload_copy, handler, session)
except AlreadyLockedError as e:
print "Seems to be locked by %s already, skipping..." % (e)
-def show_new_comments(changes_files, session):
- sources = set()
+def show_new_comments(uploads, session):
+ sources = [ upload.changes.source for upload in uploads ]
+ if len(sources) == 0:
+ return
+
query = """SELECT package, version, comment, author
FROM new_comments
- WHERE package IN ('"""
-
- for changes in changes_files:
- sources.add(os.path.basename(changes).split("_")[0])
+ WHERE package IN :sources
+ ORDER BY package, version"""
- query += "%s') ORDER BY package, version" % "', '".join(sources)
- r = session.execute(query)
+ r = session.execute(query, params=dict(sources=tuple(sources)))
for i in r:
print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
- session.commit()
+ session.rollback()
+
+################################################################################
+
+def sort_uploads(new_queue, uploads, session, nobinaries=False):
+ sources = {}
+ sorteduploads = []
+ suitesrc = [s.source for s in session.query(DBSource.source). \
+ filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))]
+ comments = [p.package for p in session.query(NewComment.package). \
+ filter_by(trainee=False, policy_queue=new_queue).distinct()]
+ for upload in uploads:
+ source = upload.changes.source
+ if not source in sources:
+ sources[source] = []
+ sources[source].append({'upload': upload,
+ 'date': upload.changes.created,
+ 'stack': 1,
+ 'binary': True if source in suitesrc else False,
+ 'comments': True if source in comments else False})
+ for src in sources:
+ if len(sources[src]) > 1:
+ changes = sources[src]
+ firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date']
+ changes.sort(key=lambda item:item['date'])
+ for i in range (0, len(changes)):
+ changes[i]['date'] = firstseen
+ changes[i]['stack'] = i + 1
+ sorteduploads += sources[src]
+ if nobinaries:
+ sorteduploads = [u["upload"] for u in sorted(sorteduploads,
+ key=lambda k: (k["comments"], k["binary"],
+ k["date"], -k["stack"]))]
+ else:
+ sorteduploads = [u["upload"] for u in sorted(sorteduploads,
+ key=lambda k: (k["comments"], -k["binary"],
+ k["date"], -k["stack"]))]
+ return sorteduploads
################################################################################
('h',"help","Process-New::Options::Help"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('t',"trainee","Process-New::Options::Trainee"),
+ ('q','queue','Process-New::Options::Queue', 'HasArg'),
('n',"no-action","Process-New::Options::No-Action")]
+ changes_files = apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv)
+
for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
if not cnf.has_key("Process-New::Options::%s" % (i)):
cnf["Process-New::Options::%s" % (i)] = ""
- changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
+ queue_name = cnf.get('Process-New::Options::Queue', 'new')
+ new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one()
if len(changes_files) == 0:
- new_queue = get_policy_queue('new', session );
- changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
+ uploads = new_queue.uploads
else:
- changes_paths = [ os.path.abspath(j) for j in changes_files ]
+ uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
+ .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
- Options = cnf.SubTree("Process-New::Options")
+ Options = cnf.subtree("Process-New::Options")
if Options["Help"]:
usage()
Priorities = Priority_Completer(session)
readline.parse_and_bind("tab: complete")
- if len(changes_paths) > 1:
+ if len(uploads) > 1:
sys.stderr.write("Sorting changes...\n")
- changes_files = sort_changes(changes_paths, session, Options["No-Binaries"])
+ uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"])
if Options["Comments"]:
- show_new_comments(changes_files, session)
+ show_new_comments(uploads, session)
else:
- for changes_file in changes_files:
- changes_file = utils.validate_changes_file_arg(changes_file, 0)
- if not changes_file:
- continue
- print "\n" + os.path.basename(changes_file)
-
- do_pkg (changes_file, session)
+ for upload in uploads:
+ do_pkg (upload, session)
end()
projects / dak.git / blobdiff