import copy, errno, os, readline, stat, sys, time
import apt_pkg, apt_inst
-import dak.lib.database, examine_package, dak.lib.queue, dak.lib.logging, dak.lib.utils
+import examine_package
+import daklib.database
+import daklib.logging
+import daklib.queue
+import daklib.utils
# Globals
Cnf = None
source_package = files[file]["source package"]
if not Upload.pkg.changes["architecture"].has_key("source") \
and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
- source_epochless_version = dak.lib.utils.re_no_epoch.sub('', source_version)
+ source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
+ found = 0
+ for q in ["Accepted", "Embargoed", "Unembargoed"]:
+ if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
+ found = 1
+ if not found:
reject("no source found for %s %s (%s)." % (source_package, source_version, file))
# Version and file overwrite checks
elif files[file]["type"] == "dsc":
reject(Upload.check_source_against_db(file))
(reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
- reject(reject_msg)
+ reject(reject_msg, "")
- if reject_message:
+ if reject_message.find("Rejected") != -1:
answer = "XXX"
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
prompt = "[R]eject, Skip, Quit ?"
while prompt.find(answer) == -1:
- answer = dak.lib.utils.our_raw_input(prompt)
- m = dak.lib.queue.re_default_answer.match(prompt)
+ answer = daklib.utils.our_raw_input(prompt)
+ m = daklib.queue.re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
################################################################################
-def determine_new (changes, files):
- new = {}
-
- # Build up a list of potentially new things
- for file in files.keys():
- f = files[file]
- # Skip byhand elements
- if f["type"] == "byhand":
- continue
- pkg = f["package"]
- priority = f["priority"]
- section = f["section"]
- # FIXME: unhardcode
- if section == "non-US/main":
- section = "non-US"
- type = get_type(f)
- component = f["component"]
-
- if type == "dsc":
- priority = "source"
- if not new.has_key(pkg):
- new[pkg] = {}
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"] = []
- else:
- old_type = new[pkg]["type"]
- if old_type != type:
- # source gets trumped by deb or udeb
- if old_type == "dsc":
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"].append(file)
- if f.has_key("othercomponents"):
- new[pkg]["othercomponents"] = f["othercomponents"]
-
- for suite in changes["suite"].keys():
- suite_id = dak.lib.database.get_suite_id(suite)
- for pkg in new.keys():
- component_id = dak.lib.database.get_component_id(new[pkg]["component"])
- type_id = dak.lib.database.get_override_type_id(new[pkg]["type"])
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
- ql = q.getresult()
- if ql:
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"]
- del new[pkg]
-
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!"
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!"
- for pkg in new.keys():
- if new[pkg].has_key("othercomponents"):
- print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
-
- return new
-
-################################################################################
-
def indiv_sg_compare (a, b):
"""Sort by source name, source, version, 'have source', and
finally by filename."""
################################################################################
-def check_valid (new):
- for pkg in new.keys():
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
- type = new[pkg]["type"]
- new[pkg]["section id"] = dak.lib.database.get_section_id(section)
- new[pkg]["priority id"] = dak.lib.database.get_priority_id(new[pkg]["priority"])
- # Sanity checks
- if (section == "debian-installer" and type != "udeb") or \
- (section != "debian-installer" and type == "udeb"):
- new[pkg]["section id"] = -1
- if (priority == "source" and type != "dsc") or \
- (priority != "source" and type == "dsc"):
- new[pkg]["priority id"] = -1
-
-################################################################################
-
def print_new (new, indexed, file=sys.stdout):
- check_valid(new)
+ daklib.queue.check_valid(new)
broken = 0
index = 0
for pkg in new.keys():
################################################################################
-def get_type (f):
- # Determine the type
- if f.has_key("dbtype"):
- type = f["dbtype"]
- elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
- type = "dsc"
- else:
- dak.lib.utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
-
- # Validate the override type
- type_id = dak.lib.database.get_override_type_id(type)
- if type_id == -1:
- dak.lib.utils.fubar("invalid type (%s) for new. Say wha?" % (type))
-
- return type
-
-################################################################################
-
def index_range (index):
if index == 1:
return "1"
def edit_new (new):
# Write the current data to a temporary file
- temp_filename = dak.lib.utils.temp_filename()
- temp_file = dak.lib.utils.open_file(temp_filename, 'w')
+ temp_filename = daklib.utils.temp_filename()
+ temp_file = daklib.utils.open_file(temp_filename, 'w')
print_new (new, 0, temp_file)
temp_file.close()
# Spawn an editor on that file
editor = os.environ.get("EDITOR","vi")
result = os.system("%s %s" % (editor, temp_filename))
if result != 0:
- dak.lib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
+ daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
# Read the edited data back in
- temp_file = dak.lib.utils.open_file(temp_filename)
+ temp_file = daklib.utils.open_file(temp_filename)
lines = temp_file.readlines()
temp_file.close()
os.unlink(temp_filename)
s[len(s):3] = [None] * (3-len(s))
(pkg, priority, section) = s[:3]
if not new.has_key(pkg):
- dak.lib.utils.warn("Ignoring unknown package '%s'" % (pkg))
+ daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
else:
# Strip off any invalid markers, print_new will readd them.
if section.endswith("[!]"):
edit_priority = edit_section = 0
while prompt.find(answer) == -1:
- answer = dak.lib.utils.our_raw_input(prompt)
- m = dak.lib.queue.re_default_answer.match(prompt)
+ answer = daklib.utils.our_raw_input(prompt)
+ m = daklib.queue.re_default_answer.match(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
readline.set_completer(Priorities.complete)
got_priority = 0
while not got_priority:
- new_priority = dak.lib.utils.our_raw_input("New priority: ").strip()
+ new_priority = daklib.utils.our_raw_input("New priority: ").strip()
if new_priority not in Priorities.priorities:
print "E: '%s' is not a valid priority, try again." % (new_priority)
else:
readline.set_completer(Sections.complete)
got_section = 0
while not got_section:
- new_section = dak.lib.utils.our_raw_input("New section: ").strip()
+ new_section = daklib.utils.our_raw_input("New section: ").strip()
if new_section not in Sections.sections:
print "E: '%s' is not a valid section, try again." % (new_section)
else:
got_answer = 0
while not got_answer:
- answer = dak.lib.utils.our_raw_input(prompt)
- if not dak.lib.utils.str_isnum(answer):
+ answer = daklib.utils.our_raw_input(prompt)
+ if not answer.isdigit():
answer = answer[:1].upper()
if answer == "E" or answer == "D":
got_answer = 1
- elif dak.lib.queue.re_isanum.match (answer):
+ elif daklib.queue.re_isanum.match (answer):
answer = int(answer)
if (answer < 1) or (answer > index):
print "%s is not a valid index (%s). Please retry." % (answer, index_range(index))
def edit_note(note):
# Write the current data to a temporary file
- temp_filename = dak.lib.utils.temp_filename()
- temp_file = dak.lib.utils.open_file(temp_filename, 'w')
+ temp_filename = daklib.utils.temp_filename()
+ temp_file = daklib.utils.open_file(temp_filename, 'w')
temp_file.write(note)
temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
- temp_file = dak.lib.utils.open_file(temp_filename)
+ temp_file = daklib.utils.open_file(temp_filename)
note = temp_file.read().rstrip()
temp_file.close()
print "Note:"
- print dak.lib.utils.prefix_multi_line_string(note," ")
+ print daklib.utils.prefix_multi_line_string(note," ")
prompt = "[D]one, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
- answer = dak.lib.utils.our_raw_input(prompt)
- m = dak.lib.queue.re_default_answer.search(prompt)
+ answer = daklib.utils.our_raw_input(prompt)
+ m = daklib.queue.re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
sys.stdout = stdout_fd
except IOError, e:
if errno.errorcode[e.errno] == 'EPIPE':
- dak.lib.utils.warn("[examine_package] Caught EPIPE; skipping.")
+ daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
pass
else:
raise
except KeyboardInterrupt:
- dak.lib.utils.warn("[examine_package] Caught C-c; skipping.")
+ daklib.utils.warn("[examine_package] Caught C-c; skipping.")
pass
################################################################################
summary = ""
for file in files.keys():
if files[file]["type"] == "deb":
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(dak.lib.utils.open_file(file)))
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
summary += "\n"
summary += "Package: %s\n" % (control.Find("Package"))
summary += "Description: %s\n" % (control.Find("Description"))
Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
- bxa_mail = dak.lib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
- dak.lib.utils.send_mail(bxa_mail)
+ bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
+ daklib.utils.send_mail(bxa_mail)
################################################################################
projectB.query("BEGIN WORK")
for suite in changes["suite"].keys():
- suite_id = dak.lib.database.get_suite_id(suite)
+ suite_id = daklib.database.get_suite_id(suite)
for pkg in new.keys():
- component_id = dak.lib.database.get_component_id(new[pkg]["component"])
- type_id = dak.lib.database.get_override_type_id(new[pkg]["type"])
+ component_id = daklib.database.get_component_id(new[pkg]["component"])
+ type_id = daklib.database.get_override_type_id(new[pkg]["type"])
priority_id = new[pkg]["priority id"]
section_id = new[pkg]["section id"]
projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
def prod_maintainer ():
# Here we prepare an editor and get them ready to prod...
- temp_filename = dak.lib.utils.temp_filename()
+ temp_filename = daklib.utils.temp_filename()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
- file = dak.lib.utils.open_file(temp_filename)
+ file = daklib.utils.open_file(temp_filename)
prod_message = "".join(file.readlines())
file.close()
print "Prod message:"
- print dak.lib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
+ print daklib.utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
prompt = "[P]rod, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
- answer = dak.lib.utils.our_raw_input(prompt)
- m = dak.lib.queue.re_default_answer.search(prompt)
+ answer = daklib.utils.our_raw_input(prompt)
+ m = daklib.queue.re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
elif answer == 'Q':
sys.exit(0)
# Otherwise, do the proding...
- user_email_address = dak.lib.utils.whoami() + " <%s>" % (
+ user_email_address = daklib.utils.whoami() + " <%s>" % (
Cnf["Dinstall::MyAdminAddress"])
Subst = Upload.Subst
Subst["__PROD_MESSAGE__"] = prod_message
Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
- prod_mail_message = dak.lib.utils.TemplateSubst(
+ prod_mail_message = daklib.utils.TemplateSubst(
Subst,Cnf["Dir::Templates"]+"/process-new.prod")
# Send the prod mail if appropriate
if not Cnf["Dinstall::Options::No-Mail"]:
- dak.lib.utils.send_mail(prod_mail_message)
+ daklib.utils.send_mail(prod_mail_message)
print "Sent proding message"
for suite in changes["suite"].keys():
override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
if override:
+ (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1,
+ daklib.database.get_suite_id(override) == -1)
+ if olderr or newerr:
+ (oinv, newinv) = ("", "")
+ if olderr: oinv = "invalid "
+ if newerr: ninv = "invalid "
+ print "warning: overriding %ssuite %s to %ssuite %s" % (
+ oinv, suite, ninv, override)
del changes["suite"][suite]
changes["suite"][override] = 1
# Validate suites
for suite in changes["suite"].keys():
- suite_id = dak.lib.database.get_suite_id(suite)
+ suite_id = daklib.database.get_suite_id(suite)
if suite_id == -1:
- dak.lib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
+ daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
# The main NEW processing loop
done = 0
while not done:
# Find out what's new
- new = determine_new(changes, files)
+ new = daklib.queue.determine_new(changes, files, projectB)
if not new:
break
prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
while prompt.find(answer) == -1:
- answer = dak.lib.utils.our_raw_input(prompt)
- m = dak.lib.queue.re_default_answer.search(prompt)
+ answer = daklib.utils.our_raw_input(prompt)
+ m = daklib.queue.re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
elif answer == 'P':
prod_maintainer()
elif answer == 'R':
- confirm = dak.lib.utils.our_raw_input("Really clear note (y/N)? ").lower()
+ confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
if confirm == "y":
del changes["process-new note"]
elif answer == 'S':
def init():
global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
- Cnf = dak.lib.utils.get_conf()
+ Cnf = daklib.utils.get_conf()
Arguments = [('a',"automatic","Process-New::Options::Automatic"),
('h',"help","Process-New::Options::Help"),
+ ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('n',"no-action","Process-New::Options::No-Action")]
- for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
if not Cnf.has_key("Process-New::Options::%s" % (i)):
Cnf["Process-New::Options::%s" % (i)] = ""
if Options["Help"]:
usage()
- Upload = dak.lib.queue.Upload(Cnf)
+ Upload = daklib.queue.Upload(Cnf)
if not Options["No-Action"]:
- Logger = Upload.Logger = dak.lib.logging.Logger(Cnf, "process-new")
+ Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
projectB = Upload.projectB
prompt = "Manual reject, [S]kip, Quit ?"
while prompt.find(answer) == -1:
- answer = dak.lib.utils.our_raw_input(prompt)
- m = dak.lib.queue.re_default_answer.search(prompt)
+ answer = daklib.utils.our_raw_input(prompt)
+ m = daklib.queue.re_default_answer.search(prompt)
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
################################################################################
+def get_accept_lock():
+ retry = 0
+ while retry < 10:
+ try:
+ lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ retry = 10
+ except OSError, e:
+ if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
+ retry += 1
+ if (retry >= 10):
+ daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
+ else:
+ print("Unable to get accepted lock (try %d of 10)" % retry)
+ time.sleep(60)
+ else:
+ raise
+
+def move_to_dir (dest, perms=0660, changesperms=0664):
+ daklib.utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
+ file_keys = Upload.pkg.files.keys()
+ for file in file_keys:
+ daklib.utils.move (file, dest, perms=perms)
+
def do_accept():
print "ACCEPT"
if not Options["No-Action"]:
- retry = 0
- while retry < 10:
- try:
- lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- retry = 10
- except OSError, e:
- if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
- retry += 1
- if (retry >= 10):
- dak.lib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
- else:
- print("Unable to get accepted lock (try %d of 10)" % retry)
- time.sleep(60)
- else:
- raise
+ get_accept_lock()
(summary, short_summary) = Upload.build_summaries()
- Upload.accept(summary, short_summary)
+ if Cnf.FindB("Dinstall::SecurityQueueHandling"):
+ Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
+ move_to_dir(Cnf["Dir::Queue::Embargoed"])
+ Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
+ # Check for override disparities
+ Upload.Subst["__SUMMARY__"] = summary
+ else:
+ Upload.accept(summary, short_summary)
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
os.unlink(Cnf["Process-New::AcceptedLockFile"])
sets = "set"
if accept_count > 1:
sets = "sets"
- sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, dak.lib.utils.size_type(int(accept_bytes))))
+ sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
Logger.log(["total",accept_count,accept_bytes])
if not Options["No-Action"]:
################################################################################
+def do_comments(dir, opref, npref, line, fn):
+ for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
+ lines = open("%s/%s" % (dir, comm)).readlines()
+ if len(lines) == 0 or lines[0] != line + "\n": continue
+ changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
+ and x.endswith(".changes") ]
+ changes_files = sort_changes(changes_files)
+ for f in changes_files:
+ f = daklib.utils.validate_changes_file_arg(f, 0)
+ if not f: continue
+ print "\n" + f
+ fn(f, "".join(lines[1:]))
+
+ if opref != npref and not Options["No-Action"]:
+ newcomm = npref + comm[len(opref):]
+ os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
+
+################################################################################
+
+def comment_accept(changes_file, comments):
+ Upload.pkg.changes_file = changes_file
+ Upload.init_vars()
+ Upload.update_vars()
+ Upload.update_subst()
+ files = Upload.pkg.files
+
+ if not recheck():
+ return # dak wants to REJECT, crap
+
+ (new, byhand) = check_status(files)
+ if not new and not byhand:
+ do_accept()
+
+################################################################################
+
+def comment_reject(changes_file, comments):
+ Upload.pkg.changes_file = changes_file
+ Upload.init_vars()
+ Upload.update_vars()
+ Upload.update_subst()
+ files = Upload.pkg.files
+
+ if not recheck():
+ pass # dak has its own reasons to reject as well, which is fine
+
+ reject(comments)
+ print "REJECT\n" + reject_message,
+ if not Options["No-Action"]:
+ Upload.do_reject(0, reject_message)
+ os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+
+################################################################################
+
def main():
changes_files = init()
if len(changes_files) > 50:
# Kill me now? **FIXME**
Cnf["Dinstall::Options::No-Mail"] = ""
- bcc = "X-DAK: dak process-new\nX-Katie: this header is obsolete"
+ bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
if Cnf.has_key("Dinstall::Bcc"):
Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
else:
Upload.Subst["__BCC__"] = bcc
- for changes_file in changes_files:
- changes_file = dak.lib.utils.validate_changes_file_arg(changes_file, 0)
- if not changes_file:
- continue
- print "\n" + changes_file
- do_pkg (changes_file)
+ commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
+ if commentsdir:
+ if changes_files != []:
+ sys.stderr.write("Can't specify any changes files if working with comments-dir")
+ sys.exit(1)
+ do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
+ do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
+ else:
+ for changes_file in changes_files:
+ changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
+ if not changes_file:
+ continue
+ print "\n" + changes_file
+ do_pkg (changes_file)
end()
projects / dak.git / blobdiff