from daklib.queue import *
from daklib import daklog
from daklib import utils
-from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
+from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
from daklib.summarystats import SummaryStats
from daklib.config import Config
+from daklib.changesutils import *
# Globals
Options = None
################################################################################
def recheck(upload, session):
- upload.recheck()
+# STU: I'm not sure, but I don't thin kthis is necessary any longer: upload.recheck(session)
if len(upload.rejects) > 0:
answer = "XXX"
if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
if answer == 'R':
upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
- os.unlink(upload.pkg.changes_file[:-8]+".dak")
+ upload.pkg.remove_known_changes(session=session)
+ session.commit()
return 0
elif answer == 'S':
return 0
################################################################################
-def indiv_sg_compare (a, b):
- """Sort by source name, source, version, 'have source', and
- finally by filename."""
- # Sort by source version
- q = apt_pkg.VersionCompare(a["version"], b["version"])
- if q:
- return -q
-
- # Sort by 'have source'
- a_has_source = a["architecture"].get("source")
- b_has_source = b["architecture"].get("source")
- if a_has_source and not b_has_source:
- return -1
- elif b_has_source and not a_has_source:
- return 1
-
- return cmp(a["filename"], b["filename"])
-
-############################################################
-
-def sg_compare (a, b):
- a = a[1]
- b = b[1]
- """Sort by have note, source already in database and time of oldest upload."""
- # Sort by have note
- a_note_state = a["note_state"]
- b_note_state = b["note_state"]
- if a_note_state < b_note_state:
- return -1
- elif a_note_state > b_note_state:
- return 1
- # Sort by source already in database (descending)
- source_in_database = cmp(a["source_in_database"], b["source_in_database"])
- if source_in_database:
- return -source_in_database
-
- # Sort by time of oldest upload
- return cmp(a["oldest"], b["oldest"])
-
-def sort_changes(changes_files, session):
- """Sort into source groups, then sort each source group by version,
- have source, filename. Finally, sort the source groups by have
- note, time of oldest upload of each source upload."""
- if len(changes_files) == 1:
- return changes_files
-
- sorted_list = []
- cache = {}
- # Read in all the .changes files
- for filename in changes_files:
- u = Upload()
- try:
- u.pkg.load_dot_dak(filename)
- u.update_subst()
- cache[filename] = copy.copy(u.pkg.changes)
- cache[filename]["filename"] = filename
- except:
- sorted_list.append(filename)
- break
- # Divide the .changes into per-source groups
- per_source = {}
- for filename in cache.keys():
- source = cache[filename]["source"]
- if not per_source.has_key(source):
- per_source[source] = {}
- per_source[source]["list"] = []
- per_source[source]["list"].append(cache[filename])
- # Determine oldest time and have note status for each source group
- for source in per_source.keys():
- q = session.query(DBSource).filter_by(source = source).all()
- per_source[source]["source_in_database"] = len(q)>0
- source_list = per_source[source]["list"]
- first = source_list[0]
- oldest = os.stat(first["filename"])[stat.ST_MTIME]
- have_note = 0
- for d in per_source[source]["list"]:
- mtime = os.stat(d["filename"])[stat.ST_MTIME]
- if mtime < oldest:
- oldest = mtime
- have_note += has_new_comment(d["source"], d["version"], session)
- per_source[source]["oldest"] = oldest
- if not have_note:
- per_source[source]["note_state"] = 0; # none
- elif have_note < len(source_list):
- per_source[source]["note_state"] = 1; # some
- else:
- per_source[source]["note_state"] = 2; # all
- per_source[source]["list"].sort(indiv_sg_compare)
- per_source_items = per_source.items()
- per_source_items.sort(sg_compare)
- for i in per_source_items:
- for j in i[1]["list"]:
- sorted_list.append(j["filename"])
- return sorted_list
-
-################################################################################
-
class Section_Completer:
def __init__ (self, session):
self.sections = []
return new
-################################################################################
-
-def edit_note(note, upload, session):
- # Write the current data to a temporary file
- (fd, temp_filename) = utils.temp_filename()
- editor = os.environ.get("EDITOR","vi")
- answer = 'E'
- while answer == 'E':
- os.system("%s %s" % (editor, temp_filename))
- temp_file = utils.open_file(temp_filename)
- newnote = temp_file.read().rstrip()
- temp_file.close()
- print "New Note:"
- print utils.prefix_multi_line_string(newnote," ")
- prompt = "[D]one, Edit, Abandon, Quit ?"
- answer = "XXX"
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt)
- m = re_default_answer.search(prompt)
- if answer == "":
- answer = m.group(1)
- answer = answer[:1].upper()
- os.unlink(temp_filename)
- if answer == 'A':
- return
- elif answer == 'Q':
- end()
- sys.exit(0)
-
- comment = NewComment()
- comment.package = upload.pkg.changes["source"]
- comment.version = upload.pkg.changes["version"]
- comment.comment = newnote
- comment.author = utils.whoami()
- comment.trainee = bool(Options["Trainee"])
- session.add(comment)
- session.commit()
################################################################################
def check_pkg (upload):
+ save_stdout = sys.stdout
try:
- less_fd = os.popen("less -R -", 'w', 0)
- stdout_fd = sys.stdout
- try:
- sys.stdout = less_fd
- changes = utils.parse_changes (upload.pkg.changes_file)
- examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
- files = upload.pkg.files
- for f in files.keys():
- if files[f].has_key("new"):
- ftype = files[f]["type"]
- if ftype == "deb":
- examine_package.check_deb(changes['distribution'], f)
- elif ftype == "dsc":
- examine_package.check_dsc(changes['distribution'], f)
- finally:
- examine_package.output_package_relations()
- sys.stdout = stdout_fd
- except IOError, e:
+ sys.stdout = os.popen("less -R -", 'w', 0)
+ changes = utils.parse_changes (upload.pkg.changes_file)
+ print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+ files = upload.pkg.files
+ for f in files.keys():
+ if files[f].has_key("new"):
+ ftype = files[f]["type"]
+ if ftype == "deb":
+ print examine_package.check_deb(changes['distribution'], f)
+ elif ftype == "dsc":
+ print examine_package.check_dsc(changes['distribution'], f)
+ print examine_package.output_package_relations()
+ except IOError as e:
if e.errno == errno.EPIPE:
utils.warn("[examine_package] Caught EPIPE; skipping.")
- pass
else:
+ sys.stdout = save_stdout
raise
except KeyboardInterrupt:
utils.warn("[examine_package] Caught C-c; skipping.")
- pass
+ sys.stdout = save_stdout
################################################################################
type_id = get_override_type(new[pkg]["type"]).overridetype_id
priority_id = new[pkg]["priority id"]
section_id = new[pkg]["section id"]
- Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
+ Logger.log(["%s (%s) overrides" % (pkg, srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
{ 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
for f in new[pkg]["files"]:
################################################################################
-def prod_maintainer (note, upload):
- cnf = Config()
- # Here we prepare an editor and get them ready to prod...
- (fd, temp_filename) = utils.temp_filename()
- temp_file = os.fdopen(fd, 'w')
- if len(note) > 0:
- for line in note:
- temp_file.write(line)
- temp_file.close()
- editor = os.environ.get("EDITOR","vi")
- answer = 'E'
- while answer == 'E':
- os.system("%s %s" % (editor, temp_filename))
- temp_fh = utils.open_file(temp_filename)
- prod_message = "".join(temp_fh.readlines())
- temp_fh.close()
- print "Prod message:"
- print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
- prompt = "[P]rod, Edit, Abandon, Quit ?"
- answer = "XXX"
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt)
- m = re_default_answer.search(prompt)
- if answer == "":
- answer = m.group(1)
- answer = answer[:1].upper()
- os.unlink(temp_filename)
- if answer == 'A':
- return
- elif answer == 'Q':
- end()
- sys.exit(0)
- # Otherwise, do the proding...
- user_email_address = utils.whoami() + " <%s>" % (
- cnf["Dinstall::MyAdminAddress"])
-
- Subst = upload.Subst
-
- Subst["__FROM_ADDRESS__"] = user_email_address
- Subst["__PROD_MESSAGE__"] = prod_message
- Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
-
- prod_mail_message = utils.TemplateSubst(
- Subst,cnf["Dir::Templates"]+"/process-new.prod")
-
- # Send the prod mail if appropriate
- if not cnf["Dinstall::Options::No-Mail"]:
- utils.send_mail(prod_mail_message)
-
- print "Sent proding message"
-
-################################################################################
-
def do_new(upload, session):
print "NEW\n"
files = upload.pkg.files
+ upload.check_files(not Options["No-Action"])
changes = upload.pkg.changes
cnf = Config()
+ # Check for a valid distribution
+ upload.check_distributions()
+
# Make a copy of distribution we can happily trample on
changes["suite"] = copy.copy(changes["distribution"])
- # Fix up the list of target suites
- for suite in changes["suite"].keys():
- override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
- if override:
- (olderr, newerr) = (get_suite(suite, session) == None,
- get_suite(override, session) == None)
- if olderr or newerr:
- (oinv, newinv) = ("", "")
- if olderr: oinv = "invalid "
- if newerr: ninv = "invalid "
- print "warning: overriding %ssuite %s to %ssuite %s" % (
- oinv, suite, ninv, override)
- del changes["suite"][suite]
- changes["suite"][override] = 1
- # Validate suites
- for suite in changes["suite"].keys():
- if get_suite(suite, session) is None:
- utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
+ # Try to get an included dsc
+ dsc = None
+ (status, _) = upload.load_dsc()
+ if status:
+ dsc = upload.pkg.dsc
# The main NEW processing loop
done = 0
+ new = {}
while not done:
# Find out what's new
- new = determine_new(changes, files)
+ new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session, new=new)
if not new:
break
try:
check_daily_lock()
done = add_overrides (new, upload, session)
+ new_accept(upload, Options["No-Action"], session)
Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
except CantGetLockError:
print "Hello? Operator! Give me the number for 911!"
elif answer == 'E' and not Options["Trainee"]:
new = edit_overrides (new, upload, session)
elif answer == 'M' and not Options["Trainee"]:
- upload.pkg.remove_known_changes()
aborted = upload.do_reject(manual=1,
reject_message=Options["Manual-Reject"],
- note=get_new_comments(changes.get("source", ""), session=session))
+ notes=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
+ upload.pkg.remove_known_changes(session=session)
+ session.commit()
Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
- os.unlink(upload.pkg.changes_file[:-8]+".dak")
done = 1
elif answer == 'N':
edit_note(get_new_comments(changes.get("source", ""), session=session),
- upload, session)
+ upload, session, bool(Options["Trainee"]))
elif answer == 'P' and not Options["Trainee"]:
prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
upload)
def usage (exit_code=0):
print """Usage: dak process-new [OPTION]... [CHANGES]...
-a, --automatic automatic run
+ -b, --no-binaries do not sort binary-NEW packages first
+ -c, --comments show NEW comments
-h, --help show this help and exit.
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
done = 0
while not done:
files = upload.pkg.files
- will_install = 1
+ will_install = True
byhand = []
for f in files.keys():
- if files[f]["type"] == "byhand":
+ if files[f]["section"] == "byhand":
if os.path.exists(f):
print "W: %s still present; please process byhand components and try again." % (f)
- will_install = 0
+ will_install = False
else:
byhand.append(f)
answer = answer[:1].upper()
if answer == 'A':
- try:
- check_daily_lock()
- done = 1
- for f in byhand:
- del files[f]
- Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
- except CantGetLockError:
- print "Hello? Operator! Give me the number for 911!"
- print "Dinstall in the locked area, cant process packages, come back later"
+ dbchg = get_dbchange(upload.pkg.changes_file, session)
+ if dbchg is None:
+ print "Warning: cannot find changes file in database; can't process BYHAND"
+ else:
+ try:
+ check_daily_lock()
+ done = 1
+ for b in byhand:
+ # Find the file entry in the database
+ found = False
+ for f in dbchg.files:
+ if f.filename == b:
+ found = True
+ f.processed = True
+ break
+
+ if not found:
+ print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b
+
+ session.commit()
+ Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'M':
- Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
- upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
- os.unlink(upload.pkg.changes_file[:-8]+".dak")
- done = 1
+ aborted = upload.do_reject(manual=1,
+ reject_message=Options["Manual-Reject"],
+ notes=get_new_comments(changes.get("source", ""), session=session))
+ if not aborted:
+ upload.pkg.remove_known_changes(session=session)
+ session.commit()
+ Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
+ done = 1
elif answer == 'S':
done = 1
elif answer == 'Q':
cnf = Config()
try:
- os.open(cnf["Process-New::DinstallLockFile"],
+ lockfile = cnf.get("Process-New::DinstallLockFile",
+ os.path.join(cnf['Dir::Lock'], 'processnew.lock'))
+
+ os.open(lockfile,
os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EEXIST or e.errno == errno.EACCES:
raise CantGetLockError
- os.unlink(cnf["Process-New::DinstallLockFile"])
+ os.unlink(lockfile)
@contextlib.contextmanager
@param package: source package name to lock
"""
- path = os.path.join(Config()["Process-New::LockDir"], package)
+ cnf = Config()
+
+ path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package)
+
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EEXIST or e.errno == errno.EACCES:
user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
raise AlreadyLockedError, user
finally:
os.unlink(path)
-def _accept(upload):
- if Options["No-Action"]:
- return
- (summary, short_summary) = upload.build_summaries()
- upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
- os.unlink(upload.pkg.changes_file[:-8]+".dak")
+class clean_holding(object):
+ def __init__(self,pkg):
+ self.pkg = pkg
-def do_accept(upload):
- print "ACCEPT"
- cnf = Config()
- if not Options["No-Action"]:
- (summary, short_summary) = upload.build_summaries()
-
- if cnf.FindB("Dinstall::SecurityQueueHandling"):
- upload.dump_vars(cnf["Dir::Queue::Embargoed"])
- upload.move_to_dir(cnf["Dir::Queue::Embargoed"])
- upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
- # Check for override disparities
- upload.Subst["__SUMMARY__"] = summary
- else:
- # Just a normal upload, accept it...
- _accept(upload)
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type, value, traceback):
+ h = Holding()
+
+ for f in self.pkg.files.keys():
+ if os.path.exists(os.path.join(h.holding_dir, f)):
+ os.unlink(os.path.join(h.holding_dir, f))
+
+
+def do_pkg(changes_full_path, session):
+ changes_dir = os.path.dirname(changes_full_path)
+ changes_file = os.path.basename(changes_full_path)
-def do_pkg(changes_file, session):
u = Upload()
- u.pkg.load_dot_dak(changes_file)
+ u.pkg.changes_file = changes_file
+ (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
+ u.load_changes(changes_file)
+ u.pkg.directory = changes_dir
u.update_subst()
+ u.logger = Logger
+ origchanges = os.path.abspath(u.pkg.changes_file)
+
+ # Try to get an included dsc
+ dsc = None
+ (status, _) = u.load_dsc()
+ if status:
+ dsc = u.pkg.dsc
cnf = Config()
bcc = "X-DAK: dak process-new"
u.Subst["__BCC__"] = bcc
files = u.pkg.files
+ u.check_distributions()
+ for deb_filename, f in files.items():
+ if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
+ u.binary_file_checks(deb_filename, session)
+ u.check_binary_against_db(deb_filename, session)
+ else:
+ u.source_file_checks(deb_filename, session)
+ u.check_source_against_db(deb_filename, session)
+
+ u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
try:
with lock_package(u.pkg.changes["source"]):
- if not recheck(u, session):
- return
+ with clean_holding(u.pkg):
+ if not recheck(u, session):
+ return
- (new, byhand) = check_status(files)
- if new or byhand:
- if new:
- do_new(u, session)
+ new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session)
if byhand:
do_byhand(u, session)
- (new, byhand) = check_status(files)
-
- if not new and not byhand:
- try:
- check_daily_lock()
- do_accept(u)
- except CantGetLockError:
- print "Hello? Operator! Give me the number for 911!"
- print "Dinstall in the locked area, cant process packages, come back later"
- except AlreadyLockedError, e:
+ elif new:
+ do_new(u, session)
+ else:
+ try:
+ check_daily_lock()
+ new_accept(u, Options["No-Action"], session)
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
+
+ except AlreadyLockedError as e:
print "Seems to be locked by %s already, skipping..." % (e)
+def show_new_comments(changes_files, session):
+ sources = set()
+ query = """SELECT package, version, comment, author
+ FROM new_comments
+ WHERE package IN ('"""
+
+ for changes in changes_files:
+ sources.add(os.path.basename(changes).split("_")[0])
+
+ query += "%s') ORDER BY package, version" % "', '".join(sources)
+ r = session.execute(query)
+
+ for i in r:
+ print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
+
+ session.commit()
+
################################################################################
def end():
session = DBConn().session()
Arguments = [('a',"automatic","Process-New::Options::Automatic"),
+ ('b',"no-binaries","Process-New::Options::No-Binaries"),
+ ('c',"comments","Process-New::Options::Comments"),
('h',"help","Process-New::Options::Help"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('t',"trainee","Process-New::Options::Trainee"),
('n',"no-action","Process-New::Options::No-Action")]
- for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
+ for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:
if not cnf.has_key("Process-New::Options::%s" % (i)):
cnf["Process-New::Options::%s" % (i)] = ""
changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
if len(changes_files) == 0:
- changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
+ new_queue = get_policy_queue('new', session );
+ changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
+ else:
+ changes_paths = [ os.path.abspath(j) for j in changes_files ]
Options = cnf.SubTree("Process-New::Options")
if not Options["No-Action"]:
try:
- Logger = daklog.Logger(cnf, "process-new")
- except CantOpenError, e:
+ Logger = daklog.Logger("process-new")
+ except CantOpenError as e:
Options["Trainee"] = "True"
Sections = Section_Completer(session)
Priorities = Priority_Completer(session)
readline.parse_and_bind("tab: complete")
- if len(changes_files) > 1:
+ if len(changes_paths) > 1:
sys.stderr.write("Sorting changes...\n")
- changes_files = sort_changes(changes_files, session)
+ changes_files = sort_changes(changes_paths, session, Options["No-Binaries"])
- # Kill me now? **FIXME**
- cnf["Dinstall::Options::No-Mail"] = ""
-
- for changes_file in changes_files:
- changes_file = utils.validate_changes_file_arg(changes_file, 0)
- if not changes_file:
- continue
- print "\n" + changes_file
+ if Options["Comments"]:
+ show_new_comments(changes_files, session)
+ else:
+ for changes_file in changes_files:
+ changes_file = utils.validate_changes_file_arg(changes_file, 0)
+ if not changes_file:
+ continue
+ print "\n" + os.path.basename(changes_file)
- do_pkg (changes_file, session)
+ do_pkg (changes_file, session)
end()
projects / dak.git / blobdiff