X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fprocess_unchecked.py;h=412ad64aa9088f5f30be8a64401d7a82d526651f;hb=6b9ada122cf00986a2f44d97bfc57b34e4e9ea98;hp=1cf2234ecd2f10a4a0f6b132980e25e40f175f67;hpb=021ab67920bbf31549ba7c9d1c85f14e32f5c428;p=dak.git diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index 1cf2234e..412ad64a 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -39,10 +39,11 @@ from types import * ################################################################################ -re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$") +re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:~]+$") re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$") re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)") re_strip_revision = re.compile(r"-([^-]+)$") +re_strip_srcver = re.compile(r"\s+\(\S+\)$") ################################################################################ @@ -202,6 +203,14 @@ def check_changes(): reject("%s: Missing mandatory field `%s'." % (filename, i)) return 0 # Avoid errors during later tests + # Strip a source version in brackets from the source field + if re_strip_srcver.search(changes["source"]): + changes["source"] = re_strip_srcver.sub('', changes["source"]) + + # Ensure the source field is a valid package name. + if not re_valid_pkg_name.match(changes["source"]): + reject("%s: invalid source name '%s'." % (filename, changes["source"])) + # Split multi-value fields into a lower-level dictionary for i in ("architecture", "distribution", "binary", "closes"): o = changes.get(i, "") @@ -246,7 +255,7 @@ def check_changes(): # Check there isn't already a changes file of the same name in one # of the queue directories. base_filename = os.path.basename(filename) - for dir in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates" ]: + for dir in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]: if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+base_filename): reject("%s: a file with this name already exists in the %s directory." % (base_filename, dir)) @@ -395,7 +404,7 @@ def check_files(): for file in file_keys: # Ensure the file does not already exist in one of the accepted directories - for dir in [ "Accepted", "Byhand", "New", "ProposedUpdates" ]: + for dir in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates" ]: if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+file): reject("%s file already exists in the %s directory." % (file, dir)) if not daklib.utils.re_taint_free.match(file): @@ -412,7 +421,7 @@ def check_files(): files[file]["type"] = "unreadable" continue # If it's byhand skip remaining checks - if files[file]["section"] == "byhand" or files[file]["section"] == "raw-installer": + if files[file]["section"] == "byhand" or files[file]["section"][4:] == "raw-": files[file]["byhand"] = 1 files[file]["type"] = "byhand" # Checks for a binary package... @@ -529,8 +538,15 @@ def check_files(): files[file]["byhand"] = 1 elif os.path.exists(Cnf["Dir::Queue::New"] + '/' + dsc_filename): files[file]["new"] = 1 - elif not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): - reject("no source found for %s %s (%s)." % (source_package, source_version, file)) + else: + dsc_file_exists = 0 + for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]: + if Cnf.has_key("Dir::Queue::%s" % (myq)): + if os.path.exists(Cnf["Dir::Queue::"+myq] + '/' + dsc_filename): + dsc_file_exists = 1 + break + if not dsc_file_exists: + reject("no source found for %s %s (%s)." % (source_package, source_version, file)) # Check the version and for file overwrites reject(Upload.check_binary_against_db(file),"") @@ -741,6 +757,7 @@ def check_dsc(): m = daklib.utils.re_issource.match(f) if not m: reject("%s: %s in Files field not recognised as source." % (dsc_filename, f)) + continue type = m.group(3) if type == "orig.tar.gz" or type == "tar.gz": has_tar = 1 @@ -971,7 +988,7 @@ def check_timestamps(): apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz") except SystemError, e: # If we can't find a data.tar.gz, look for data.tar.bz2 instead. - if not re.match(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)): + if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)): raise deb_file.seek(0) apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2") @@ -997,6 +1014,48 @@ def check_timestamps(): except: reject("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value)) +################################################################################ + +def check_signed_by_key(): + """Ensure the .changes is signed by an authorized uploader.""" + + # We only check binary-only uploads right now + if changes["architecture"].has_key("source"): + return + + if not Cnf.Exists("Binary-Upload-Restrictions"): + return + + restrictions = Cnf.SubTree("Binary-Upload-Restrictions") + + # If the restrictions only apply to certain components make sure + # that the upload is actual targeted there. + if restrictions.Exists("Components"): + restricted_components = restrictions.SubTree("Components").ValueList() + is_restricted = False + for file in files: + if files[file]["component"] in restricted_components: + is_restricted = True + break + if not is_restricted: + return + + # Assuming binary only upload restrictions are in place we then + # iterate over suite and architecture checking the key is in the + # allowed list. If no allowed list exists for a given suite or + # architecture it's assumed to be open to anyone. + for suite in changes["distribution"].keys(): + if not restrictions.Exists(suite): + continue + for arch in changes["architecture"].keys(): + if not restrictions.SubTree(suite).Exists(arch): + continue + allowed_keys = restrictions.SubTree("%s::%s" % (suite, arch)).ValueList() + if changes["fingerprint"] not in allowed_keys: + base_filename = os.path.basename(pkg.changes_file) + reject("%s: not signed by authorised uploader for %s/%s" + % (base_filename, suite, arch)) + ################################################################################ ################################################################################ @@ -1035,16 +1094,19 @@ def action (): # q-unapproved hax0ring queue_info = { "New": { "is": is_new, "process": acknowledge_new }, + "Autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand }, "Byhand" : { "is": is_byhand, "process": do_byhand }, + "OldStableUpdate" : { "is": is_oldstableupdate, + "process": do_oldstableupdate }, "StableUpdate" : { "is": is_stableupdate, "process": do_stableupdate }, "Unembargo" : { "is": is_unembargo, "process": queue_unembargo }, "Embargo" : { "is": is_embargo, "process": queue_embargo }, } - queues = [ "New", "Byhand" ] + queues = [ "New", "Autobyhand", "Byhand" ] if Cnf.FindB("Dinstall::SecurityQueueHandling"): queues += [ "Unembargo", "Embargo" ] else: - queues += [ "StableUpdate" ] + queues += [ "OldStableUpdate", "StableUpdate" ] (prompt, answer) = ("", "XXX") if Options["No-Action"] or Options["Automatic"]: @@ -1099,7 +1161,7 @@ def action (): accept(summary, short_summary) remove_from_unchecked() elif answer == queuekey: - queue_info[queue]["process"](summary) + queue_info[queue]["process"](summary, short_summary) remove_from_unchecked() elif answer == 'Q': sys.exit(0) @@ -1134,7 +1196,12 @@ def is_unembargo (): if ql: return 1 - if pkg.directory == Cnf["Dir::Queue::Disembargo"].rstrip("/"): + oldcwd = os.getcwd() + os.chdir(Cnf["Dir::Queue::Disembargo"]) + disdir = os.getcwd() + os.chdir(oldcwd) + + if pkg.directory == disdir: if changes["architecture"].has_key("source"): if Options["No-Action"]: return 1 @@ -1145,7 +1212,7 @@ def is_unembargo (): return 0 -def queue_unembargo (summary): +def queue_unembargo (summary, short_summary): print "Moving to UNEMBARGOED holding area." Logger.log(["Moving to unembargoed", pkg.changes_file]) @@ -1162,7 +1229,7 @@ def queue_unembargo (summary): def is_embargo (): return 0 -def queue_embargo (summary): +def queue_embargo (summary, short_summary): print "Moving to EMBARGOED holding area." Logger.log(["Moving to embargoed", pkg.changes_file]) @@ -1177,11 +1244,22 @@ def queue_embargo (summary): ################################################################################ def is_stableupdate (): - if changes["distribution"].has_key("proposed-updates"): - return 1 - return 0 + if not changes["distribution"].has_key("proposed-updates"): + return 0 -def do_stableupdate (summary): + if not changes["architecture"].has_key("source"): + pusuite = daklib.database.get_suite_id("proposed-updates") + q = Upload.projectB.query( + "SELECT S.source FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.version = '%s' AND sa.suite = %d" % + (changes["source"], changes["version"], pusuite)) + ql = q.getresult() + if ql: + # source is already in proposed-updates so no need to hold + return 0 + + return 1 + +def do_stableupdate (summary, short_summary): print "Moving to PROPOSED-UPDATES holding area." Logger.log(["Moving to proposed-updates", pkg.changes_file]); @@ -1194,13 +1272,109 @@ def do_stableupdate (summary): ################################################################################ +def is_oldstableupdate (): + if not changes["distribution"].has_key("oldstable-proposed-updates"): + return 0 + + if not changes["architecture"].has_key("source"): + pusuite = daklib.database.get_suite_id("oldstable-proposed-updates") + q = Upload.projectB.query( + "SELECT S.source FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.version = '%s' AND sa.suite = %d" % + (changes["source"], changes["version"], pusuite)) + ql = q.getresult() + if ql: + # source is already in oldstable-proposed-updates so no need to hold + return 0 + + return 1 + +def do_oldstableupdate (summary, short_summary): + print "Moving to OLDSTABLE-PROPOSED-UPDATES holding area." + Logger.log(["Moving to oldstable-proposed-updates", pkg.changes_file]); + + Upload.dump_vars(Cnf["Dir::Queue::OldProposedUpdates"]); + move_to_dir(Cnf["Dir::Queue::OldProposedUpdates"]) + + # Check for override disparities + Upload.Subst["__SUMMARY__"] = summary; + Upload.check_override(); + +################################################################################ + +def is_autobyhand (): + all_auto = 1 + any_auto = 0 + for file in files.keys(): + if files[file].has_key("byhand"): + any_auto = 1 + + # filename is of form "PKG_VER_ARCH.EXT" where PKG, VER and ARCH + # don't contain underscores, and ARCH doesn't contain dots. + # further VER matches the .changes Version:, and ARCH should be in + # the .changes Architecture: list. + if file.count("_") < 2: + all_auto = 0 + continue + + (pkg, ver, archext) = file.split("_", 2) + if archext.count(".") < 1 or changes["version"] != ver: + all_auto = 0 + continue + + ABH = Cnf.SubTree("AutomaticByHandPackages") + if not ABH.has_key(pkg) or \ + ABH["%s::Source" % (pkg)] != changes["source"]: + print "not match %s %s" % (pkg, changes["source"]) + all_auto = 0 + continue + + (arch, ext) = archext.split(".", 1) + if arch not in changes["architecture"]: + all_auto = 0 + continue + + files[file]["byhand-arch"] = arch + files[file]["byhand-script"] = ABH["%s::Script" % (pkg)] + + return any_auto and all_auto + +def do_autobyhand (summary, short_summary): + print "Attempting AUTOBYHAND." + byhandleft = 0 + for file in files.keys(): + byhandfile = file + if not files[file].has_key("byhand"): + continue + if not files[file].has_key("byhand-script"): + byhandleft = 1 + continue + + os.system("ls -l %s" % byhandfile) + result = os.system("%s %s %s %s %s" % ( + files[file]["byhand-script"], byhandfile, + changes["version"], files[file]["byhand-arch"], + os.path.abspath(pkg.changes_file))) + if result == 0: + os.unlink(byhandfile) + del files[file] + else: + print "Error processing %s, left as byhand." % (file) + byhandleft = 1 + + if byhandleft: + do_byhand(summary, short_summary) + else: + accept(summary, short_summary) + +################################################################################ + def is_byhand (): for file in files.keys(): if files[file].has_key("byhand"): return 1 return 0 -def do_byhand (summary): +def do_byhand (summary, short_summary): print "Moving to BYHAND holding area." Logger.log(["Moving to byhand", pkg.changes_file]) @@ -1219,7 +1393,7 @@ def is_new (): return 1 return 0 -def acknowledge_new (summary): +def acknowledge_new (summary, short_summary): Subst = Upload.Subst print "Moving to NEW holding area." @@ -1288,6 +1462,7 @@ def process_it (changes_file): check_md5sums() check_urgency() check_timestamps() + check_signed_by_key() Upload.update_subst(reject_message) action() except SystemExit: