X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fqueue.py;h=b4c62d38a204ded115d6bd7738e1913ffe02438a;hb=b8690d6b1256a4a95198b874d9eb1aab62a825fb;hp=e3874b620583f9c293de9ab7c22e136a0c954333;hpb=cc6ef10f9e5994c51e68622d964a2f798ae801b9;p=dak.git diff --git a/daklib/queue.py b/daklib/queue.py index e3874b62..b4c62d38 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -53,8 +53,18 @@ from dbconn import * from summarystats import SummaryStats from utils import parse_changes, check_dsc_files from textutils import fix_maintainer -from binary import Binary from lintian import parse_lintian_output, generate_reject_messages +from contents import UnpackedSource + +# suppress some deprecation warnings in squeeze related to apt_pkg +# module +import warnings +warnings.filterwarnings('ignore', \ + "apt_pkg.ParseSection\(\) is deprecated. Please see apt_pkg\.TagSection\(\) for the replacement\.", \ + DeprecationWarning) +warnings.filterwarnings('ignore', \ + "Attribute 'Find' of the 'apt_pkg\.TagSection' object is deprecated, use 'find' instead\.", \ + DeprecationWarning) ############################################################################### @@ -162,18 +172,22 @@ def determine_new(filename, changes, files, warn=1, session = None): # Fix up the list of target suites cnf = Config() for suite in changes["suite"].keys(): - override = cnf.Find("Suite::%s::OverrideSuite" % (suite)) - if override: - (olderr, newerr) = (get_suite(suite, session) == None, - get_suite(override, session) == None) - if olderr or newerr: - (oinv, newinv) = ("", "") - if olderr: oinv = "invalid " - if newerr: ninv = "invalid " - print "warning: overriding %ssuite %s to %ssuite %s" % ( - oinv, suite, ninv, override) - del changes["suite"][suite] - changes["suite"][override] = 1 + oldsuite = get_suite(suite, session) + if not oldsuite: + print "WARNING: Invalid suite %s found" % suite + continue + + if oldsuite.overridesuite: + newsuite = get_suite(oldsuite.overridesuite, session) + + if newsuite: + print "INFORMATION: Using overrides from suite %s instead of suite %s" % ( + oldsuite.overridesuite, suite) + del changes["suite"][suite] + changes["suite"][oldsuite.overridesuite] = 1 + else: + print "WARNING: Told to use overridesuite %s for %s but it doesn't exist. Bugger" % ( + oldsuite.overridesuite, suite) # Check for unprocessed byhand files if dbchg is not None: @@ -328,7 +342,7 @@ def prod_maintainer(notes, upload): ################################################################################ -def edit_note(note, upload, session): +def edit_note(note, upload, session, trainee=False): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() editor = os.environ.get("EDITOR","vi") @@ -360,12 +374,50 @@ def edit_note(note, upload, session): comment.version = upload.pkg.changes["version"] comment.comment = newnote comment.author = utils.whoami() - comment.trainee = bool(Options["Trainee"]) + comment.trainee = trainee session.add(comment) session.commit() ############################################################################### +# suite names DMs can upload to +dm_suites = ['unstable', 'experimental'] + +def get_newest_source(source, session): + 'returns the newest DBSource object in dm_suites' + ## the most recent version of the package uploaded to unstable or + ## experimental includes the field "DM-Upload-Allowed: yes" in the source + ## section of its control file + q = session.query(DBSource).filter_by(source = source). \ + filter(DBSource.suites.any(Suite.suite_name.in_(dm_suites))). \ + order_by(desc('source.version')) + return q.first() + +def get_suite_version_by_source(source, session): + 'returns a list of tuples (suite_name, version) for source package' + q = session.query(Suite.suite_name, DBSource.version). \ + join(Suite.sources).filter_by(source = source) + return q.all() + +def get_source_by_package_and_suite(package, suite_name, session): + ''' + returns a DBSource query filtered by DBBinary.package and this package's + suite_name + ''' + return session.query(DBSource). \ + join(DBSource.binaries).filter_by(package = package). \ + join(DBBinary.suites).filter_by(suite_name = suite_name) + +def get_suite_version_by_package(package, arch_string, session): + ''' + returns a list of tuples (suite_name, version) for binary package and + arch_string + ''' + return session.query(Suite.suite_name, DBBinary.version). \ + join(Suite.binaries).filter_by(package = package). \ + join(DBBinary.architecture). \ + filter(Architecture.arch_string.in_([arch_string, 'all'])).all() + class Upload(object): """ Everything that has to do with an upload processed. @@ -721,6 +773,30 @@ class Upload(object): if not re_valid_pkg_name.match(prov): self.rejects.append("%s: Invalid Provides field content %s." % (f, prov)) + # If there is a Built-Using field, we need to check we can find the + # exact source version + built_using = control.Find("Built-Using") + if built_using: + try: + entry["built-using"] = [] + for dep in apt_pkg.parse_depends(built_using): + bu_s, bu_v, bu_e = dep[0] + # Check that it's an exact match dependency and we have + # some form of version + if bu_e != "=" or len(bu_v) < 1: + self.rejects.append("%s: Built-Using contains non strict dependency (%s %s %s)" % (f, bu_s, bu_e, bu_v)) + else: + # Find the source id for this version + bu_so = get_sources_from_name(bu_s, version=bu_v, session = session) + if len(bu_so) != 1: + self.rejects.append("%s: Built-Using (%s = %s): Cannot find source package" % (f, bu_s, bu_v)) + else: + entry["built-using"].append( (bu_so[0].source, bu_so[0].version, ) ) + + except ValueError, e: + self.rejects.append("%s: Cannot parse Built-Using field: %s" % (f, str(e))) + + # Check the section & priority match those given in the .changes (non-fatal) if control.Find("Section") and entry["section"] != "" \ and entry["section"] != control.Find("Section"): @@ -791,7 +867,8 @@ class Upload(object): (source_version, f, self.pkg.changes["version"])) else: # Check in the SQL database - if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session): + if not source_exists(source_package, source_version, suites = \ + self.pkg.changes["distribution"].keys(), session = session): # Check in one of the other directories source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) @@ -801,7 +878,7 @@ class Upload(object): entry["new"] = 1 else: dsc_file_exists = False - for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates", "Lenny-Volatile-Proposed-Updates"]: + for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]: if cnf.has_key("Dir::Queue::%s" % (myq)): if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)): dsc_file_exists = True @@ -813,13 +890,6 @@ class Upload(object): # Check the version and for file overwrites self.check_binary_against_db(f, session) - # Temporarily disable contents generation until we change the table storage layout - #b = Binary(f) - #b.scan_package() - #if len(b.rejects) > 0: - # for j in b.rejects: - # self.rejects.append(j) - def source_file_checks(self, f, session): entry = self.pkg.files[f] @@ -929,9 +999,11 @@ class Upload(object): # Check for packages that have moved from one component to another entry['suite'] = suite - res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session) - if res.rowcount > 0: - entry["othercomponents"] = res.fetchone()[0] + arch_list = [entry["architecture"], 'all'] + component = get_component_by_package_suite(self.pkg.files[f]['package'], \ + [suite], arch_list = arch_list, session = session) + if component is not None: + entry["othercomponents"] = component def check_files(self, action=True): file_keys = self.pkg.files.keys() @@ -1194,11 +1266,10 @@ class Upload(object): os.symlink(self.pkg.orig_files[orig_file]["path"], dest) # Extract the source - cmd = "dpkg-source -sn -x %s" % (dsc_filename) - (result, output) = commands.getstatusoutput(cmd) - if (result != 0): - self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result)) - self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] ")) + try: + unpacked = UnpackedSource(dsc_filename) + except: + self.rejects.append("'dpkg-source -x' failed for %s." % dsc_filename) return if not cnf.Find("Dir::Queue::BTSVersionTrack"): @@ -1210,19 +1281,19 @@ class Upload(object): upstr_version = re_strip_revision.sub('', upstr_version) # Ensure the changelog file exists - changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version) - if not os.path.exists(changelog_filename): + changelog_file = unpacked.get_changelog_file() + if changelog_file is None: self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename)) return # Parse the changelog self.pkg.dsc["bts changelog"] = "" - changelog_file = utils.open_file(changelog_filename) for line in changelog_file.readlines(): m = re_changelog_versions.match(line) if m: self.pkg.dsc["bts changelog"] += line changelog_file.close() + unpacked.cleanup() # Check we found at least one revision in the changelog if not self.pkg.dsc["bts changelog"]: @@ -1676,22 +1747,13 @@ class Upload(object): if rej: return - ## the most recent version of the package uploaded to unstable or - ## experimental includes the field "DM-Upload-Allowed: yes" in the source - ## section of its control file - q = session.query(DBSource).filter_by(source=self.pkg.changes["source"]) - q = q.join(SrcAssociation) - q = q.join(Suite).filter(Suite.suite_name.in_(['unstable', 'experimental'])) - q = q.order_by(desc('source.version')).limit(1) + r = get_newest_source(self.pkg.changes["source"], session) - r = q.all() - - if len(r) != 1: + if r is None: rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"] self.rejects.append(rej) return - r = r[0] if not r.dm_upload_allowed: rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version) self.rejects.append(rej) @@ -1723,11 +1785,7 @@ class Upload(object): ## none of the packages are being taken over from other source packages for b in self.pkg.changes["binary"].keys(): for suite in self.pkg.changes["distribution"].keys(): - q = session.query(DBSource) - q = q.join(DBBinary).filter_by(package=b) - q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite) - - for s in q.all(): + for s in get_source_by_package_and_suite(b, suite, session): if s.source != self.pkg.changes["source"]: self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite)) @@ -1914,6 +1972,7 @@ distribution.""" for dist in self.pkg.changes["distribution"].keys(): suite = get_suite(dist) + if suite is None: continue announce_list = suite.announce if announce_list == "" or lists_done.has_key(announce_list): continue @@ -2052,7 +2111,7 @@ distribution.""" # Copy the .changes file across for suite which need it. copy_changes = dict([(x.copychanges, '') - for x in session.query(Suite).filter(Suite.suite_name.in_([self.pkg.changes["distribution"].keys()])).all() + for x in session.query(Suite).filter(Suite.suite_name.in_(self.pkg.changes["distribution"].keys())).all() if x.copychanges is not None]) for dest in copy_changes.keys(): @@ -2361,8 +2420,9 @@ distribution.""" file_type = binary_type # Override suite name; used for example with proposed-updates - if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "": - suite = cnf["Suite::%s::OverrideSuite" % (suite)] + oldsuite = get_suite(suite, session) + if (not oldsuite is None) and oldsuite.overridesuite: + suite = oldsuite.overridesuite result = get_override(package, suite, component, file_type, session) @@ -2482,12 +2542,10 @@ distribution.""" ################################################################################ def check_binary_against_db(self, filename, session): # Ensure version is sane - q = session.query(BinAssociation) - q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[filename]["package"]) - q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[filename]["architecture"], 'all'])) - - self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ], - filename, self.pkg.files[filename]["version"], sourceful=False) + self.cross_suite_version_check( \ + get_suite_version_by_package(self.pkg.files[filename]["package"], \ + self.pkg.files[filename]["architecture"], session), + filename, self.pkg.files[filename]["version"], sourceful=False) # Check for any existing copies of the file q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"]) @@ -2504,11 +2562,9 @@ distribution.""" version = self.pkg.dsc.get("version") # Ensure version is sane - q = session.query(SrcAssociation) - q = q.join(DBSource).filter(DBSource.source==source) - - self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ], - filename, version, sourceful=True) + self.cross_suite_version_check( \ + get_suite_version_by_source(source, session), filename, version, + sourceful=True) ################################################################################ def check_dsc_against_db(self, filename, session): @@ -2675,7 +2731,8 @@ distribution.""" source_version = entry["source version"] source_package = entry["source package"] if not self.pkg.changes["architecture"].has_key("source") \ - and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session): + and not source_exists(source_package, source_version, \ + suites = self.pkg.changes["distribution"].keys(), session = session): source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) found = False @@ -2722,7 +2779,9 @@ distribution.""" source_version = entry["source version"] source_package = entry["source package"] if not self.pkg.changes["architecture"].has_key("source") \ - and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()): + and not source_exists(source_package, source_version, \ + suites = self.pkg.changes["distribution"].keys(), \ + session = session): self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile)) # Version and file overwrite checks