X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fqueue.py;h=9130c3a38ab1758382c0bbc851039cf48773ab77;hb=2918ca9e2fa7db73192a6ad22c1dd405204e2386;hp=c045c19eaa60eeecd43aa5b9e9608011cdddc48a;hpb=53181135f740882c19553c2a8146de6a3041910c;p=dak.git diff --git a/daklib/queue.py b/daklib/queue.py index c045c19e..9130c3a3 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -92,10 +92,13 @@ def get_type(f, session): # Determine what parts in a .changes are NEW -def determine_new(changes, files, warn=1, session = None): +def determine_new(filename, changes, files, warn=1, session = None): """ Determine what parts in a C{changes} file are NEW. + @type filename: str + @param filename: changes filename + @type changes: Upload.Pkg.changes dict @param changes: Changes dictionary @@ -109,13 +112,22 @@ def determine_new(changes, files, warn=1, session = None): @return: dictionary of NEW components. """ + # TODO: This should all use the database instead of parsing the changes + # file again new = {} + byhand = {} + + dbchg = get_dbchange(filename, session) + if dbchg is None: + print "Warning: cannot find changes file in database; won't check byhand" # Build up a list of potentially new things for name, f in files.items(): - # Skip byhand elements -# if f["type"] == "byhand": -# continue + # Keep a record of byhand elements + if f["section"] == "byhand": + byhand[name] = 1 + continue + pkg = f["package"] priority = f["priority"] section = f["section"] @@ -150,19 +162,40 @@ def determine_new(changes, files, warn=1, session = None): # Fix up the list of target suites cnf = Config() for suite in changes["suite"].keys(): - override = cnf.Find("Suite::%s::OverrideSuite" % (suite)) - if override: - (olderr, newerr) = (get_suite(suite, session) == None, - get_suite(override, session) == None) - if olderr or newerr: - (oinv, newinv) = ("", "") - if olderr: oinv = "invalid " - if newerr: ninv = "invalid " - print "warning: overriding %ssuite %s to %ssuite %s" % ( - oinv, suite, ninv, override) - del changes["suite"][suite] - changes["suite"][override] = 1 + oldsuite = get_suite(suite, session) + if not oldsuite: + print "WARNING: Invalid suite %s found" % suite + continue + + if oldsuite.overridesuite: + newsuite = get_suite(oldsuite.overridesuite, session) + + if newsuite: + print "INFORMATION: Using overrides from suite %s instead of suite %s" % ( + oldsuite.overridesuite, suite) + del changes["suite"][suite] + changes["suite"][oldsuite.overridesuite] = 1 + else: + print "WARNING: Told to use overridesuite %s for %s but it doesn't exist. Bugger" % ( + oldsuite.overridesuite, suite) + + # Check for unprocessed byhand files + if dbchg is not None: + for b in byhand.keys(): + # Find the file entry in the database + found = False + for f in dbchg.files: + if f.filename == b: + found = True + # If it's processed, we can ignore it + if f.processed: + del byhand[b] + break + + if not found: + print "Warning: Couldn't find BYHAND item %s in the database; assuming unprocessed" + # Check for new stuff for suite in changes["suite"].keys(): for pkg in new.keys(): ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session) @@ -180,7 +213,7 @@ def determine_new(changes, files, warn=1, session = None): if new[pkg].has_key("othercomponents"): print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]) - return new + return new, byhand ################################################################################ @@ -299,7 +332,7 @@ def prod_maintainer(notes, upload): ################################################################################ -def edit_note(note, upload, session): +def edit_note(note, upload, session, trainee=False): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() editor = os.environ.get("EDITOR","vi") @@ -331,12 +364,25 @@ def edit_note(note, upload, session): comment.version = upload.pkg.changes["version"] comment.comment = newnote comment.author = utils.whoami() - comment.trainee = bool(Options["Trainee"]) + comment.trainee = trainee session.add(comment) session.commit() ############################################################################### +# suite names DMs can upload to +dm_suites = ['unstable', 'experimental'] + +def get_newest_source(source, session): + 'returns the newest DBSource object in dm_suites' + ## the most recent version of the package uploaded to unstable or + ## experimental includes the field "DM-Upload-Allowed: yes" in the source + ## section of its control file + q = session.query(DBSource).filter_by(source = source). \ + filter(DBSource.suites.any(Suite.suite_name.in_(dm_suites))). \ + order_by(desc('source.version')) + return q.first() + class Upload(object): """ Everything that has to do with an upload processed. @@ -762,7 +808,8 @@ class Upload(object): (source_version, f, self.pkg.changes["version"])) else: # Check in the SQL database - if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session): + if not source_exists(source_package, source_version, suites = \ + self.pkg.changes["distribution"].keys(), session = session): # Check in one of the other directories source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) @@ -772,7 +819,7 @@ class Upload(object): entry["new"] = 1 else: dsc_file_exists = False - for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates", "Lenny-Volatile-Proposed-Updates"]: + for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]: if cnf.has_key("Dir::Queue::%s" % (myq)): if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)): dsc_file_exists = True @@ -1102,11 +1149,24 @@ class Upload(object): session = DBConn().session() self.check_source_against_db(dsc_filename, session) self.check_dsc_against_db(dsc_filename, session) - session.close() + + dbchg = get_dbchange(self.pkg.changes_file, session) # Finally, check if we're missing any files for f in self.later_check_files: - self.rejects.append("Could not find file %s references in changes" % f) + print 'XXX: %s' % f + # Check if we've already processed this file if we have a dbchg object + ok = False + if dbchg: + for pf in dbchg.files: + if pf.filename == f and pf.processed: + self.notes.append('%s was already processed so we can go ahead' % f) + ok = True + del self.pkg.files[f] + if not ok: + self.rejects.append("Could not find file %s references in changes" % f) + + session.close() return True @@ -1634,22 +1694,13 @@ class Upload(object): if rej: return - ## the most recent version of the package uploaded to unstable or - ## experimental includes the field "DM-Upload-Allowed: yes" in the source - ## section of its control file - q = session.query(DBSource).filter_by(source=self.pkg.changes["source"]) - q = q.join(SrcAssociation) - q = q.join(Suite).filter(Suite.suite_name.in_(['unstable', 'experimental'])) - q = q.order_by(desc('source.version')).limit(1) - - r = q.all() + r = get_newest_source(self.pkg.changes["source"], session) - if len(r) != 1: + if r is None: rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"] self.rejects.append(rej) return - r = r[0] if not r.dm_upload_allowed: rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version) self.rejects.append(rej) @@ -1871,7 +1922,9 @@ distribution.""" self.Subst["__SHORT_SUMMARY__"] = short_summary for dist in self.pkg.changes["distribution"].keys(): - announce_list = cnf.Find("Suite::%s::Announce" % (dist)) + suite = get_suite(dist) + if suite is None: continue + announce_list = suite.announce if announce_list == "" or lists_done.has_key(announce_list): continue @@ -2008,10 +2061,9 @@ distribution.""" stats.accept_bytes += float(entry["size"]) # Copy the .changes file across for suite which need it. - copy_changes = {} - for suite_name in self.pkg.changes["distribution"].keys(): - if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)): - copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = "" + copy_changes = dict([(x.copychanges, '') + for x in session.query(Suite).filter(Suite.suite_name.in_(self.pkg.changes["distribution"].keys())).all() + if x.copychanges is not None]) for dest in copy_changes.keys(): utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest)) @@ -2319,8 +2371,9 @@ distribution.""" file_type = binary_type # Override suite name; used for example with proposed-updates - if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "": - suite = cnf["Suite::%s::OverrideSuite" % (suite)] + oldsuite = get_suite(suite, session) + if (not oldsuite is None) and oldsuite.overridesuite: + suite = oldsuite.overridesuite result = get_override(package, suite, component, file_type, session) @@ -2633,7 +2686,8 @@ distribution.""" source_version = entry["source version"] source_package = entry["source package"] if not self.pkg.changes["architecture"].has_key("source") \ - and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session): + and not source_exists(source_package, source_version, \ + suites = self.pkg.changes["distribution"].keys(), session = session): source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) found = False @@ -2680,7 +2734,9 @@ distribution.""" source_version = entry["source version"] source_package = entry["source package"] if not self.pkg.changes["architecture"].has_key("source") \ - and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()): + and not source_exists(source_package, source_version, \ + suites = self.pkg.changes["distribution"].keys(), \ + session = session): self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile)) # Version and file overwrite checks