X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fqueue.py;h=25890367a8807b5e9b1915e1f9d13f86af3c3b96;hb=8f1a6ffdbf7683fba88d8d5a1d8cea4dc5512abb;hp=44f63263d4830481e89e18f9ef24af63abe333ec;hpb=bb6bedc0f40d647db2d41eb5583342d06056bf9a;p=dak.git diff --git a/daklib/queue.py b/daklib/queue.py old mode 100755 new mode 100644 index 44f63263..25890367 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -54,6 +54,7 @@ from summarystats import SummaryStats from utils import parse_changes, check_dsc_files from textutils import fix_maintainer from binary import Binary +from lintian import parse_lintian_output, generate_reject_messages ############################################################################### @@ -438,12 +439,6 @@ class Upload(object): self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"]) self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"]) - # Check there isn't already a changes file of the same name in one - # of the queue directories. - base_filename = os.path.basename(filename) - if get_knownchange(base_filename): - self.rejects.append("%s: a file with this name already exists." % (base_filename)) - # Check the .changes is non-empty if not self.pkg.files: self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename)) @@ -666,7 +661,7 @@ class Upload(object): entry["new"] = 1 else: dsc_file_exists = False - for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]: + for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]: if cnf.has_key("Dir::Queue::%s" % (myq)): if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)): dsc_file_exists = True @@ -726,7 +721,6 @@ class Upload(object): def per_suite_file_checks(self, f, suite, session): cnf = Config() entry = self.pkg.files[f] - archive = utils.where_am_i() # Skip byhand if entry.has_key("byhand"): @@ -770,9 +764,9 @@ class Upload(object): # Determine the location location = cnf["Dir::Pool"] - l = get_location(location, entry["component"], archive, session) + l = get_location(location, entry["component"], session=session) if l is None: - self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (entry["component"], archive)) + self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %)" % entry["component"]) entry["location id"] = -1 else: entry["location id"] = l.location_id @@ -822,8 +816,11 @@ class Upload(object): session = DBConn().session() try: - changes = session.query(KnownChange).filter_by(changesname=base_filename).one() - if not changes.approved_for: + dbc = session.query(DBChange).filter_by(changesname=base_filename).one() + # if in the pool or in a queue other than unchecked, reject + if (dbc.in_queue is None) \ + or (dbc.in_queue is not None + and dbc.in_queue.queue_name != 'unchecked'): self.rejects.append("%s file already known to dak" % base_filename) except NoResultFound, e: # not known, good @@ -1228,7 +1225,7 @@ class Upload(object): continue # Look in some other queues for the file - queues = ('Accepted', 'New', 'Byhand', 'ProposedUpdates', + queues = ('New', 'Byhand', 'ProposedUpdates', 'OldProposedUpdates', 'Embargoed', 'Unembargoed') for queue in queues: @@ -1251,6 +1248,11 @@ class Upload(object): ########################################################################### def check_lintian(self): + """ + Extends self.rejects by checking the output of lintian against tags + specified in Dinstall::LintianTags. + """ + cnf = Config() # Don't reject binary uploads @@ -1258,24 +1260,22 @@ class Upload(object): return # Only check some distributions - valid_dist = False for dist in ('unstable', 'experimental'): if dist in self.pkg.changes['distribution']: - valid_dist = True break - - if not valid_dist: + else: return + # If we do not have a tagfile, don't do anything tagfile = cnf.get("Dinstall::LintianTags") if tagfile is None: - # We don't have a tagfile, so just don't do anything. return # Parse the yaml file sourcefile = file(tagfile, 'r') sourcecontent = sourcefile.read() sourcefile.close() + try: lintiantags = yaml.load(sourcecontent)['lintian'] except yaml.YAMLError, msg: @@ -1285,78 +1285,42 @@ class Upload(object): # Try and find all orig mentioned in the .dsc symlinked = self.ensure_orig() - # Now setup the input file for lintian. lintian wants "one tag per line" only, - # so put it together like it. We put all types of tags in one file and then sort - # through lintians output later to see if its a fatal tag we detected, or not. - # So we only run lintian once on all tags, even if we might reject on some, but not - # reject on others. - # Additionally build up a set of tags - tags = set() - (fd, temp_filename) = utils.temp_filename() + # Setup the input file for lintian + fd, temp_filename = utils.temp_filename() temptagfile = os.fdopen(fd, 'w') - for tagtype in lintiantags: - for tag in lintiantags[tagtype]: - temptagfile.write("%s\n" % tag) - tags.add(tag) + for tags in lintiantags.values(): + temptagfile.writelines(['%s\n' % x for x in tags]) temptagfile.close() - # So now we should look at running lintian at the .changes file, capturing output - # to then parse it. - command = "lintian --show-overrides --tags-from-file %s %s" % (temp_filename, self.pkg.changes_file) - (result, output) = commands.getstatusoutput(command) + try: + cmd = "lintian --show-overrides --tags-from-file %s %s" % \ + (temp_filename, self.pkg.changes_file) - # We are done with lintian, remove our tempfile and any symlinks we created - os.unlink(temp_filename) - for symlink in symlinked: - os.unlink(symlink) + result, output = commands.getstatusoutput(cmd) + finally: + # Remove our tempfile and any symlinks we created + os.unlink(temp_filename) - if (result == 2): - utils.warn("lintian failed for %s [return code: %s]." % (self.pkg.changes_file, result)) - utils.warn(utils.prefix_multi_line_string(output, " [possible output:] ")) + for symlink in symlinked: + os.unlink(symlink) - if len(output) == 0: - return + if result == 2: + utils.warn("lintian failed for %s [return code: %s]." % \ + (self.pkg.changes_file, result)) + utils.warn(utils.prefix_multi_line_string(output, \ + " [possible output:] ")) def log(*txt): if self.logger: - self.logger.log([self.pkg.changes_file, "check_lintian"] + list(txt)) - - # We have output of lintian, this package isn't clean. Lets parse it and see if we - # are having a victim for a reject. - # W: tzdata: binary-without-manpage usr/sbin/tzconfig - for line in output.split('\n'): - m = re_parse_lintian.match(line) - if m is None: - continue - - etype = m.group(1) - epackage = m.group(2) - etag = m.group(3) - etext = m.group(4) - - # So lets check if we know the tag at all. - if etag not in tags: - continue + self.logger.log( + [self.pkg.changes_file, "check_lintian"] + list(txt) + ) - if etype == 'O': - # We know it and it is overriden. Check that override is allowed. - if etag in lintiantags['warning']: - # The tag is overriden, and it is allowed to be overriden. - # Don't add a reject message. - pass - elif etag in lintiantags['error']: - # The tag is overriden - but is not allowed to be - self.rejects.append("%s: Overriden tag %s found, but this tag may not be overwritten." % (epackage, etag)) - log("ftpmaster does not allow tag to be overridable", etag) - else: - # Tag is known, it is not overriden, direct reject. - self.rejects.append("%s: Found lintian output: '%s %s', automatically rejected package." % (epackage, etag, etext)) - # Now tell if they *might* override it. - if etag in lintiantags['warning']: - log("auto rejecting", "overridable", etag) - self.rejects.append("%s: If you have a good reason, you may override this lintian tag." % (epackage)) - else: - log("auto rejecting", "not overridable", etag) + # Generate messages + parsed_tags = parse_lintian_output(output) + self.rejects.extend( + generate_reject_messages(parsed_tags, lintiantags, log=log) + ) ########################################################################### def check_urgency(self): @@ -1846,15 +1810,19 @@ distribution.""" print "Installing." self.logger.log(["installing changes", self.pkg.changes_file]) + poolfiles = [] + # Add the .dsc file to the DB first for newfile, entry in self.pkg.files.items(): if entry["type"] == "dsc": - dsc_component, dsc_location_id = add_dsc_to_db(self, newfile, session) + dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session) + for j in pfs: + poolfiles.append(j) # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb) for newfile, entry in self.pkg.files.items(): if entry["type"] == "deb": - add_deb_to_db(self, newfile, session) + poolfiles.append(add_deb_to_db(self, newfile, session)) # If this is a sourceful diff only upload that is moving # cross-component we need to copy the .orig files into the new @@ -1889,6 +1857,8 @@ distribution.""" session.add(dscf) session.flush() + poolfiles.append(newf) + # Install the files into the pool for newfile, entry in self.pkg.files.items(): destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile) @@ -1962,11 +1932,14 @@ distribution.""" os.rename(temp_filename, filename) os.chmod(filename, 0644) - # auto-build queue -# res = get_or_set_queue('buildd', session).autobuild_upload(self.pkg, session) -# if res: -# utils.fubar(res) -# now_date = datetime.now() + session.commit() + + # Set up our copy queues (e.g. buildd queues) + for suite_name in self.pkg.changes["distribution"].keys(): + suite = get_suite(suite_name, session) + for q in suite.copy_queues: + for f in poolfiles: + q.add_file_from_pool(f) session.commit() @@ -2021,22 +1994,22 @@ distribution.""" os.unlink(os.path.join(from_dir, f)) if os.path.exists(os.path.join(h.holding_dir, f)): os.unlink(os.path.join(h.holding_dir, f)) - + os.unlink(os.path.join(from_dir, self.pkg.changes_file)) if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)): os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file)) ########################################################################### - def move_to_dir (self, dest, perms=0660, changesperms=0664): + def move_to_queue (self, queue): """ - Move files to dest with certain perms/changesperms + Move files to a destination queue using the permissions in the table """ h = Holding() utils.move(os.path.join(h.holding_dir, self.pkg.changes_file), - dest, perms=changesperms) + queue.path, perms=int(queue.change_perms, 8)) for f in self.pkg.files.keys(): - utils.move(os.path.join(h.holding_dir, f), dest, perms=perms) + utils.move(os.path.join(h.holding_dir, f), queue.path, perms=int(queue.perms, 8)) ########################################################################### @@ -2357,8 +2330,6 @@ distribution.""" ################################################################################ def check_source_against_db(self, filename, session): - """ - """ source = self.pkg.dsc.get("source") version = self.pkg.dsc.get("version") @@ -2482,7 +2453,7 @@ distribution.""" else: # TODO: Record the queues and info in the DB so we don't hardcode all this crap # Not there? Check the queue directories... - for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]: + for directory in [ "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]: if not Cnf.has_key("Dir::Queue::%s" % (directory)): continue in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name) @@ -2531,7 +2502,7 @@ distribution.""" source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) found = False - for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]: + for q in ["Embargoed", "Unembargoed", "Newstage"]: if cnf.has_key("Dir::Queue::%s" % (q)): if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): found = True