@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2001 - 2006 James Troup <james@nocrew.org>
-@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
@license: GNU General Public License version 2 or later
"""
###############################################################################
-import cPickle
import errno
import os
-import pg
import stat
import sys
import time
import utils
import commands
import shutil
+import textwrap
from types import *
+from sqlalchemy.sql.expression import desc
+from sqlalchemy.orm.exc import NoResultFound
+
+import yaml
from dak_exceptions import *
from changes import *
from regexes import *
from config import Config
from holding import Holding
+from urgencylog import UrgencyLog
from dbconn import *
from summarystats import SummaryStats
-from utils import parse_changes
+from utils import parse_changes, check_dsc_files, build_package_set
from textutils import fix_maintainer
-from binary import Binary
+from lintian import parse_lintian_output, generate_reject_messages
+from contents import UnpackedSource
###############################################################################
-def get_type(f, session=None):
+def get_type(f, session):
"""
Get the file type of C{f}
@type f: dict
@param f: file entry from Changes object
+ @type session: SQLA Session
+ @param session: SQL Alchemy session object
+
@rtype: string
@return: filetype
"""
- if session is None:
- session = DBConn().session()
-
# Determine the type
if f.has_key("dbtype"):
- file_type = file["dbtype"]
- elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
+ file_type = f["dbtype"]
+ elif re_source_ext.match(f["type"]):
file_type = "dsc"
+ elif f['architecture'] == 'source' and f["type"] == 'unreadable':
+ utils.warn('unreadable source file (will continue and hope for the best)')
+ return f["type"]
else:
+ file_type = f["type"]
utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
# Validate the override type
# Determine what parts in a .changes are NEW
-def determine_new(changes, files, warn=1):
+def determine_new(filename, changes, files, warn=1, session = None, dsc = None, new = None):
"""
Determine what parts in a C{changes} file are NEW.
+ @type filename: str
+ @param filename: changes filename
+
@type changes: Upload.Pkg.changes dict
@param changes: Changes dictionary
@type warn: bool
@param warn: Warn if overrides are added for (old)stable
+ @type dsc: Upload.Pkg.dsc dict
+ @param dsc: (optional); Dsc dictionary
+
+ @type new: dict
+ @param new: new packages as returned by a previous call to this function, but override information may have changed
+
@rtype: dict
@return: dictionary of NEW components.
"""
- new = {}
-
- session = DBConn().session()
+ # TODO: This should all use the database instead of parsing the changes
+ # file again
+ byhand = {}
+ if new is None:
+ new = {}
+
+ dbchg = get_dbchange(filename, session)
+ if dbchg is None:
+ print "Warning: cannot find changes file in database; won't check byhand"
+
+ # Try to get the Package-Set field from an included .dsc file (if possible).
+ if dsc:
+ for package, entry in build_package_set(dsc, session).items():
+ if not new.has_key(package):
+ new[package] = entry
# Build up a list of potentially new things
for name, f in files.items():
- # Skip byhand elements
- if f["type"] == "byhand":
+ # Keep a record of byhand elements
+ if f["section"] == "byhand":
+ byhand[name] = 1
continue
+
pkg = f["package"]
priority = f["priority"]
section = f["section"]
- file_type = get_type(f)
+ file_type = get_type(f, session)
component = f["component"]
if file_type == "dsc":
if f.has_key("othercomponents"):
new[pkg]["othercomponents"] = f["othercomponents"]
+ # Fix up the list of target suites
+ cnf = Config()
+ for suite in changes["suite"].keys():
+ oldsuite = get_suite(suite, session)
+ if not oldsuite:
+ print "WARNING: Invalid suite %s found" % suite
+ continue
+
+ if oldsuite.overridesuite:
+ newsuite = get_suite(oldsuite.overridesuite, session)
+
+ if newsuite:
+ print "INFORMATION: Using overrides from suite %s instead of suite %s" % (
+ oldsuite.overridesuite, suite)
+ del changes["suite"][suite]
+ changes["suite"][oldsuite.overridesuite] = 1
+ else:
+ print "WARNING: Told to use overridesuite %s for %s but it doesn't exist. Bugger" % (
+ oldsuite.overridesuite, suite)
+
+ # Check for unprocessed byhand files
+ if dbchg is not None:
+ for b in byhand.keys():
+ # Find the file entry in the database
+ found = False
+ for f in dbchg.files:
+ if f.filename == b:
+ found = True
+ # If it's processed, we can ignore it
+ if f.processed:
+ del byhand[b]
+ break
+
+ if not found:
+ print "Warning: Couldn't find BYHAND item %s in the database; assuming unprocessed"
+
+ # Check for new stuff
for suite in changes["suite"].keys():
for pkg in new.keys():
ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
if new[pkg].has_key("othercomponents"):
print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
- return new
+ return new, byhand
################################################################################
-def check_valid(new):
+def check_valid(new, session = None):
"""
Check if section and priority for NEW packages exist in database.
Additionally does sanity checks:
priority_name = new[pkg]["priority"]
file_type = new[pkg]["type"]
- section = get_section(section_name)
+ section = get_section(section_name, session)
if section is None:
new[pkg]["section id"] = -1
else:
new[pkg]["section id"] = section.section_id
- priority = get_priority(priority_name)
+ priority = get_priority(priority_name, session)
if priority is None:
new[pkg]["priority id"] = -1
else:
###############################################################################
-def lookup_uid_from_fingerprint(fpr, session):
- uid = None
- uid_name = ""
- # This is a stupid default, but see the comments below
- is_dm = False
-
- user = get_uid_from_fingerprint(fpr, session)
-
- if user is not None:
- uid = user.uid
- if user.name is None:
- uid_name = ''
- else:
- uid_name = user.name
-
- # Check the relevant fingerprint (which we have to have)
- for f in user.fingerprint:
- if f.fingerprint == fpr:
- is_dm = f.keyring.debian_maintainer
- break
-
- return (uid, uid_name, is_dm)
-
-###############################################################################
-
# Used by Upload.check_timestamps
class TarTime(object):
def __init__(self, future_cutoff, past_cutoff):
self.future_files = {}
self.ancient_files = {}
- def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
- if MTime > self.future_cutoff:
+ def callback(self, member, data):
+ if member.mtime > self.future_cutoff:
self.future_files[Name] = MTime
- if MTime < self.past_cutoff:
+ if member.mtime < self.past_cutoff:
self.ancient_files[Name] = MTime
###############################################################################
+def prod_maintainer(notes, upload):
+ cnf = Config()
+
+ # Here we prepare an editor and get them ready to prod...
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ for note in notes:
+ temp_file.write(note.comment)
+ temp_file.close()
+ editor = os.environ.get("EDITOR","vi")
+ answer = 'E'
+ while answer == 'E':
+ os.system("%s %s" % (editor, temp_filename))
+ temp_fh = utils.open_file(temp_filename)
+ prod_message = "".join(temp_fh.readlines())
+ temp_fh.close()
+ print "Prod message:"
+ print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
+ prompt = "[P]rod, Edit, Abandon, Quit ?"
+ answer = "XXX"
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
+ if answer == "":
+ answer = m.group(1)
+ answer = answer[:1].upper()
+ os.unlink(temp_filename)
+ if answer == 'A':
+ return
+ elif answer == 'Q':
+ end()
+ sys.exit(0)
+ # Otherwise, do the proding...
+ user_email_address = utils.whoami() + " <%s>" % (
+ cnf["Dinstall::MyAdminAddress"])
+
+ Subst = upload.Subst
+
+ Subst["__FROM_ADDRESS__"] = user_email_address
+ Subst["__PROD_MESSAGE__"] = prod_message
+ Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
+
+ prod_mail_message = utils.TemplateSubst(
+ Subst,cnf["Dir::Templates"]+"/process-new.prod")
+
+ # Send the prod mail
+ utils.send_mail(prod_mail_message)
+
+ print "Sent prodding message"
+
+################################################################################
+
+def edit_note(note, upload, session, trainee=False):
+ # Write the current data to a temporary file
+ (fd, temp_filename) = utils.temp_filename()
+ editor = os.environ.get("EDITOR","vi")
+ answer = 'E'
+ while answer == 'E':
+ os.system("%s %s" % (editor, temp_filename))
+ temp_file = utils.open_file(temp_filename)
+ newnote = temp_file.read().rstrip()
+ temp_file.close()
+ print "New Note:"
+ print utils.prefix_multi_line_string(newnote," ")
+ prompt = "[D]one, Edit, Abandon, Quit ?"
+ answer = "XXX"
+ while prompt.find(answer) == -1:
+ answer = utils.our_raw_input(prompt)
+ m = re_default_answer.search(prompt)
+ if answer == "":
+ answer = m.group(1)
+ answer = answer[:1].upper()
+ os.unlink(temp_filename)
+ if answer == 'A':
+ return
+ elif answer == 'Q':
+ end()
+ sys.exit(0)
+
+ comment = NewComment()
+ comment.package = upload.pkg.changes["source"]
+ comment.version = upload.pkg.changes["version"]
+ comment.comment = newnote
+ comment.author = utils.whoami()
+ comment.trainee = trainee
+ session.add(comment)
+ session.commit()
+
+###############################################################################
+
+# suite names DMs can upload to
+dm_suites = ['unstable', 'experimental']
+
+def get_newest_source(source, session):
+ 'returns the newest DBSource object in dm_suites'
+ ## the most recent version of the package uploaded to unstable or
+ ## experimental includes the field "DM-Upload-Allowed: yes" in the source
+ ## section of its control file
+ q = session.query(DBSource).filter_by(source = source). \
+ filter(DBSource.suites.any(Suite.suite_name.in_(dm_suites))). \
+ order_by(desc('source.version'))
+ return q.first()
+
+def get_suite_version_by_source(source, session):
+ 'returns a list of tuples (suite_name, version) for source package'
+ q = session.query(Suite.suite_name, DBSource.version). \
+ join(Suite.sources).filter_by(source = source)
+ return q.all()
+
+def get_source_by_package_and_suite(package, suite_name, session):
+ '''
+ returns a DBSource query filtered by DBBinary.package and this package's
+ suite_name
+ '''
+ return session.query(DBSource). \
+ join(DBSource.binaries).filter_by(package = package). \
+ join(DBBinary.suites).filter_by(suite_name = suite_name)
+
+def get_suite_version_by_package(package, arch_string, session):
+ '''
+ returns a list of tuples (suite_name, version) for binary package and
+ arch_string
+ '''
+ return session.query(Suite.suite_name, DBBinary.version). \
+ join(Suite.binaries).filter_by(package = package). \
+ join(DBBinary.architecture). \
+ filter(Architecture.arch_string.in_([arch_string, 'all'])).all()
+
class Upload(object):
"""
Everything that has to do with an upload processed.
cnf = Config()
self.Subst = {}
self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
- self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
+ if cnf.has_key("Dinstall::BugServer"):
+ self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
self.warnings = []
self.notes = []
+ self.later_check_files = []
+
self.pkg.reset()
def package_info(self):
- msg = ''
-
- if len(self.rejects) > 0:
- msg += "Reject Reasons:\n"
- msg += "\n".join(self.rejects)
+ """
+ Format various messages from this Upload to send to the maintainer.
+ """
- if len(self.warnings) > 0:
- msg += "Warnings:\n"
- msg += "\n".join(self.warnings)
+ msgs = (
+ ('Reject Reasons', self.rejects),
+ ('Warnings', self.warnings),
+ ('Notes', self.notes),
+ )
- if len(self.notes) > 0:
- msg += "Notes:\n"
- msg += "\n".join(self.notes)
+ msg = ''
+ for title, messages in msgs:
+ if messages:
+ msg += '\n\n%s:\n%s' % (title, '\n'.join(messages))
+ msg += '\n\n'
return msg
# If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
if not self.pkg.changes.has_key("architecture") or not \
- isinstance(changes["architecture"], DictType):
+ isinstance(self.pkg.changes["architecture"], dict):
self.pkg.changes["architecture"] = { "Unknown" : "" }
# and maintainer2047 may not exist.
(self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
- self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], changes["maintainer2047"])
+ self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
else:
self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
- if "sponsoremail" in self.pkg.changes:
- self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
+ # Process policy doesn't set the fingerprint field and I don't want to make it
+ # do it for now as I don't want to have to deal with the case where we accepted
+ # the package into PU-NEW, but the fingerprint has gone away from the keyring in
+ # the meantime so the package will be remarked as rejectable. Urgh.
+ # TODO: Fix this properly
+ if self.pkg.changes.has_key('fingerprint'):
+ session = DBConn().session()
+ fpr = get_fingerprint(self.pkg.changes['fingerprint'], session)
+ if fpr and self.check_if_upload_is_sponsored("%s@debian.org" % fpr.uid.uid, fpr.uid.name):
+ if self.pkg.changes.has_key("sponsoremail"):
+ self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
+ session.close()
if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
self.Subst["__REJECT_MESSAGE__"] = self.package_info()
self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
+ self.Subst["__SUITE__"] = ", ".join(self.pkg.changes["distribution"])
###########################################################################
def load_changes(self, filename):
"""
- @rtype boolean
- @rvalue: whether the changes file was valid or not. We may want to
+ Load a changes file and setup a dictionary around it. Also checks for mandantory
+ fields within.
+
+ @type filename: string
+ @param filename: Changes filename, full path.
+
+ @rtype: boolean
+ @return: whether the changes file was valid or not. We may want to
reject even if this is True (see what gets put in self.rejects).
This is simply to prevent us even trying things later which will
fail because we couldn't properly parse the file.
except CantOpenError:
self.rejects.append("%s: can't read file." % (filename))
return False
- except ParseChangesError, line:
+ except ParseChangesError as line:
self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
return False
except ChangesUnicodeError:
# Parse the Files field from the .changes into another dictionary
try:
self.pkg.files.update(utils.build_file_list(self.pkg.changes))
- except ParseChangesError, line:
+ except ParseChangesError as line:
self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
return False
- except UnknownFormatError, format:
+ except UnknownFormatError as format:
self.rejects.append("%s: unknown format '%s'." % (filename, format))
return False
self.pkg.changes["maintainername"],
self.pkg.changes["maintaineremail"]) = \
fix_maintainer (self.pkg.changes["maintainer"])
- except ParseMaintError, msg:
+ except ParseMaintError as msg:
self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
- % (filename, changes["maintainer"], msg))
+ % (filename, self.pkg.changes["maintainer"], msg))
# ...likewise for the Changed-By: field if it exists.
try:
self.pkg.changes["changedbyname"],
self.pkg.changes["changedbyemail"]) = \
fix_maintainer (self.pkg.changes.get("changed-by", ""))
- except ParseMaintError, msg:
+ except ParseMaintError as msg:
self.pkg.changes["changedby822"] = ""
self.pkg.changes["changedby2047"] = ""
self.pkg.changes["changedbyname"] = ""
self.pkg.changes["changedbyemail"] = ""
self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
- % (filename, changes["changed-by"], msg))
+ % (filename, self.pkg.changes["changed-by"], msg))
# Ensure all the values in Closes: are numbers
if self.pkg.changes.has_key("closes"):
self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
- # Check there isn't already a changes file of the same name in one
- # of the queue directories.
- base_filename = os.path.basename(filename)
- for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
- if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
- self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
-
# Check the .changes is non-empty
if not self.pkg.files:
- self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
+ self.rejects.append("%s: nothing to do (Files field is empty)." % (os.path.basename(self.pkg.changes_file)))
return False
# Changes was syntactically valid even if we'll reject
(source, dest) = args[1:3]
if self.pkg.changes["distribution"].has_key(source):
for arch in self.pkg.changes["architecture"].keys():
- if arch not in [ arch_string for a in get_suite_architectures(source) ]:
+ if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
del self.pkg.changes["distribution"][source]
self.pkg.changes["distribution"][dest] = 1
# Ensure target distributions exist
for suite in self.pkg.changes["distribution"].keys():
- if not Cnf.has_key("Suite::%s" % (suite)):
+ if not get_suite(suite.lower()):
self.rejects.append("Unknown distribution `%s'." % (suite))
###########################################################################
try:
control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
except:
- self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
+ self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_info()[0]))
deb_file.close()
# Can't continue, none of the checks on control would work.
return
self.rejects.append("%s: invalid version number '%s'." % (f, version))
# Ensure the architecture of the .deb is one we know about.
- default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
+ default_suite = cnf.get("Dinstall::DefaultSuite", "unstable")
architecture = control.Find("Architecture")
upload_suite = self.pkg.changes["distribution"].keys()[0]
- if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
- and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
+ if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session = session)] \
+ and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session = session)]:
self.rejects.append("Unknown architecture '%s'." % (architecture))
# Ensure the architecture of the .deb is one of the ones
if not re_valid_pkg_name.match(prov):
self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
+ # If there is a Built-Using field, we need to check we can find the
+ # exact source version
+ built_using = control.Find("Built-Using")
+ if built_using:
+ try:
+ entry["built-using"] = []
+ for dep in apt_pkg.parse_depends(built_using):
+ bu_s, bu_v, bu_e = dep[0]
+ # Check that it's an exact match dependency and we have
+ # some form of version
+ if bu_e != "=" or len(bu_v) < 1:
+ self.rejects.append("%s: Built-Using contains non strict dependency (%s %s %s)" % (f, bu_s, bu_e, bu_v))
+ else:
+ # Find the source id for this version
+ bu_so = get_sources_from_name(bu_s, version=bu_v, session = session)
+ if len(bu_so) != 1:
+ self.rejects.append("%s: Built-Using (%s = %s): Cannot find source package" % (f, bu_s, bu_v))
+ else:
+ entry["built-using"].append( (bu_so[0].source, bu_so[0].version, ) )
+
+ except ValueError as e:
+ self.rejects.append("%s: Cannot parse Built-Using field: %s" % (f, str(e)))
+
+
# Check the section & priority match those given in the .changes (non-fatal)
if control.Find("Section") and entry["section"] != "" \
and entry["section"] != control.Find("Section"):
(source_version, f, self.pkg.changes["version"]))
else:
# Check in the SQL database
- if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
+ if not source_exists(source_package, source_version, suites = \
+ self.pkg.changes["distribution"].keys(), session = session):
# Check in one of the other directories
source_epochless_version = re_no_epoch.sub('', source_version)
dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
+
+ byhand_dir = get_policy_queue('byhand', session).path
+ new_dir = get_policy_queue('new', session).path
+
+ if os.path.exists(os.path.join(byhand_dir, dsc_filename)):
entry["byhand"] = 1
- elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
+ elif os.path.exists(os.path.join(new_dir, dsc_filename)):
entry["new"] = 1
else:
dsc_file_exists = False
- for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
- if cnf.has_key("Dir::Queue::%s" % (myq)):
- if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
+ # TODO: Don't hardcode this list: use all relevant queues
+ # The question is how to determine what is relevant
+ for queue_name in ["embargoed", "unembargoed", "proposedupdates", "oldproposedupdates"]:
+ queue = get_policy_queue(queue_name, session)
+ if queue:
+ if os.path.exists(os.path.join(queue.path, dsc_filename)):
dsc_file_exists = True
break
# Check the version and for file overwrites
self.check_binary_against_db(f, session)
- b = Binary(f)
- b.scan_package()
- if len(b.rejects) > 0:
- for j in b.rejects:
- self.rejects.append(j)
-
def source_file_checks(self, f, session):
entry = self.pkg.files[f]
self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
# Ensure the source version matches the version in the .changes file
- if entry["type"] == "orig.tar.gz":
+ if re_is_orig_source.match(f):
changes_version = self.pkg.changes["chopversion2"]
else:
changes_version = self.pkg.changes["chopversion"]
def per_suite_file_checks(self, f, suite, session):
cnf = Config()
entry = self.pkg.files[f]
- archive = utils.where_am_i()
# Skip byhand
if entry.has_key("byhand"):
return
+ # Check we have fields we need to do these checks
+ oktogo = True
+ for m in ['component', 'package', 'priority', 'size', 'md5sum']:
+ if not entry.has_key(m):
+ self.rejects.append("file '%s' does not have field %s set" % (f, m))
+ oktogo = False
+
+ if not oktogo:
+ return
+
# Handle component mappings
for m in cnf.ValueList("ComponentMappings"):
(source, dest) = m.split()
entry["component"] = dest
# Ensure the component is valid for the target suite
- if cnf.has_key("Suite:%s::Components" % (suite)) and \
- entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
+ if entry["component"] not in get_component_names(session):
self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
return
# Validate the component
- component = entry["component"]
- if not get_component(component, session):
- self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
+ if not get_component(entry["component"], session):
+ self.rejects.append("file '%s' has unknown component '%s'." % (f, entry["component"]))
return
# See if the package is NEW
# Determine the location
location = cnf["Dir::Pool"]
- l = get_location(location, entry["component"], archive, session)
+ l = get_location(location, entry["component"], session=session)
if l is None:
- self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
+ self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s)" % entry["component"])
entry["location id"] = -1
else:
entry["location id"] = l.location_id
# Check for packages that have moved from one component to another
entry['suite'] = suite
- res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
- if res.rowcount > 0:
- entry["othercomponents"] = res.fetchone()[0]
+ arch_list = [entry["architecture"], 'all']
+ component = get_component_by_package_suite(self.pkg.files[f]['package'], \
+ [suite], arch_list = arch_list, session = session)
+ if component is not None:
+ entry["othercomponents"] = component
def check_files(self, action=True):
- archive = utils.where_am_i()
file_keys = self.pkg.files.keys()
holding = Holding()
cnf = Config()
- # XXX: As far as I can tell, this can no longer happen - see
- # comments by AJ in old revisions - mhy
- # if reprocess is 2 we've already done this and we're checking
- # things again for the new .orig.tar.gz.
- # [Yes, I'm fully aware of how disgusting this is]
- if action and self.reprocess < 2:
+ if action:
cwd = os.getcwd()
os.chdir(self.pkg.directory)
for f in file_keys:
ret = holding.copy_to_holding(f)
if ret is not None:
- # XXX: Should we bail out here or try and continue?
- self.rejects.append(ret)
+ self.warnings.append('Could not copy %s to holding; will attempt to find in DB later' % f)
os.chdir(cwd)
- # Check there isn't already a .changes or .dak file of the same name in
- # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
+ # check we already know the changes file
# [NB: this check must be done post-suite mapping]
base_filename = os.path.basename(self.pkg.changes_file)
- dot_dak_filename = base_filename[:-8] + ".dak"
- for suite in self.pkg.changes["distribution"].keys():
- copychanges = "Suite::%s::CopyChanges" % (suite)
- if cnf.has_key(copychanges) and \
- os.path.exists(os.path.join(cnf[copychanges], base_filename)):
- self.rejects.append("%s: a file with this name already exists in %s" \
- % (base_filename, cnf[copychanges]))
-
- copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
- if cnf.has_key(copy_dot_dak) and \
- os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
- self.rejects.append("%s: a file with this name already exists in %s" \
- % (dot_dak_filename, Cnf[copy_dot_dak]))
-
- self.reprocess = 0
+ session = DBConn().session()
+
+ try:
+ dbc = session.query(DBChange).filter_by(changesname=base_filename).one()
+ # if in the pool or in a queue other than unchecked, reject
+ if (dbc.in_queue is None) \
+ or (dbc.in_queue is not None
+ and dbc.in_queue.queue_name not in ["unchecked", "newstage"]):
+ self.rejects.append("%s file already known to dak" % base_filename)
+ except NoResultFound as e:
+ # not known, good
+ pass
+
has_binaries = False
has_source = False
- session = DBConn().session()
-
for f, entry in self.pkg.files.items():
# Ensure the file does not already exist in one of the accepted directories
- for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
- if not cnf.has_key("Dir::Queue::%s" % (d)): continue
- if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
- self.rejects.append("%s file already exists in the %s directory." % (f, d))
+ # TODO: Dynamically generate this list
+ for queue_name in [ "byhand", "new", "proposedupdates", "oldproposedupdates", "embargoed", "unembargoed" ]:
+ queue = get_policy_queue(queue_name, session)
+ if queue and os.path.exists(os.path.join(queue.path, f)):
+ self.rejects.append("%s file already exists in the %s queue." % (f, queue_name))
if not re_taint_free.match(f):
self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
if os.path.exists(f):
self.rejects.append("Can't read `%s'. [permission denied]" % (f))
else:
- self.rejects.append("Can't read `%s'. [file not found]" % (f))
+ # Don't directly reject, mark to check later to deal with orig's
+ # we can find in the pool
+ self.later_check_files.append(f)
entry["type"] = "unreadable"
continue
for suite in self.pkg.changes["distribution"].keys():
self.per_suite_file_checks(f, suite, session)
+ session.close()
+
# If the .changes file says it has source, it must have source.
if self.pkg.changes["architecture"].has_key("source"):
if not has_source:
self.rejects.append("no source found and Architecture line in changes mention source.")
- if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
+ if (not has_binaries) and (not cnf.FindB("Dinstall::AllowSourceOnlyUploads")):
self.rejects.append("source only uploads are not supported.")
###########################################################################
- def check_dsc(self, action=True):
- """Returns bool indicating whether or not the source changes are valid"""
- # Ensure there is source to check
- if not self.pkg.changes["architecture"].has_key("source"):
- return True
- # Find the .dsc
+ def __dsc_filename(self):
+ """
+ Returns: (Status, Dsc_Filename)
+ where
+ Status: Boolean; True when there was no error, False otherwise
+ Dsc_Filename: String; name of the dsc file if Status is True, reason for the error otherwise
+ """
dsc_filename = None
- for f, entry in self.pkg.files.items():
- if entry["type"] == "dsc":
+
+ # find the dsc
+ for name, entry in self.pkg.files.items():
+ if entry.has_key("type") and entry["type"] == "dsc":
if dsc_filename:
- self.rejects.append("can not process a .changes file with multiple .dsc's.")
- return False
+ return False, "cannot process a .changes file with multiple .dsc's."
else:
- dsc_filename = f
+ dsc_filename = name
- # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
if not dsc_filename:
- self.rejects.append("source uploads must contain a dsc file")
- return False
+ return False, "source uploads must contain a dsc file"
+
+ return True, dsc_filename
+
+ def load_dsc(self, action=True, signing_rules=1):
+ """
+ Find and load the dsc from self.pkg.files into self.dsc
+
+ Returns: (Status, Reason)
+ where
+ Status: Boolean; True when there was no error, False otherwise
+ Reason: String; When Status is False this describes the error
+ """
+
+ # find the dsc
+ (status, dsc_filename) = self.__dsc_filename()
+ if not status:
+ # If status is false, dsc_filename has the reason
+ return False, dsc_filename
- # Parse the .dsc file
try:
- self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
+ self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=signing_rules, dsc_file=1))
except CantOpenError:
- # if not -n copy_to_holding() will have done this for us...
if not action:
- self.rejects.append("%s: can't read file." % (dsc_filename))
- except ParseChangesError, line:
- self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
- except InvalidDscError, line:
- self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
+ return False, "%s: can't read file." % (dsc_filename)
+ except ParseChangesError as line:
+ return False, "%s: parse error, can't grok: %s." % (dsc_filename, line)
+ except InvalidDscError as line:
+ return False, "%s: syntax error on line %s." % (dsc_filename, line)
except ChangesUnicodeError:
- self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
+ return False, "%s: dsc file not proper utf-8." % (dsc_filename)
+
+ return True, None
+
+ ###########################################################################
+
+ def check_dsc(self, action=True, session=None):
+ """Returns bool indicating whether or not the source changes are valid"""
+ # Ensure there is source to check
+ if not self.pkg.changes["architecture"].has_key("source"):
+ return True
+
+ if session is None:
+ session = DBConn().session()
+
+ (status, reason) = self.load_dsc(action=action)
+ if not status:
+ self.rejects.append(reason)
+ return False
+ (status, dsc_filename) = self.__dsc_filename()
+ if not status:
+ # If status is false, dsc_filename has the reason
+ self.rejects.append(dsc_filename)
+ return False
# Build up the file list of files mentioned by the .dsc
try:
except NoFilesFieldError:
self.rejects.append("%s: no Files: field." % (dsc_filename))
return False
- except UnknownFormatError, format:
+ except UnknownFormatError as format:
self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
return False
- except ParseChangesError, line:
+ except ParseChangesError as line:
self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
return False
if not re_valid_version.match(self.pkg.dsc["version"]):
self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
- # Bumping the version number of the .dsc breaks extraction by stable's
- # dpkg-source. So let's not do that...
- if self.pkg.dsc["format"] != "1.0":
- self.rejects.append("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename))
+ # Only a limited list of source formats are allowed in each suite
+ for dist in self.pkg.changes["distribution"].keys():
+ suite = get_suite(dist, session=session)
+ if not suite:
+ self.rejects.append("%s: cannot find suite %s when checking source formats" % (dsc_filename, dist))
+ continue
+ allowed = [ x.format_name for x in suite.srcformats ]
+ if self.pkg.dsc["format"] not in allowed:
+ self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
# Validate the Maintainer field
try:
# We ignore the return value
fix_maintainer(self.pkg.dsc["maintainer"])
- except ParseMaintError, msg:
+ except ParseMaintError as msg:
self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
% (dsc_filename, self.pkg.dsc["maintainer"], msg))
for field_name in [ "build-depends", "build-depends-indep" ]:
field = self.pkg.dsc.get(field_name)
if field:
- # Check for broken dpkg-dev lossage...
- if field.startswith("ARRAY"):
- self.rejects.append("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % \
- (dsc_filename, field_name.title()))
-
# Have apt try to parse them...
try:
apt_pkg.ParseSrcDepends(field)
if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
- # Ensure there is a .tar.gz in the .dsc file
- has_tar = False
- for f in self.pkg.dsc_files.keys():
- m = re_issource.match(f)
- if not m:
- self.rejects.append("%s: %s in Files field not recognised as source." % (dsc_filename, f))
- continue
- ftype = m.group(3)
- if ftype == "orig.tar.gz" or ftype == "tar.gz":
- has_tar = True
-
- if not has_tar:
- self.rejects.append("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
+ # Ensure the Files field contain only what's expected
+ self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
# Ensure source is newer than existing source in target suites
- self.check_source_against_db(dsc_filename)
+ session = DBConn().session()
+ self.check_source_against_db(dsc_filename, session)
+ self.check_dsc_against_db(dsc_filename, session)
- self.check_dsc_against_db(dsc_filename)
+ dbchg = get_dbchange(self.pkg.changes_file, session)
- return True
+ # Finally, check if we're missing any files
+ for f in self.later_check_files:
+ print 'XXX: %s' % f
+ # Check if we've already processed this file if we have a dbchg object
+ ok = False
+ if dbchg:
+ for pf in dbchg.files:
+ if pf.filename == f and pf.processed:
+ self.notes.append('%s was already processed so we can go ahead' % f)
+ ok = True
+ del self.pkg.files[f]
+ if not ok:
+ self.rejects.append("Could not find file %s references in changes" % f)
+
+ session.close()
+
+ return (len(self.rejects) == 0)
###########################################################################
if not os.path.exists(src):
return
ftype = m.group(3)
- if ftype == "orig.tar.gz" and self.pkg.orig_tar_gz:
+ if re_is_orig_source.match(f) and self.pkg.orig_files.has_key(f) and \
+ self.pkg.orig_files[f].has_key("path"):
continue
dest = os.path.join(os.getcwd(), f)
os.symlink(src, dest)
- # If the orig.tar.gz is not a part of the upload, create a symlink to the
- # existing copy.
- if self.pkg.orig_tar_gz:
- dest = os.path.join(os.getcwd(), os.path.basename(self.pkg.orig_tar_gz))
- os.symlink(self.pkg.orig_tar_gz, dest)
+ # If the orig files are not a part of the upload, create symlinks to the
+ # existing copies.
+ for orig_file in self.pkg.orig_files.keys():
+ if not self.pkg.orig_files[orig_file].has_key("path"):
+ continue
+ dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
+ os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
# Extract the source
- cmd = "dpkg-source -sn -x %s" % (dsc_filename)
- (result, output) = commands.getstatusoutput(cmd)
- if (result != 0):
- self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
- self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
+ try:
+ unpacked = UnpackedSource(dsc_filename)
+ except Exception as e:
+ self.rejects.append("'dpkg-source -x' failed for %s. (%s)" % (dsc_filename, str(e)))
return
- if not cnf.Find("Dir::Queue::BTSVersionTrack"):
+ if not cnf.Find("Dir::BTSVersionTrack"):
return
# Get the upstream version
upstr_version = re_strip_revision.sub('', upstr_version)
# Ensure the changelog file exists
- changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
- if not os.path.exists(changelog_filename):
+ changelog_file = unpacked.get_changelog_file()
+ if changelog_file is None:
self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
return
# Parse the changelog
self.pkg.dsc["bts changelog"] = ""
- changelog_file = utils.open_file(changelog_filename)
for line in changelog_file.readlines():
m = re_changelog_versions.match(line)
if m:
self.pkg.dsc["bts changelog"] += line
changelog_file.close()
+ unpacked.cleanup()
# Check we found at least one revision in the changelog
if not self.pkg.dsc["bts changelog"]:
self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
def check_source(self):
- # XXX: I'm fairly sure reprocess == 2 can never happen
- # AJT disabled the is_incoming check years ago - mhy
- # We should probably scrap or rethink the whole reprocess thing
# Bail out if:
# a) there's no source
- # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
- # or c) the orig.tar.gz is MIA
- if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
- or self.pkg.orig_tar_gz == -1:
+ if not self.pkg.changes["architecture"].has_key("source"):
return
tmpdir = utils.temp_dirname()
try:
shutil.rmtree(tmpdir)
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EACCES:
print "foobar"
utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
if result != 0:
utils.fubar("'%s' failed with result %s." % (cmd, result))
shutil.rmtree(tmpdir)
- except Exception, e:
+ except Exception as e:
print "foobar2 (%s)" % e
utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
self.ensure_hashes()
+ ###########################################################################
+
+ def ensure_orig(self, target_dir='.', session=None):
+ """
+ Ensures that all orig files mentioned in the changes file are present
+ in target_dir. If they do not exist, they are symlinked into place.
+
+ An list containing the symlinks that were created are returned (so they
+ can be removed).
+ """
+
+ symlinked = []
+ cnf = Config()
+
+ for filename, entry in self.pkg.dsc_files.iteritems():
+ if not re_is_orig_source.match(filename):
+ # File is not an orig; ignore
+ continue
+
+ if os.path.exists(filename):
+ # File exists, no need to continue
+ continue
+
+ def symlink_if_valid(path):
+ f = utils.open_file(path)
+ md5sum = apt_pkg.md5sum(f)
+ f.close()
+
+ fingerprint = (os.stat(path)[stat.ST_SIZE], md5sum)
+ expected = (int(entry['size']), entry['md5sum'])
+
+ if fingerprint != expected:
+ return False
+
+ dest = os.path.join(target_dir, filename)
+
+ os.symlink(path, dest)
+ symlinked.append(dest)
+
+ return True
+
+ session_ = session
+ if session is None:
+ session_ = DBConn().session()
+
+ found = False
+
+ # Look in the pool
+ for poolfile in get_poolfile_like_name('%s' % filename, session_):
+ poolfile_path = os.path.join(
+ poolfile.location.path, poolfile.filename
+ )
+
+ if symlink_if_valid(poolfile_path):
+ found = True
+ break
+
+ if session is None:
+ session_.close()
+
+ if found:
+ continue
+
+ # Look in some other queues for the file
+ queue_names = ['new', 'byhand',
+ 'proposedupdates', 'oldproposedupdates',
+ 'embargoed', 'unembargoed']
+
+ for queue_name in queue_names:
+ queue = get_policy_queue(queue_name, session)
+ if not queue:
+ continue
+
+ queuefile_path = os.path.join(queue.path, filename)
+
+ if not os.path.exists(queuefile_path):
+ # Does not exist in this queue
+ continue
+
+ if symlink_if_valid(queuefile_path):
+ break
+
+ return symlinked
+
+ ###########################################################################
+
+ def check_lintian(self):
+ """
+ Extends self.rejects by checking the output of lintian against tags
+ specified in Dinstall::LintianTags.
+ """
+
+ cnf = Config()
+
+ # Don't reject binary uploads
+ if not self.pkg.changes['architecture'].has_key('source'):
+ return
+
+ # Only check some distributions
+ for dist in ('unstable', 'experimental'):
+ if dist in self.pkg.changes['distribution']:
+ break
+ else:
+ return
+
+ # If we do not have a tagfile, don't do anything
+ tagfile = cnf.get("Dinstall::LintianTags")
+ if not tagfile:
+ return
+
+ # Parse the yaml file
+ sourcefile = file(tagfile, 'r')
+ sourcecontent = sourcefile.read()
+ sourcefile.close()
+
+ try:
+ lintiantags = yaml.load(sourcecontent)['lintian']
+ except yaml.YAMLError as msg:
+ utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
+ return
+
+ # Try and find all orig mentioned in the .dsc
+ symlinked = self.ensure_orig()
+
+ # Setup the input file for lintian
+ fd, temp_filename = utils.temp_filename()
+ temptagfile = os.fdopen(fd, 'w')
+ for tags in lintiantags.values():
+ temptagfile.writelines(['%s\n' % x for x in tags])
+ temptagfile.close()
+
+ try:
+ cmd = "lintian --show-overrides --tags-from-file %s %s" % \
+ (temp_filename, self.pkg.changes_file)
+
+ result, output = commands.getstatusoutput(cmd)
+ finally:
+ # Remove our tempfile and any symlinks we created
+ os.unlink(temp_filename)
+
+ for symlink in symlinked:
+ os.unlink(symlink)
+
+ if result == 2:
+ utils.warn("lintian failed for %s [return code: %s]." % \
+ (self.pkg.changes_file, result))
+ utils.warn(utils.prefix_multi_line_string(output, \
+ " [possible output:] "))
+
+ def log(*txt):
+ if self.logger:
+ self.logger.log(
+ [self.pkg.changes_file, "check_lintian"] + list(txt)
+ )
+
+ # Generate messages
+ parsed_tags = parse_lintian_output(output)
+ self.rejects.extend(
+ generate_reject_messages(parsed_tags, lintiantags, log=log)
+ )
+
###########################################################################
def check_urgency(self):
cnf = Config()
if entry["type"] == "deb":
tar.reset()
try:
- deb_file = utils.open_file(filename)
- apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
- deb_file.seek(0)
- try:
- apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
- except SystemError, e:
- # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
- if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
- raise
- deb_file.seek(0)
- apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
-
- deb_file.close()
+ deb = apt_inst.DebFile(filename)
+ deb.control.go(tar.callback)
future_files = tar.future_files.keys()
if future_files:
self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
% (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
except:
- self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
+ self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_info()[0], sys.exc_info()[1]))
+ def check_if_upload_is_sponsored(self, uid_email, uid_name):
+ uid_email = '@'.join(uid_email.split('@')[:2])
+ if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
+ sponsored = False
+ elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
+ sponsored = False
+ if uid_name == "":
+ sponsored = True
+ else:
+ sponsored = True
+ sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
+ debian_emails = filter(lambda addr: addr.endswith('@debian.org'), sponsor_addresses)
+ if uid_email not in debian_emails:
+ if debian_emails:
+ uid_email = debian_emails[0]
+ if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
+ if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
+ self.pkg.changes["changedbyemail"] not in sponsor_addresses):
+ self.pkg.changes["sponsoremail"] = uid_email
+
+ return sponsored
+
+
+ ###########################################################################
+ # check_signed_by_key checks
###########################################################################
+
def check_signed_by_key(self):
"""Ensure the .changes is signed by an authorized uploader."""
session = DBConn().session()
- (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
-
- # match claimed name with actual name:
- if uid is None:
- # This is fundamentally broken but need us to refactor how we get
- # the UIDs/Fingerprints in order for us to fix it properly
- uid, uid_email = self.pkg.changes["fingerprint"], uid
- may_nmu, may_sponsor = 1, 1
- # XXX by default new dds don't have a fingerprint/uid in the db atm,
- # and can't get one in there if we don't allow nmu/sponsorship
- elif is_dm is False:
- # If is_dm is False, we allow full upload rights
- uid_email = "%s@debian.org" % (uid)
- may_nmu, may_sponsor = 1, 1
- else:
- # Assume limited upload rights unless we've discovered otherwise
- uid_email = uid
- may_nmu, may_sponsor = 0, 0
+ # First of all we check that the person has proper upload permissions
+ # and that this upload isn't blocked
+ fpr = get_fingerprint(self.pkg.changes['fingerprint'], session=session)
- if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
- sponsored = 0
- elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
- sponsored = 0
- if uid_name == "": sponsored = 1
+ if fpr is None:
+ self.rejects.append("Cannot find fingerprint %s" % self.pkg.changes["fingerprint"])
+ return
+
+ # TODO: Check that import-keyring adds UIDs properly
+ if not fpr.uid:
+ self.rejects.append("Cannot find uid for fingerprint %s. Please contact ftpmaster@debian.org" % fpr.fingerprint)
+ return
+
+ # Check that the fingerprint which uploaded has permission to do so
+ self.check_upload_permissions(fpr, session)
+
+ # Check that this package is not in a transition
+ self.check_transition(session)
+
+ session.close()
+
+
+ def check_upload_permissions(self, fpr, session):
+ # Check any one-off upload blocks
+ self.check_upload_blocks(fpr, session)
+
+ # If the source_acl is None, source is never allowed
+ if fpr.source_acl is None:
+ if self.pkg.changes["architecture"].has_key("source"):
+ rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
+ rej += '\nPlease contact ftpmaster if you think this is incorrect'
+ self.rejects.append(rej)
+ return
+ # Do DM as a special case
+ # DM is a special case unfortunately, so we check it first
+ # (keys with no source access get more access than DMs in one
+ # way; DMs can only upload for their packages whether source
+ # or binary, whereas keys with no access might be able to
+ # upload some binaries)
+ elif fpr.source_acl.access_level == 'dm':
+ self.check_dm_upload(fpr, session)
else:
- sponsored = 1
- if ("source" in self.pkg.changes["architecture"] and
- uid_email and utils.is_email_alias(uid_email)):
- sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
- if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
- self.pkg.changes["changedbyemail"] not in sponsor_addresses):
- self.pkg.changes["sponsoremail"] = uid_email
-
- if sponsored and not may_sponsor:
- self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
-
- if not sponsored and not may_nmu:
- should_reject = True
- highest_sid, highest_version = None, None
-
- # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
- # It ignores higher versions with the dm_upload_allowed flag set to false
- # I'm keeping the existing behaviour for now until I've gone back and
- # checked exactly what the GR says - mhy
- for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
- if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
- highest_sid = si.source_id
- highest_version = si.version
-
- if highest_sid is None:
- self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
+ # If not a DM, we allow full upload rights
+ uid_email = "%s@debian.org" % (fpr.uid.uid)
+ self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
+
+
+ # Check binary upload permissions
+ # By this point we know that DMs can't have got here unless they
+ # are allowed to deal with the package concerned so just apply
+ # normal checks
+ if fpr.binary_acl.access_level == 'full':
+ return
+
+ # Otherwise we're in the map case
+ tmparches = self.pkg.changes["architecture"].copy()
+ tmparches.pop('source', None)
+
+ for bam in fpr.binary_acl_map:
+ tmparches.pop(bam.architecture.arch_string, None)
+
+ if len(tmparches.keys()) > 0:
+ if fpr.binary_reject:
+ rej = "changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
+ if len(tmparches.keys()) == 1:
+ rej += "\n\narchitecture involved is: %s" % ",".join(tmparches.keys())
+ else:
+ rej += "\n\narchitectures involved are: %s" % ",".join(tmparches.keys())
+ self.rejects.append(rej)
else:
- for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
- (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
- if email == uid_email or name == uid_name:
- should_reject = False
- break
-
- if should_reject is True:
- self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
-
- for b in self.pkg.changes["binary"].keys():
- for suite in self.pkg.changes["distribution"].keys():
- q = session.query(DBSource)
- q = q.join(DBBinary).filter_by(package=b)
- q = q.join(BinAssociation).join(Suite).filter_by(suite)
-
- for s in q.all():
- if s.source != self.pkg.changes["source"]:
- self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
-
- for f in self.pkg.files.keys():
- if self.pkg.files[f].has_key("byhand"):
- self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
- if self.pkg.files[f].has_key("new"):
- self.rejects.append("%s may not upload NEW file %s" % (uid, f))
+ # TODO: This is where we'll implement reject vs throw away binaries later
+ rej = "Uhm. I'm meant to throw away the binaries now but that's not implemented yet"
+ rej += "\nPlease complain to ftpmaster@debian.org as this shouldn't have been turned on"
+ rej += "\nFingerprint: %s", (fpr.fingerprint)
+ self.rejects.append(rej)
+
+
+ def check_upload_blocks(self, fpr, session):
+ """Check whether any upload blocks apply to this source, source
+ version, uid / fpr combination"""
+
+ def block_rej_template(fb):
+ rej = 'Manual upload block in place for package %s' % fb.source
+ if fb.version is not None:
+ rej += ', version %s' % fb.version
+ return rej
+
+ for fb in session.query(UploadBlock).filter_by(source = self.pkg.changes['source']).all():
+ # version is None if the block applies to all versions
+ if fb.version is None or fb.version == self.pkg.changes['version']:
+ # Check both fpr and uid - either is enough to cause a reject
+ if fb.fpr is not None:
+ if fb.fpr.fingerprint == fpr.fingerprint:
+ self.rejects.append(block_rej_template(fb) + ' for fingerprint %s\nReason: %s' % (fpr.fingerprint, fb.reason))
+ if fb.uid is not None:
+ if fb.uid == fpr.uid:
+ self.rejects.append(block_rej_template(fb) + ' for uid %s\nReason: %s' % (fb.uid.uid, fb.reason))
+
+
+ def check_dm_upload(self, fpr, session):
+ # Quoth the GR (http://www.debian.org/vote/2007/vote_003):
+ ## none of the uploaded packages are NEW
+ rej = False
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f].has_key("byhand"):
+ self.rejects.append("%s may not upload BYHAND file %s" % (fpr.uid.uid, f))
+ rej = True
+ if self.pkg.files[f].has_key("new"):
+ self.rejects.append("%s may not upload NEW file %s" % (fpr.uid.uid, f))
+ rej = True
+
+ if rej:
+ return
+
+ r = get_newest_source(self.pkg.changes["source"], session)
+
+ if r is None:
+ rej = "Could not find existing source package %s in unstable or experimental and this is a DM upload" % self.pkg.changes["source"]
+ self.rejects.append(rej)
+ return
+
+ if not r.dm_upload_allowed:
+ rej = "Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version (%s)" % (self.pkg.changes["source"], r.version)
+ self.rejects.append(rej)
+ return
+
+ ## the Maintainer: field of the uploaded .changes file corresponds with
+ ## the owner of the key used (ie, non-developer maintainers may not sponsor
+ ## uploads)
+ if self.check_if_upload_is_sponsored(fpr.uid.uid, fpr.uid.name):
+ self.rejects.append("%s (%s) is not authorised to sponsor uploads" % (fpr.uid.uid, fpr.fingerprint))
+
+ ## the most recent version of the package uploaded to unstable or
+ ## experimental lists the uploader in the Maintainer: or Uploaders: fields (ie,
+ ## non-developer maintainers cannot NMU or hijack packages)
+
+ # uploader includes the maintainer
+ accept = False
+ for uploader in r.uploaders:
+ (rfc822, rfc2047, name, email) = uploader.get_split_maintainer()
+ # Eww - I hope we never have two people with the same name in Debian
+ if email == fpr.uid.uid or name == fpr.uid.name:
+ accept = True
+ break
+
+ if not accept:
+ self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (fpr.uid.uid, self.pkg.changes["source"]))
+ return
+
+ ## none of the packages are being taken over from other source packages
+ for b in self.pkg.changes["binary"].keys():
+ for suite in self.pkg.changes["distribution"].keys():
+ for s in get_source_by_package_and_suite(b, suite, session):
+ if s.source != self.pkg.changes["source"]:
+ self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (fpr.uid.uid, b, s, suite))
+
+
+
+ def check_transition(self, session):
+ cnf = Config()
+
+ sourcepkg = self.pkg.changes["source"]
+
+ # No sourceful upload -> no need to do anything else, direct return
+ # We also work with unstable uploads, not experimental or those going to some
+ # proposed-updates queue
+ if "source" not in self.pkg.changes["architecture"] or \
+ "unstable" not in self.pkg.changes["distribution"]:
+ return
+
+ # Also only check if there is a file defined (and existant) with
+ # checks.
+ transpath = cnf.get("Dinstall::ReleaseTransitions", "")
+ if transpath == "" or not os.path.exists(transpath):
+ return
+
+ # Parse the yaml file
+ sourcefile = file(transpath, 'r')
+ sourcecontent = sourcefile.read()
+ try:
+ transitions = yaml.load(sourcecontent)
+ except yaml.YAMLError as msg:
+ # This shouldn't happen, there is a wrapper to edit the file which
+ # checks it, but we prefer to be safe than ending up rejecting
+ # everything.
+ utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
+ return
+
+ # Now look through all defined transitions
+ for trans in transitions:
+ t = transitions[trans]
+ source = t["source"]
+ expected = t["new"]
+
+ # Will be None if nothing is in testing.
+ current = get_source_in_suite(source, "testing", session)
+ if current is not None:
+ compare = apt_pkg.VersionCompare(current.version, expected)
+
+ if current is None or compare < 0:
+ # This is still valid, the current version in testing is older than
+ # the new version we wait for, or there is none in testing yet
+
+ # Check if the source we look at is affected by this.
+ if sourcepkg in t['packages']:
+ # The source is affected, lets reject it.
+
+ rejectmsg = "%s: part of the %s transition.\n\n" % (
+ sourcepkg, trans)
+
+ if current is not None:
+ currentlymsg = "at version %s" % (current.version)
+ else:
+ currentlymsg = "not present in testing"
+
+ rejectmsg += "Transition description: %s\n\n" % (t["reason"])
+
+ rejectmsg += "\n".join(textwrap.wrap("""Your package
+is part of a testing transition designed to get %s migrated (it is
+currently %s, we need version %s). This transition is managed by the
+Release Team, and %s is the Release-Team member responsible for it.
+Please mail debian-release@lists.debian.org or contact %s directly if you
+need further assistance. You might want to upload to experimental until this
+transition is done."""
+ % (source, currentlymsg, expected,t["rm"], t["rm"])))
+
+ self.rejects.append(rejectmsg)
+ return
###########################################################################
+ # End check_signed_by_key checks
+ ###########################################################################
+
def build_summaries(self):
""" Build a summary of changes the upload introduces. """
for bug in bugs:
summary += "%s " % (bug)
if action:
+ self.update_subst()
self.Subst["__BUG_NUMBER__"] = bug
if self.pkg.changes["distribution"].has_key("stable"):
self.Subst["__STABLE_WARNING__"] = """
distribution."""
else:
self.Subst["__STABLE_WARNING__"] = ""
- mail_message = utils.TemplateSubst(self.Subst, template)
- utils.send_mail(mail_message)
+ mail_message = utils.TemplateSubst(self.Subst, template)
+ utils.send_mail(mail_message)
# Clear up after ourselves
del self.Subst["__BUG_NUMBER__"]
"""
cnf = Config()
- announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
+
+ # Skip all of this if not sending mail to avoid confusing people
+ if cnf.has_key("Dinstall::Options::No-Mail") and cnf["Dinstall::Options::No-Mail"]:
+ return ""
# Only do announcements for source uploads with a recent dpkg-dev installed
if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
self.pkg.changes["architecture"].has_key("source"):
return ""
- lists_done = {}
- summary = ""
+ announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
- self.Subst["__SHORT_SUMMARY__"] = short_summary
+ lists_todo = {}
+ summary = ""
+ # Get a unique list of target lists
for dist in self.pkg.changes["distribution"].keys():
- announce_list = cnf.Find("Suite::%s::Announce" % (dist))
- if announce_list == "" or lists_done.has_key(announce_list):
- continue
+ suite = get_suite(dist)
+ if suite is None: continue
+ for tgt in suite.announce:
+ lists_todo[tgt] = 1
+
+ self.Subst["__SHORT_SUMMARY__"] = short_summary
- lists_done[announce_list] = 1
+ for announce_list in lists_todo.keys():
summary += "Announcing to %s\n" % (announce_list)
if action:
+ self.update_subst()
self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
if cnf.get("Dinstall::TrackingServer") and \
self.pkg.changes["architecture"].has_key("source"):
del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
- if cnf.FindB("Dinstall::CloseBugs"):
+ if cnf.FindB("Dinstall::CloseBugs") and cnf.has_key("Dinstall::BugServer"):
summary = self.close_bugs(summary, action)
del self.Subst["__SHORT_SUMMARY__"]
return summary
###########################################################################
-
- def accept (self, summary, short_summary, targetdir=None):
+ @session_wrapper
+ def accept (self, summary, short_summary, session=None):
"""
Accept an upload.
- This moves all files referenced from the .changes into the I{accepted}
- queue, sends the accepted mail, announces to lists, closes bugs and
+ This moves all files referenced from the .changes into the pool,
+ sends the accepted mail, announces to lists, closes bugs and
also checks for override disparities. If enabled it will write out
the version history for the BTS Version Tracking and will finally call
L{queue_build}.
@type short_summary: string
@param short_summary: Short summary
-
"""
cnf = Config()
stats = SummaryStats()
- accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
+ print "Installing."
+ self.logger.log(["installing changes", self.pkg.changes_file])
- if targetdir is None:
- targetdir = cnf["Dir::Queue::Accepted"]
+ binaries = []
+ poolfiles = []
- print "Accepting."
- if self.logger:
- self.logger.log(["Accepting changes", self.pkg.changes_file])
+ # Add the .dsc file to the DB first
+ for newfile, entry in self.pkg.files.items():
+ if entry["type"] == "dsc":
+ source, dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
+ for j in pfs:
+ poolfiles.append(j)
- self.pkg.write_dot_dak(targetdir)
+ # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
+ for newfile, entry in self.pkg.files.items():
+ if entry["type"] == "deb":
+ b, pf = add_deb_to_db(self, newfile, session)
+ binaries.append(b)
+ poolfiles.append(pf)
+
+ # If this is a sourceful diff only upload that is moving
+ # cross-component we need to copy the .orig files into the new
+ # component too for the same reasons as above.
+ # XXX: mhy: I think this should be in add_dsc_to_db
+ if self.pkg.changes["architecture"].has_key("source"):
+ for orig_file in self.pkg.orig_files.keys():
+ if not self.pkg.orig_files[orig_file].has_key("id"):
+ continue # Skip if it's not in the pool
+ orig_file_id = self.pkg.orig_files[orig_file]["id"]
+ if self.pkg.orig_files[orig_file]["location"] == dsc_location_id:
+ continue # Skip if the location didn't change
- # Move all the files into the accepted directory
- utils.move(self.pkg.changes_file, targetdir)
+ # Do the move
+ oldf = get_poolfile_by_id(orig_file_id, session)
+ old_filename = os.path.join(oldf.location.path, oldf.filename)
+ old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
+ 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
- for name, entry in sorted(self.pkg.files.items()):
- utils.move(name, targetdir)
+ new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
+
+ # TODO: Care about size/md5sum collisions etc
+ (found, newf) = check_poolfile(new_filename, old_dat['size'], old_dat['md5sum'], dsc_location_id, session)
+
+ # TODO: Uhm, what happens if newf isn't None - something has gone badly and we should cope
+ if newf is None:
+ utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
+ newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
+
+ session.flush()
+
+ # Don't reference the old file from this changes
+ for p in poolfiles:
+ if p.file_id == oldf.file_id:
+ poolfiles.remove(p)
+
+ poolfiles.append(newf)
+
+ # Fix up the DSC references
+ toremove = []
+
+ for df in source.srcfiles:
+ if df.poolfile.file_id == oldf.file_id:
+ # Add a new DSC entry and mark the old one for deletion
+ # Don't do it in the loop so we don't change the thing we're iterating over
+ newdscf = DSCFile()
+ newdscf.source_id = source.source_id
+ newdscf.poolfile_id = newf.file_id
+ session.add(newdscf)
+
+ toremove.append(df)
+
+ for df in toremove:
+ session.delete(df)
+
+ # Flush our changes
+ session.flush()
+
+ # Make sure that our source object is up-to-date
+ session.expire(source)
+
+ # Add changelog information to the database
+ self.store_changelog()
+
+ # Install the files into the pool
+ for newfile, entry in self.pkg.files.items():
+ destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
+ utils.move(newfile, destination)
+ self.logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
stats.accept_bytes += float(entry["size"])
- stats.accept_count += 1
+ # Copy the .changes file across for suite which need it.
+ copy_changes = dict([(x.copychanges, '')
+ for x in session.query(Suite).filter(Suite.suite_name.in_(self.pkg.changes["distribution"].keys())).all()
+ if x.copychanges is not None])
- # Send accept mail, announce to lists, close bugs and check for
- # override disparities
- if not cnf["Dinstall::Options::No-Mail"]:
- self.Subst["__SUITE__"] = ""
- self.Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
- utils.send_mail(mail_message)
- self.announce(short_summary, 1)
+ for dest in copy_changes.keys():
+ utils.copy(self.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
- ## Helper stuff for DebBugs Version Tracking
- if cnf.Find("Dir::Queue::BTSVersionTrack"):
- # ??? once queue/* is cleared on *.d.o and/or reprocessed
- # the conditionalization on dsc["bts changelog"] should be
- # dropped.
+ # We're done - commit the database changes
+ session.commit()
+ # Our SQL session will automatically start a new transaction after
+ # the last commit
+
+ # Now ensure that the metadata has been added
+ # This has to be done after we copy the files into the pool
+ # For source if we have it:
+ if self.pkg.changes["architecture"].has_key("source"):
+ import_metadata_into_db(source, session)
+
+ # Now for any of our binaries
+ for b in binaries:
+ import_metadata_into_db(b, session)
+
+ session.commit()
+
+ # Move the .changes into the 'done' directory
+ ye, mo, da = time.gmtime()[0:3]
+ donedir = os.path.join(cnf["Dir::Done"], str(ye), "%0.2d" % mo, "%0.2d" % da)
+ if not os.path.isdir(donedir):
+ os.makedirs(donedir)
+
+ utils.move(self.pkg.changes_file,
+ os.path.join(donedir, os.path.basename(self.pkg.changes_file)))
+
+ if self.pkg.changes["architecture"].has_key("source"):
+ UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])
- # Write out the version history from the changelog
- if self.pkg.changes["architecture"].has_key("source") and \
- self.pkg.dsc.has_key("bts changelog"):
+ self.update_subst()
+ self.Subst["__SUMMARY__"] = summary
+ mail_message = utils.TemplateSubst(self.Subst,
+ os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted'))
+ utils.send_mail(mail_message)
+ self.announce(short_summary, 1)
- (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ ## Helper stuff for DebBugs Version Tracking
+ if cnf.Find("Dir::BTSVersionTrack"):
+ if self.pkg.changes["architecture"].has_key("source"):
+ (fd, temp_filename) = utils.temp_filename(cnf["Dir::BTSVersionTrack"], prefix=".")
version_history = os.fdopen(fd, 'w')
version_history.write(self.pkg.dsc["bts changelog"])
version_history.close()
- filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+ filename = "%s/%s" % (cnf["Dir::BTSVersionTrack"],
self.pkg.changes_file[:-8]+".versions")
os.rename(temp_filename, filename)
- os.chmod(filename, 0644)
+ os.chmod(filename, 0o644)
# Write out the binary -> source mapping.
- (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ (fd, temp_filename) = utils.temp_filename(cnf["Dir::BTSVersionTrack"], prefix=".")
debinfo = os.fdopen(fd, 'w')
for name, entry in sorted(self.pkg.files.items()):
if entry["type"] == "deb":
entry["source version"]])
debinfo.write(line+"\n")
debinfo.close()
- filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+ filename = "%s/%s" % (cnf["Dir::BTSVersionTrack"],
self.pkg.changes_file[:-8]+".debinfo")
os.rename(temp_filename, filename)
- os.chmod(filename, 0644)
-
- # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
- # <Ganneff> we do call queue_build too
- # <mhy> well yes, we'd have had to if we were inserting into accepted
- # <Ganneff> now. thats database only.
- # <mhy> urgh, that's going to get messy
- # <Ganneff> so i make the p-n call to it *also* using accepted/
- # <mhy> but then the packages will be in the queue_build table without the files being there
- # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
- # <mhy> ah, good point
- # <Ganneff> so it will work out, as unchecked move it over
- # <mhy> that's all completely sick
- # <Ganneff> yes
-
- # This routine returns None on success or an error on failure
- res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
- if res:
- utils.fubar(res)
+ os.chmod(filename, 0o644)
+
+ session.commit()
+
+ # Set up our copy queues (e.g. buildd queues)
+ for suite_name in self.pkg.changes["distribution"].keys():
+ suite = get_suite(suite_name, session)
+ for q in suite.copy_queues:
+ for f in poolfiles:
+ q.add_file_from_pool(f)
+
+ session.commit()
+ # Finally...
+ stats.accept_count += 1
def check_override(self):
"""
cnf = Config()
- # Abandon the check if:
- # a) override disparity checks have been disabled
- # b) we're not sending mail
- if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
- cnf["Dinstall::Options::No-Mail"]:
+ # Abandon the check if override disparity checks have been disabled
+ if not cnf.FindB("Dinstall::OverrideDisparityCheck"):
return
summary = self.pkg.check_override()
overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
+ self.update_subst()
self.Subst["__SUMMARY__"] = summary
mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
utils.send_mail(mail_message)
###########################################################################
- def remove(self, dir=None):
+ def remove(self, from_dir=None):
"""
Used (for instance) in p-u to remove the package from unchecked
+
+ Also removes the package from holding area.
"""
- if dir is None:
- os.chdir(self.pkg.directory)
- else:
- os.chdir(dir)
+ if from_dir is None:
+ from_dir = self.pkg.directory
+ h = Holding()
for f in self.pkg.files.keys():
- os.unlink(f)
- os.unlink(self.pkg.changes_file)
+ os.unlink(os.path.join(from_dir, f))
+ if os.path.exists(os.path.join(h.holding_dir, f)):
+ os.unlink(os.path.join(h.holding_dir, f))
+
+ os.unlink(os.path.join(from_dir, self.pkg.changes_file))
+ if os.path.exists(os.path.join(h.holding_dir, self.pkg.changes_file)):
+ os.unlink(os.path.join(h.holding_dir, self.pkg.changes_file))
###########################################################################
- def move_to_dir (self, dest, perms=0660, changesperms=0664):
+ def move_to_queue (self, queue):
"""
- Move files to dest with certain perms/changesperms
+ Move files to a destination queue using the permissions in the table
"""
- utils.move(self.pkg.changes_file, dest, perms=changesperms)
+ h = Holding()
+ utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
+ queue.path, perms=int(queue.change_perms, 8))
for f in self.pkg.files.keys():
- utils.move(f, dest, perms=perms)
+ utils.move(os.path.join(h.holding_dir, f), queue.path, perms=int(queue.perms, 8))
###########################################################################
directory it will be moved to the morgue to make way for
the new file.
- @type files: dict
- @param files: file dictionary
+ @type reject_files: dict
+ @param reject_files: file dictionary
"""
if os.access(file_entry, os.R_OK) == 0:
continue
- dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
+ dest_file = os.path.join(cnf["Dir::Reject"], file_entry)
try:
- dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
- except OSError, e:
- # File exists? Let's try and move it to the morgue
+ dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o644)
+ except OSError as e:
+ # File exists? Let's find a new name by adding a number
if e.errno == errno.EEXIST:
- morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
try:
- morgue_file = utils.find_next_free(morgue_file)
+ dest_file = utils.find_next_free(dest_file, 255)
except NoFreeFilenameError:
# Something's either gone badly Pete Tong, or
# someone is trying to exploit us.
- utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
+ utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Reject"]))
return
- utils.move(dest_file, morgue_file, perms=0660)
+
+ # Make sure we really got it
try:
- dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
- except OSError, e:
+ dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0o644)
+ except OSError as e:
# Likewise
utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
return
raise
# If we got here, we own the destination file, so we can
# safely overwrite it.
- utils.move(file_entry, dest_file, 1, perms=0660)
+ utils.move(file_entry, dest_file, 1, perms=0o660)
os.close(dest_fd)
###########################################################################
- def do_reject (self, manual=0, reject_message="", note=""):
+ def do_reject (self, manual=0, reject_message="", notes=""):
"""
Reject an upload. If called without a reject message or C{manual} is
true, spawn an editor so the user can write one.
if manual and not reject_message:
(fd, temp_filename) = utils.temp_filename()
temp_file = os.fdopen(fd, 'w')
- if len(note) > 0:
- for line in note:
- temp_file.write(line)
+ if len(notes) > 0:
+ for note in notes:
+ temp_file.write("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+ % (note.author, note.version, note.notedate, note.comment))
temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
cnf = Config()
reason_filename = self.pkg.changes_file[:-8] + ".reason"
- reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
+ reason_filename = os.path.join(cnf["Dir::Reject"], reason_filename)
# Move all the files into the reject directory
reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
# so let's just raise an exception ...
if os.path.exists(reason_filename):
os.unlink(reason_filename)
- reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0o644)
rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
+ self.update_subst()
if not manual:
self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
- self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
+ self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)"
os.write(reason_fd, reject_message)
reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
else:
user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
- self.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__REJECT_MESSAGE__"] = ""
+ self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
# Write the rejection email out as the <foo>.reason file
os.write(reason_fd, reject_mail_message)
os.close(reason_fd)
- # Send the rejection mail if appropriate
- if not cnf["Dinstall::Options::No-Mail"]:
- utils.send_mail(reject_mail_message)
+ # Send the rejection mail
+ utils.send_mail(reject_mail_message)
if self.logger:
self.logger.log(["rejected", self.pkg.changes_file])
+ stats = SummaryStats()
+ stats.reject_count += 1
return 0
################################################################################
- def in_override_p(self, package, component, suite, binary_type, file, session=None):
+ def in_override_p(self, package, component, suite, binary_type, filename, session):
"""
Check if a package already has override entries in the DB
@type binary_type: string
@param binary_type: type of the package
- @type file: string
- @param file: filename we check
+ @type filename: string
+ @param filename: filename we check
@return: the database result. But noone cares anyway.
cnf = Config()
- if session is None:
- session = DBConn().session()
-
if binary_type == "": # must be source
file_type = "dsc"
else:
file_type = binary_type
# Override suite name; used for example with proposed-updates
- if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
- suite = cnf["Suite::%s::OverrideSuite" % (suite)]
+ oldsuite = get_suite(suite, session)
+ if (not oldsuite is None) and oldsuite.overridesuite:
+ suite = oldsuite.overridesuite
result = get_override(package, suite, component, file_type, session)
# Remember the section and priority so we can check them later if appropriate
if len(result) > 0:
result = result[0]
- self.pkg.files[file]["override section"] = result.section.section
- self.pkg.files[file]["override priority"] = result.priority.priority
+ self.pkg.files[filename]["override section"] = result.section.section
+ self.pkg.files[filename]["override priority"] = result.priority.priority
return result
return None
Description: TODO
"""
+ Cnf = Config()
anyversion = None
- anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
+ anysuite = [suite] + [ vc.reference.suite_name for vc in get_version_checks(suite, "Enhances") ]
for (s, v) in sv_list:
if s in [ x.lower() for x in anysuite ]:
if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
################################################################################
- def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
+ def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
"""
@type sv_list: list
@param sv_list: list of (suite, version) tuples to check
- @type file: string
- @param file: XXX
+ @type filename: string
+ @param filename: XXX
@type new_version: string
@param new_version: XXX
# Check versions for each target suite
for target_suite in self.pkg.changes["distribution"].keys():
- must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
- must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+ # Check we can find the target suite
+ ts = get_suite(target_suite)
+ if ts is None:
+ self.rejects.append("Cannot find target suite %s to perform version checks" % target_suite)
+ continue
+
+ must_be_newer_than = [ vc.reference.suite_name for vc in get_version_checks(target_suite, "MustBeNewerThan") ]
+ must_be_older_than = [ vc.reference.suite_name for vc in get_version_checks(target_suite, "MustBeOlderThan") ]
# Enforce "must be newer than target suite" even if conffile omits it
if target_suite not in must_be_newer_than:
vercmp = apt_pkg.VersionCompare(new_version, existent_version)
if suite in must_be_newer_than and sourceful and vercmp < 1:
- self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
if suite in must_be_older_than and vercmp > -1:
cansave = 0
self.rejects.append("Won't propogate NEW packages.")
elif apt_pkg.VersionCompare(new_version, add_version) < 0:
# propogation would be redundant. no need to reject though.
- self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
cansave = 1
elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
apt_pkg.VersionCompare(add_version, target_version) >= 0:
cansave = 1
if not cansave:
- self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
################################################################################
- def check_binary_against_db(self, file, session=None):
- if session is None:
- session = DBConn().session()
-
+ def check_binary_against_db(self, filename, session):
# Ensure version is sane
- q = session.query(BinAssociation)
- q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
- q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
-
- self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
- file, self.pkg.files[file]["version"], sourceful=False)
+ self.cross_suite_version_check( \
+ get_suite_version_by_package(self.pkg.files[filename]["package"], \
+ self.pkg.files[filename]["architecture"], session),
+ filename, self.pkg.files[filename]["version"], sourceful=False)
# Check for any existing copies of the file
- q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
- q = q.filter_by(version=self.pkg.files[file]["version"])
- q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
+ q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"])
+ q = q.filter_by(version=self.pkg.files[filename]["version"])
+ q = q.join(Architecture).filter_by(arch_string=self.pkg.files[filename]["architecture"])
if q.count() > 0:
- self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
+ self.rejects.append("%s: can not overwrite existing copy already in the archive." % filename)
################################################################################
- def check_source_against_db(self, file, session=None):
- """
- """
- if session is None:
- session = DBConn().session()
-
+ def check_source_against_db(self, filename, session):
source = self.pkg.dsc.get("source")
version = self.pkg.dsc.get("version")
# Ensure version is sane
- q = session.query(SrcAssociation)
- q = q.join(DBSource).filter(DBSource.source==source)
-
- self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
- file, version, sourceful=True)
+ self.cross_suite_version_check( \
+ get_suite_version_by_source(source, session), filename, version,
+ sourceful=True)
################################################################################
- def check_dsc_against_db(self, file, session=None):
+ def check_dsc_against_db(self, filename, session):
"""
@warning: NB: this function can remove entries from the 'files' index [if
- the .orig.tar.gz is a duplicate of the one in the archive]; if
+ the orig tarball is a duplicate of the one in the archive]; if
you're iterating over 'files' and call this function as part of
the loop, be sure to add a check to the top of the loop to
ensure you haven't just tried to dereference the deleted entry.
"""
- if session is None:
- session = DBConn().session()
-
- self.pkg.orig_tar_gz = None
+ Cnf = Config()
+ self.pkg.orig_files = {} # XXX: do we need to clear it?
+ orig_files = self.pkg.orig_files
# Try and find all files mentioned in the .dsc. This has
# to work harder to cope with the multiple possible
found = "%s in incoming" % (dsc_name)
# Check the file does not already exist in the archive
- ql = get_poolfile_like_name(dsc_name)
+ ql = get_poolfile_like_name(dsc_name, session)
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
if len(ql) > 0:
# Ignore exact matches for .orig.tar.gz
match = 0
- if dsc_name.endswith(".orig.tar.gz"):
+ if re_is_orig_source.match(dsc_name):
for i in ql:
if self.pkg.files.has_key(dsc_name) and \
int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
# This would fix the stupidity of changing something we often iterate over
# whilst we're doing it
del self.pkg.files[dsc_name]
- self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename)
+ dsc_entry["files id"] = i.file_id
+ if not orig_files.has_key(dsc_name):
+ orig_files[dsc_name] = {}
+ orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
match = 1
+ # Don't bitch that we couldn't find this file later
+ try:
+ self.later_check_files.remove(dsc_name)
+ except ValueError:
+ pass
+
+
if not match:
self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
- elif dsc_name.endswith(".orig.tar.gz"):
+ elif re_is_orig_source.match(dsc_name):
# Check in the pool
ql = get_poolfile_like_name(dsc_name, session)
old_file_fh.close()
actual_size = os.stat(old_file)[stat.ST_SIZE]
found = old_file
- suite_type = f.location.archive_type
+ suite_type = x.location.archive_type
# need this for updating dsc_files in install()
- dsc_entry["files id"] = f.file_id
+ dsc_entry["files id"] = x.file_id
# See install() in process-accepted...
- self.pkg.orig_tar_id = f.file_id
- self.pkg.orig_tar_gz = old_file
- self.pkg.orig_tar_location = f.location.location_id
+ if not orig_files.has_key(dsc_name):
+ orig_files[dsc_name] = {}
+ orig_files[dsc_name]["id"] = x.file_id
+ orig_files[dsc_name]["path"] = old_file
+ orig_files[dsc_name]["location"] = x.location.location_id
else:
- # TODO: Record the queues and info in the DB so we don't hardcode all this crap
+ # TODO: Determine queue list dynamically
# Not there? Check the queue directories...
- for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
- in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (directory)], dsc_name)
+ for queue_name in [ "byhand", "new", "proposedupdates", "oldproposedupdates", "embargoed", "unembargoed" ]:
+ queue = get_policy_queue(queue_name, session)
+ if not queue:
+ continue
+
+ in_otherdir = os.path.join(queue.path, dsc_name)
+
if os.path.exists(in_otherdir):
in_otherdir_fh = utils.open_file(in_otherdir)
actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
in_otherdir_fh.close()
actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
found = in_otherdir
- self.pkg.orig_tar_gz = in_otherdir
+ if not orig_files.has_key(dsc_name):
+ orig_files[dsc_name] = {}
+ orig_files[dsc_name]["path"] = in_otherdir
if not found:
- self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
- self.pkg.orig_tar_gz = -1
+ self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (filename, dsc_name))
continue
else:
- self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
+ self.rejects.append("%s refers to %s, but I can't find it in the queue." % (filename, dsc_name))
continue
if actual_md5 != dsc_entry["md5sum"]:
- self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
+ self.rejects.append("md5sum for %s doesn't match %s." % (found, filename))
if actual_size != int(dsc_entry["size"]):
- self.rejects.append("size for %s doesn't match %s." % (found, file))
+ self.rejects.append("size for %s doesn't match %s." % (found, filename))
################################################################################
- def accepted_checks(self, overwrite_checks=True, session=None):
+ # This is used by process-new and process-holding to recheck a changes file
+ # at the time we're running. It mainly wraps various other internal functions
+ # and is similar to accepted_checks - these should probably be tidied up
+ # and combined
+ def recheck(self, session):
+ cnf = Config()
+ for f in self.pkg.files.keys():
+ # The .orig.tar.gz can disappear out from under us is it's a
+ # duplicate of one in the archive.
+ if not self.pkg.files.has_key(f):
+ continue
+
+ entry = self.pkg.files[f]
+
+ # Check that the source still exists
+ if entry["type"] == "deb":
+ source_version = entry["source version"]
+ source_package = entry["source package"]
+ if not self.pkg.changes["architecture"].has_key("source") \
+ and not source_exists(source_package, source_version, \
+ suites = self.pkg.changes["distribution"].keys(), session = session):
+ source_epochless_version = re_no_epoch.sub('', source_version)
+ dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
+ found = False
+ for queue_name in ["embargoed", "unembargoed", "newstage"]:
+ queue = get_policy_queue(queue_name, session)
+ if queue and os.path.exists(os.path.join(queue.path, dsc_filename)):
+ found = True
+ if not found:
+ self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
+
+ # Version and file overwrite checks
+ if entry["type"] == "deb":
+ self.check_binary_against_db(f, session)
+ elif entry["type"] == "dsc":
+ self.check_source_against_db(f, session)
+ self.check_dsc_against_db(f, session)
+
+ ################################################################################
+ def accepted_checks(self, overwrite_checks, session):
# Recheck anything that relies on the database; since that's not
# frozen between accept and our run time when called from p-a.
# overwrite_checks is set to False when installing to stable/oldstable
- if session is None:
- session = DBConn().session()
-
propogate={}
nopropogate={}
+ # Find the .dsc (again)
+ dsc_filename = None
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f]["type"] == "dsc":
+ dsc_filename = f
+
for checkfile in self.pkg.files.keys():
# The .orig.tar.gz can disappear out from under us is it's a
# duplicate of one in the archive.
source_version = entry["source version"]
source_package = entry["source package"]
if not self.pkg.changes["architecture"].has_key("source") \
- and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
+ and not source_exists(source_package, source_version, \
+ suites = self.pkg.changes["distribution"].keys(), \
+ session = session):
self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
# Version and file overwrite checks
# propogate in the case it is in the override tables:
for suite in self.pkg.changes.get("propdistribution", {}).keys():
- if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile):
+ if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
propogate[suite] = 1
else:
nopropogate[suite] = 1
for checkfile in self.pkg.files.keys():
# Check the package is still in the override tables
for suite in self.pkg.changes["distribution"].keys():
- if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile):
+ if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
self.rejects.append("%s is NEW for %s." % (checkfile, suite))
- ################################################################################
- # This is not really a reject, but an unaccept, but since a) the code for
- # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
- # extremely rare, for now we'll go with whining at our admin folks...
-
- def do_unaccept(self):
- cnf = Config()
-
- self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
- self.Subst["__REJECT_MESSAGE__"] = self.package_info()
- self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
- self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
- if cnf.has_key("Dinstall::Bcc"):
- self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
-
- template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
-
- reject_mail_message = utils.TemplateSubst(self.Subst, template)
-
- # Write the rejection email out as the <foo>.reason file
- reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
- reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
-
- # If we fail here someone is probably trying to exploit the race
- # so let's just raise an exception ...
- if os.path.exists(reject_filename):
- os.unlink(reject_filename)
-
- fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
- os.write(fd, reject_mail_message)
- os.close(fd)
-
- utils.send_mail(reject_mail_message)
-
- del self.Subst["__REJECTOR_ADDRESS__"]
- del self.Subst["__REJECT_MESSAGE__"]
- del self.Subst["__CC__"]
-
################################################################################
# If any file of an upload has a recent mtime then chances are good
# the file is still being uploaded.
os.chdir(cwd)
return too_new
+
+ def store_changelog(self):
+
+ # Skip binary-only upload if it is not a bin-NMU
+ if not self.pkg.changes['architecture'].has_key('source'):
+ from daklib.regexes import re_bin_only_nmu
+ if not re_bin_only_nmu.search(self.pkg.changes['version']):
+ return
+
+ session = DBConn().session()
+
+ # Check if upload already has a changelog entry
+ query = """SELECT changelog_id FROM changes WHERE source = :source
+ AND version = :version AND architecture = :architecture AND changelog_id != 0"""
+ if session.execute(query, {'source': self.pkg.changes['source'], \
+ 'version': self.pkg.changes['version'], \
+ 'architecture': " ".join(self.pkg.changes['architecture'].keys())}).rowcount:
+ session.commit()
+ return
+
+ # Add current changelog text into changelogs_text table, return created ID
+ query = "INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id"
+ ID = session.execute(query, {'changelog': self.pkg.changes['changes']}).fetchone()[0]
+
+ # Link ID to the upload available in changes table
+ query = """UPDATE changes SET changelog_id = :id WHERE source = :source
+ AND version = :version AND architecture = :architecture"""
+ session.execute(query, {'id': ID, 'source': self.pkg.changes['source'], \
+ 'version': self.pkg.changes['version'], \
+ 'architecture': " ".join(self.pkg.changes['architecture'].keys())})
+
+ session.commit()