#!/usr/bin/env python
-""" Checks Debian packages from Incoming """
-# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+"""
+Checks Debian packages from Incoming
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback
-import apt_inst, apt_pkg
-from daklib import database
-from daklib import logging
+import commands
+import errno
+import fcntl
+import os
+import re
+import shutil
+import stat
+import sys
+import time
+import traceback
+import tarfile
+import apt_inst
+import apt_pkg
+from debian_bundle import deb822
+from daklib.dbconn import *
+from daklib.binary import Binary
+from daklib import daklog
from daklib import queue
from daklib import utils
+from daklib.textutils import fix_maintainer
from daklib.dak_exceptions import *
from daklib.regexes import re_valid_version, re_valid_pkg_name, re_changelog_versions, \
re_strip_revision, re_strip_srcver, re_spacestrip, \
('n',"no-action","Dinstall::Options::No-Action"),
('p',"no-lock", "Dinstall::Options::No-Lock"),
('s',"no-mail", "Dinstall::Options::No-Mail"),
- ('d',"directory", "Dinstall::Options::Directory")]
+ ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
"override-distribution", "version", "directory"]:
if len(changes_files) > 0:
utils.warn("Directory provided so ignoring files given on command line")
- dir = Cnf["Dinstall::Options::Directory"]
- try:
- # Much of the rest of p-u depends on being in the right place
- os.chdir(dir)
- changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')]
- except OSError, e:
- utils.fubar("Failed to read list from directory %s (%s)" % (dir, e))
+ changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
Upload = queue.Upload(Cnf)
try:
(changes["maintainer822"], changes["maintainer2047"],
changes["maintainername"], changes["maintaineremail"]) = \
- utils.fix_maintainer (changes["maintainer"])
+ fix_maintainer (changes["maintainer"])
except ParseMaintError, msg:
reject("%s: Maintainer field ('%s') failed to parse: %s" \
% (filename, changes["maintainer"], msg))
try:
(changes["changedby822"], changes["changedby2047"],
changes["changedbyname"], changes["changedbyemail"]) = \
- utils.fix_maintainer (changes.get("changed-by", ""))
+ fix_maintainer (changes.get("changed-by", ""))
except ParseMaintError, msg:
(changes["changedby822"], changes["changedby2047"],
changes["changedbyname"], changes["changedbyemail"]) = \
(source, dest) = args[1:3]
if changes["distribution"].has_key(source):
for arch in changes["architecture"].keys():
- if arch not in database.get_suite_architectures(source):
+ if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),"")
del changes["distribution"][source]
changes["distribution"][dest] = 1
################################################################################
-def check_deb_ar(filename):
- """
- Sanity check the ar of a .deb, i.e. that there is:
-
- 1. debian-binary
- 2. control.tar.gz
- 3. data.tar.gz or data.tar.bz2
-
- in that order, and nothing else.
- """
- cmd = "ar t %s" % (filename)
- (result, output) = commands.getstatusoutput(cmd)
- if result != 0:
- reject("%s: 'ar t' invocation failed." % (filename))
- reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
- chunks = output.split('\n')
- if len(chunks) != 3:
- reject("%s: found %d chunks, expected 3." % (filename, len(chunks)))
- if chunks[0] != "debian-binary":
- reject("%s: first chunk is '%s', expected 'debian-binary'." % (filename, chunks[0]))
- if chunks[1] != "control.tar.gz":
- reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (filename, chunks[1]))
- if chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
- reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (filename, chunks[2]))
-
-################################################################################
-
def check_files():
global reprocess
has_binaries = 0
has_source = 0
+ s = DBConn().session()
+
for f in file_keys:
# Ensure the file does not already exist in one of the accepted directories
for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
deb_file.close()
# Can't continue, none of the checks on control would work.
continue
+
+ # Check for mandantory "Description:"
+ deb_file.seek ( 0 )
+ try:
+ apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
+ except:
+ reject("%s: Missing Description in binary package" % (f))
+ continue
+
deb_file.close()
# Check for mandatory fields
default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable")
architecture = control.Find("Architecture")
upload_suite = changes["distribution"].keys()[0]
- if architecture not in database.get_suite_architectures(default_suite) and architecture not in database.get_suite_architectures(upload_suite):
+ if architecture not in [a.arch_string for a in get_suite_architectures(default_suite)] \
+ and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite)]:
reject("Unknown architecture '%s'." % (architecture))
# Ensure the architecture of the .deb is one of the ones
# Check the version and for file overwrites
reject(Upload.check_binary_against_db(f),"")
- check_deb_ar(f)
+ Binary(f, reject).scan_package()
# Checks for a source package...
else:
# Validate the component
component = files[f]["component"]
- component_id = database.get_component_id(component)
+ component_id = DBConn().get_component_id(component)
if component_id == -1:
reject("file '%s' has unknown component '%s'." % (f, component))
continue
# Determine the location
location = Cnf["Dir::Pool"]
- location_id = database.get_location_id (location, component, archive)
+ location_id = DBConn().get_location_id(location, component, archive)
if location_id == -1:
reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
files[f]["location id"] = location_id
# Check the md5sum & size against existing files (if any)
files[f]["pool name"] = utils.poolify (changes["source"], files[f]["component"])
- files_id = database.get_files_id(files[f]["pool name"] + f, files[f]["size"], files[f]["md5sum"], files[f]["location id"])
+ files_id = DBConn().get_files_id(files[f]["pool name"] + f, files[f]["size"], files[f]["md5sum"], files[f]["location id"])
if files_id == -1:
reject("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
elif files_id == -2:
files[f]["files id"] = files_id
# Check for packages that have moved from one component to another
- q = Upload.projectB.query("""
-SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
- component c, architecture a, files f
- WHERE b.package = '%s' AND s.suite_name = '%s'
- AND (a.arch_string = '%s' OR a.arch_string = 'all')
- AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
- AND f.location = l.id AND l.component = c.id AND b.file = f.id"""
- % (files[f]["package"], suite,
- files[f]["architecture"]))
- ql = q.getresult()
- if ql:
- files[f]["othercomponents"] = ql[0][0]
+ files[f]['suite'] = suite
+ ql = get_binary_components(files[f]['package'], suite, files[f][architecture])
+ if ql.rowcount > 0:
+ files[f]["othercomponents"] = ql.fetchone()[0]
# If the .changes file says it has source, it must have source.
if changes["architecture"].has_key("source"):
# Validate the Maintainer field
try:
- utils.fix_maintainer (dsc["maintainer"])
+ fix_maintainer (dsc["maintainer"])
except ParseMaintError, msg:
reject("%s: Maintainer field ('%s') failed to parse: %s" \
% (dsc_filename, dsc["maintainer"], msg))
or pkg.orig_tar_gz == -1:
return
- # Create a temporary directory to extract the source into
- if Options["No-Action"]:
- tmpdir = tempfile.mkdtemp()
- else:
- # We're in queue/holding and can create a random directory.
- tmpdir = "%s" % (os.getpid())
- os.mkdir(tmpdir)
+ tmpdir = utils.temp_dirname()
# Move into the temporary directory
cwd = os.getcwd()
################################################################################
def lookup_uid_from_fingerprint(fpr):
- q = Upload.projectB.query("SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
- qs = q.getresult()
- if len(qs) == 0:
- return (None, None, None)
- else:
- return qs[0]
+ uid = None
+ uid_name = ""
+ # This is a stupid default, but see the comments below
+ is_dm = False
+
+ user = get_uid_from_fingerprint(changes["fingerprint"])
+
+ if user is not None:
+ uid = user.uid
+ if user.name is None:
+ uid_name = ''
+ else:
+ uid_name = user.name
+
+ # Check the relevant fingerprint (which we have to have)
+ for f in uid.fingerprint:
+ if f.fingerprint == changes['fingerprint']:
+ is_dm = f.keyring.debian_maintainer
+ break
+
+ return (uid, uid_name, is_dm)
def check_signed_by_key():
"""Ensure the .changes is signed by an authorized uploader."""
+ session = DBConn().session()
- (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(changes["fingerprint"])
- if uid_name == None:
- uid_name = ""
+ (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(changes["fingerprint"], session=session)
# match claimed name with actual name:
- if uid == None:
+ if uid is None:
+ # This is fundamentally broken but need us to refactor how we get
+ # the UIDs/Fingerprints in order for us to fix it properly
uid, uid_email = changes["fingerprint"], uid
may_nmu, may_sponsor = 1, 1
# XXX by default new dds don't have a fingerprint/uid in the db atm,
# and can't get one in there if we don't allow nmu/sponsorship
- elif is_dm is "t":
- uid_email = uid
- may_nmu, may_sponsor = 0, 0
- else:
+ elif is_dm is False:
+ # If is_dm is False, we allow full upload rights
uid_email = "%s@debian.org" % (uid)
may_nmu, may_sponsor = 1, 1
+ else:
+ # Assume limited upload rights unless we've discovered otherwise
+ uid_email = uid
+ may_nmu, may_sponsor = 0, 0
+
if uid_email in [changes["maintaineremail"], changes["changedbyemail"]]:
sponsored = 0
reject("%s is not authorised to sponsor uploads" % (uid))
if not sponsored and not may_nmu:
- source_ids = []
- q = Upload.projectB.query("SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.dm_upload_allowed = 'yes'" % (changes["source"]))
-
- highest_sid, highest_version = None, None
-
should_reject = True
- for si in q.getresult():
- if highest_version == None or apt_pkg.VersionCompare(si[1], highest_version) == 1:
- highest_sid = si[0]
- highest_version = si[1]
+ highest_sid, highest_version = None, None
- if highest_sid == None:
- reject("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % changes["source"])
+ # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
+ # It ignores higher versions with the dm_upload_allowed flag set to false
+ # I'm keeping the existing behaviour for now until I've gone back and
+ # checked exactly what the GR says - mhy
+ for si in get_sources_from_name(source=changes['source'], dm_upload_allowed=True, session=session):
+ if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
+ highest_sid = si.source_id
+ highest_version = si.version
+
+ if highest_sid is None:
+ reject("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % changes["source"])
else:
- q = Upload.projectB.query("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT su.maintainer FROM src_uploaders su JOIN source s ON (s.id = su.source) WHERE su.source = %s)" % (highest_sid))
- for m in q.getresult():
- (rfc822, rfc2047, name, email) = utils.fix_maintainer(m[0])
+ for sup in s.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid)
+ (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
if email == uid_email or name == uid_name:
- should_reject=False
+ should_reject = False
break
- if should_reject == True:
+ if should_reject is True:
reject("%s is not in Maintainer or Uploaders of source package %s" % (uid, changes["source"]))
for b in changes["binary"].keys():
for suite in changes["distribution"].keys():
- suite_id = database.get_suite_id(suite)
- q = Upload.projectB.query("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = '%s' AND ba.suite = %s" % (b, suite_id))
- for s in q.getresult():
- if s[0] != changes["source"]:
+ q = session.query(DBSource)
+ q = q.join(DBBinary).filter_by(package=b)
+ q = q.join(BinAssociation).join(Suite).filter_by(suite)
+
+ for s in q.all():
+ if s.source != changes["source"]:
reject("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
for f in files.keys():
################################################################################
def is_unembargo ():
- q = Upload.projectB.query(
- "SELECT package FROM disembargo WHERE package = '%s' AND version = '%s'" %
- (changes["source"], changes["version"]))
- ql = q.getresult()
- if ql:
+ session = DBConn().session()
+ q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", changes)
+ if q.rowcount > 0:
return 1
oldcwd = os.getcwd()
if pkg.directory == disdir:
if changes["architecture"].has_key("source"):
- if Options["No-Action"]: return 1
+ if Options["No-Action"]:
+ return 1
+
+ session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", changes)
+ session.commit()
- Upload.projectB.query(
- "INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" %
- (changes["source"], changes["version"]))
return 1
return 0
return 0
if not changes["architecture"].has_key("source"):
- pusuite = database.get_suite_id("proposed-updates")
- q = Upload.projectB.query(
- "SELECT S.source FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.version = '%s' AND sa.suite = %d" %
- (changes["source"], changes["version"], pusuite))
- ql = q.getresult()
- if ql:
- # source is already in proposed-updates so no need to hold
+ s = DBConn().session()
+ q = s.query(SrcAssociation.sa_id)
+ q = q.join(Suite).filter_by(suite_name='proposed-updates')
+ q = q.join(DBSource).filter_by(source=changes['source'])
+ q = q.filter_by(version=changes['version']).limit(1)
+
+ if q.count() < 1:
return 0
return 1
return 0
if not changes["architecture"].has_key("source"):
- pusuite = database.get_suite_id("oldstable-proposed-updates")
- q = Upload.projectB.query(
- "SELECT S.source FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.version = '%s' AND sa.suite = %d" %
- (changes["source"], changes["version"], pusuite))
- ql = q.getresult()
- if ql:
- # source is already in oldstable-proposed-updates so no need to hold
+ s = DBConn().session()
+ q = s.query(SrcAssociation.sa_id)
+ q = q.join(Suite).filter_by(suite_name='oldstable-proposed-updates')
+ q = q.join(DBSource).filter_by(source=changes['source'])
+ q = q.filter_by(version=changes['version']).limit(1)
+
+ if q.count() < 1:
return 0
return 1
Logger.log(["Moving to new", pkg.changes_file])
Upload.dump_vars(Cnf["Dir::Queue::New"])
- move_to_dir(Cnf["Dir::Queue::New"])
+ move_to_dir(Cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
if not Options["No-Mail"]:
print "Sending new ack."
changes_files.remove(f)
if changes_files == []:
- utils.fubar("Need at least one .changes file as an argument.")
+ if Cnf["Dinstall::Options::Directory"] == "":
+ utils.fubar("Need at least one .changes file as an argument.")
+ else:
+ sys.exit(0)
# Check that we aren't going to clash with the daily cron job
utils.fubar("Couldn't obtain lock; assuming another 'dak process-unchecked' is already running.")
else:
raise
- Logger = Upload.Logger = logging.Logger(Cnf, "process-unchecked")
+ Logger = Upload.Logger = daklog.Logger(Cnf, "process-unchecked")
# debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
bcc = "X-DAK: dak process-unchecked\nX-Katie: $Revision: 1.65 $"
projects / dak.git / blobdiff