# I know what I say. I dont know python and I wrote it. So go and read some other stuff.
import commands
-import pg
import re
import sys
import time
import os
import apt_pkg
-from daklib import database
-from daklib import logging
-from daklib import queue
+
+from daklib import daklog
from daklib import utils
+from daklib.dbconn import DBConn, add_database_user, get_or_set_uid
from daklib.regexes import re_gpg_fingerprint, re_user_address, re_user_mails, re_user_name
################################################################################
Cnf = None
-projectB = None
Logger = None
-Upload = None
-Subst = None
################################################################################
################################################################################
def main():
- global Cnf, projectB
+ global Cnf
keyrings = None
Cnf = utils.get_conf()
if Options["help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
if not keyrings:
keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
utils.warn("Could not prepare password information for mail, not sending password.")
# Now add user to the database.
- projectB.query("BEGIN WORK")
- uid_id = database.get_or_set_uid_id(uid)
- projectB.query('CREATE USER "%s"' % (uid))
- projectB.query("COMMIT WORK")
+ # Note that we provide a session, so we're responsible for committing
+ uidobj = get_or_set_uid(uid, session=session)
+ uid_id = uidobj.uid_id
+ add_database_user(uid)
+ session.commit()
# The following two are kicked out in rhona, so we don't set them. kelly adds
# them as soon as she installs a package with unknown ones, so no problems to expect here.
# Just leave the comment in, to not think about "Why the hell aren't they added" in
# a year, if we ever touch uma again.
# maint_id = database.get_or_set_maintainer_id(name)
-# projectB.query("INSERT INTO fingerprint (fingerprint, uid) VALUES ('%s', '%s')" % (primary_key, uid_id))
+# session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)",
+# {'fingerprint': primary_key, 'uid': uid_id})
# Lets add user to the email-whitelist file if its configured.
if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "":
# Should we send mail to the newly added user?
if Cnf.FindB("Add-User::SendEmail"):
mail = name + "<" + emails[0] +">"
- Upload = queue.Upload(Cnf)
- Subst = Upload.Subst
+ Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
--- /dev/null
+#!/usr/bin/env python
+
+"""Configure dak parameters in the database"""
+# Copyright (C) 2009 Mark Hymers <mhy@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import sys
+
+import apt_pkg
+
+from daklib import utils
+from daklib.dbconn import *
+from daklib.config import Config
+
+################################################################################
+
+dispatch = {}
+dryrun = False
+
+################################################################################
+def warn(msg):
+ print >> sys.stderr, msg
+
+def die(msg, exit_code=1):
+ print >> sys.stderr, msg
+ sys.exit(exit_code)
+
+def die_arglen(args, args_needed, msg):
+ if len(args) < args_needed:
+ die(msg)
+
+def usage(exit_code=0):
+ """Perform administrative work on the dak database."""
+
+ print """Usage: dak admin COMMAND
+Perform administrative work on the dak database.
+
+ -h, --help show this help and exit.
+ -n, --dry-run don't do anything, just show what would have been done
+ (only applies to add or rm operations).
+
+ Commands can use a long or abbreviated form:
+
+ architecture / a:
+ a list show a list of architectures
+ a rm ARCH remove an architecture (will only work if
+ no longer linked to any suites)
+ a add ARCH DESCRIPTION [SUITELIST]
+ add architecture ARCH with DESCRIPTION.
+ If SUITELIST is given, add to each of the
+ suites at the same time
+
+ suite / s:
+ s list show a list of suites
+ s show SUITE show config details for a suite
+
+ suite-architecture / s-a:
+ s-a list-suite ARCH show the suites an ARCH is in
+ s-a list-arch SUITE show the architectures in a SUITE
+ s-a add SUITE ARCH add ARCH to suite
+ s-a rm SUITE ARCH remove ARCH from suite (will only work if
+ no packages remain for the arch in the suite)
+"""
+ sys.exit(exit_code)
+
+################################################################################
+
+def __architecture_list(d, args):
+ q = d.session().query(Architecture).order_by('arch_string')
+ for j in q.all():
+ # HACK: We should get rid of source from the arch table
+ if j.arch_string == 'source': continue
+ print j.arch_string
+ sys.exit(0)
+
+def __architecture_add(d, args):
+ die_arglen(args, 3, "E: adding an architecture requires a name and a description")
+ print "Adding architecture %s" % args[2]
+ suites = [str(x) for x in args[4:]]
+ print suites
+ if not dryrun:
+ try:
+ s = d.session()
+ a = Architecture()
+ a.arch_string = str(args[2]).lower()
+ a.description = str(args[3])
+ s.add(a)
+ for sn in suites:
+ su = get_suite(sn ,s)
+ if su is not None:
+ archsu = SuiteArchitecture()
+ archsu.arch_id = a.arch_id
+ archsu.suite_id = su.suite_id
+ s.add(archsu)
+ else:
+ warn("W: Cannot find suite %s" % su)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Integrity error adding architecture %s (it probably already exists)" % args[2])
+ except SQLAlchemyError, e:
+ die("E: Error adding architecture %s (%s)" % (args[2], e))
+ print "Architecture %s added" % (args[2])
+
+def __architecture_rm(d, args):
+ die_arglen(args, 3, "E: removing an architecture requires at least a name")
+ print "Removing architecture %s" % args[2]
+ if not dryrun:
+ try:
+ s = d.session()
+ a = get_architecture(args[2].lower(), s)
+ if a is None:
+ die("E: Cannot find architecture %s" % args[2])
+ s.delete(a)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Integrity error removing architecture %s (suite-arch entries probably still exist)" % args[2])
+ except SQLAlchemyError, e:
+ die("E: Error removing architecture %s (%s)" % (args[2], e))
+ print "Architecture %s removed" % args[2]
+
+def architecture(command):
+ args = [str(x) for x in command]
+ Cnf = utils.get_conf()
+ d = DBConn()
+
+ die_arglen(args, 2, "E: architecture needs at least a command")
+
+ mode = args[1].lower()
+ if mode == 'list':
+ __architecture_list(d, args)
+ elif mode == 'add':
+ __architecture_add(d, args)
+ elif mode == 'rm':
+ __architecture_rm(d, args)
+ else:
+ die("E: architecture command unknown")
+
+dispatch['architecture'] = architecture
+dispatch['a'] = architecture
+
+################################################################################
+
+def __suite_list(d, args):
+ s = d.session()
+ for j in s.query(Suite).order_by('suite_name').all():
+ print j.suite_name
+
+def __suite_show(d, args):
+ if len(args) < 2:
+ die("E: showing an suite entry requires a suite")
+
+ s = d.session()
+ su = get_suite(args[2].lower())
+ if su is None:
+ die("E: can't find suite entry for %s" % (args[2].lower()))
+
+ print su.details()
+
+def suite(command):
+ args = [str(x) for x in command]
+ Cnf = utils.get_conf()
+ d = DBConn()
+
+ die_arglen(args, 2, "E: suite needs at least a command")
+
+ mode = args[1].lower()
+
+ if mode == 'list':
+ __suite_list(d, args)
+ elif mode == 'show':
+ __suite_show(d, args)
+ else:
+ die("E: suite command unknown")
+
+dispatch['suite'] = suite
+dispatch['s'] = suite
+
+################################################################################
+
+def __suite_architecture_list(d, args):
+ s = d.session()
+ for j in s.query(Suite).order_by('suite_name').all():
+ print j.suite_name + ' ' + \
+ ','.join([a.architecture.arch_string for a in j.suitearchitectures])
+
+def __suite_architecture_listarch(d, args):
+ die_arglen(args, 3, "E: suite-architecture list-arch requires a suite")
+ a = get_suite_architectures(args[2].lower())
+ for j in a:
+ # HACK: We should get rid of source from the arch table
+ if j.arch_string != 'source':
+ print j.arch_string
+
+
+def __suite_architecture_listsuite(d, args):
+ die_arglen(args, 3, "E: suite-architecture list-suite requires an arch")
+ for j in get_architecture_suites(args[2].lower()):
+ print j.suite_name
+
+
+def __suite_architecture_add(d, args):
+ if len(args) < 3:
+ die("E: adding a suite-architecture entry requires a suite and arch")
+
+ s = d.session()
+
+ suite = get_suite(args[2].lower(), s)
+ if suite is None: die("E: Can't find suite %s" % args[2].lower())
+
+ arch = get_architecture(args[3].lower(), s)
+ if arch is None: die("E: Can't find architecture %s" % args[3].lower())
+
+ if not dryrun:
+ try:
+ sa = SuiteArchitecture()
+ sa.arch_id = arch.arch_id
+ sa.suite_id = suite.suite_id
+ s.add(sa)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Can't add suite-architecture entry (%s, %s) - probably already exists" % (args[2].lower(), args[3].lower()))
+ except SQLAlchemyError, e:
+ die("E: Can't add suite-architecture entry (%s, %s) - %s" % (args[2].lower(), args[3].lower(), e))
+
+ print "Added suite-architecture entry for %s, %s" % (args[2].lower(), args[3].lower())
+
+
+def __suite_architecture_rm(d, args):
+ if len(args) < 3:
+ die("E: removing an suite-architecture entry requires a suite and arch")
+
+ s = d.session()
+ if not dryrun:
+ try:
+ sa = get_suite_architecture(args[2].lower(), args[3].lower(), s)
+ if sa is None:
+ die("E: can't find suite-architecture entry for %s, %s" % (args[2].lower(), args[3].lower()))
+ s.delete(sa)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Can't remove suite-architecture entry (%s, %s) - it's probably referenced" % (args[2].lower(), args[3].lower()))
+ except SQLAlchemyError, e:
+ die("E: Can't remove suite-architecture entry (%s, %s) - %s" % (args[2].lower(), args[3].lower(), e))
+
+ print "Removed suite-architecture entry for %s, %s" % (args[2].lower(), args[3].lower())
+
+
+def suite_architecture(command):
+ args = [str(x) for x in command]
+ Cnf = utils.get_conf()
+ d = DBConn()
+
+ die_arglen(args, 2, "E: suite-architecture needs at least a command")
+
+ mode = args[1].lower()
+
+ if mode == 'list':
+ __suite_architecture_list(d, args)
+ elif mode == 'list-arch':
+ __suite_architecture_listarch(d, args)
+ elif mode == 'list-suite':
+ __suite_architecture_listsuite(d, args)
+ elif mode == 'add':
+ __suite_architecture_add(d, args)
+ elif mode == 'rm':
+ __suite_architecture_rm(d, args)
+ else:
+ die("E: suite-architecture command unknown")
+
+dispatch['suite-architecture'] = suite_architecture
+dispatch['s-a'] = suite_architecture
+
+################################################################################
+
+def main():
+ """Perform administrative work on the dak database"""
+ global dryrun
+ Cnf = utils.get_conf()
+ arguments = [('h', "help", "Admin::Options::Help"),
+ ('n', "dry-run", "Admin::Options::Dry-Run")]
+ for i in [ "help", "dry-run" ]:
+ if not Cnf.has_key("Admin::Options::%s" % (i)):
+ Cnf["Admin::Options::%s" % (i)] = ""
+
+ arguments = apt_pkg.ParseCommandLine(Cnf, arguments, sys.argv)
+
+ options = Cnf.SubTree("Admin::Options")
+ if options["Help"] or len(arguments) < 1:
+ usage()
+ if options["Dry-Run"]:
+ dryrun = True
+
+ subcommand = str(arguments[0])
+
+ if subcommand in dispatch.keys():
+ dispatch[subcommand](arguments)
+ else:
+ die("E: Unknown command")
+
+################################################################################
+
+if __name__ == '__main__':
+ main()
import commands
import os
-import pg
import stat
import sys
import time
import apt_pkg
import apt_inst
-from daklib import database
+
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_issource
+from daklib.config import Config
################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
db_files = {} #: Cache of filenames as known by the database
waste = 0.0 #: How many bytes are "wasted" by files not referenced in database
excluded = {} #: List of files which are excluded from files check
"""
global db_files
+ cnf = Config()
+
print "Building list of database files..."
- q = projectB.query("SELECT l.path, f.filename, f.last_used FROM files f, location l WHERE f.location = l.id ORDER BY l.path, f.filename")
- ql = q.getresult()
+ q = DBConn().session().query(PoolFile).join(Location).order_by('path', 'location')
print "Missing files:"
db_files.clear()
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
+
+ for f in q.all():
+ filename = os.path.abspath(f.location.path, f.filename)
db_files[filename] = ""
if os.access(filename, os.R_OK) == 0:
- if i[2]:
- print "(last used: %s) %s" % (i[2], filename)
+ if f.last_used:
+ print "(last used: %s) %s" % (f.last_used, filename)
else:
print "%s" % (filename)
- filename = Cnf["Dir::Override"]+'override.unreferenced'
+ filename = os.path.join(cnf["Dir::Override"], 'override.unreferenced')
if os.path.exists(filename):
f = utils.open_file(filename)
for filename in f.readlines():
print "Existent files not in db:"
- os.path.walk(Cnf["Dir::Root"]+'pool/', process_dir, None)
+ os.path.walk(cnf["Dir::Root"] + 'pool/', process_dir, None)
print
print "%s wasted..." % (utils.size_type(waste))
"""
Parse every .dsc file in the archive and check for it's validity.
"""
+
+ cnf = Config()
+
count = 0
suite = 'unstable'
- for component in Cnf.SubTree("Component").List():
+
+ for component in cnf.SubTree("Component").List():
component = component.lower()
- list_filename = '%s%s_%s_source.list' % (Cnf["Dir::Lists"], suite, component)
+ list_filename = '%s%s_%s_source.list' % (cnf["Dir::Lists"], suite, component)
list_file = utils.open_file(list_filename)
+
for line in list_file.readlines():
f = line[:-1]
try:
"""
Check for missing overrides in stable and unstable.
"""
- for suite in [ "stable", "unstable" ]:
- print suite
- print "-"*len(suite)
+ session = DBConn().session()
+
+ for suite_name in [ "stable", "unstable" ]:
+ print suite_name
+ print "-" * len(suite_name)
print
- suite_id = database.get_suite_id(suite)
- q = projectB.query("""
+ suite = get_suite(suite)
+ q = s.execute("""
SELECT DISTINCT b.package FROM binaries b, bin_associations ba
- WHERE b.id = ba.bin AND ba.suite = %s AND NOT EXISTS
- (SELECT 1 FROM override o WHERE o.suite = %s AND o.package = b.package)"""
- % (suite_id, suite_id))
- print q
- q = projectB.query("""
+ WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS
+ (SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)"""
+ % {'suiteid': suite.suite_id})
+
+ for j in q.fetchall():
+ print j[0]
+
+ q = s.execute("""
SELECT DISTINCT s.source FROM source s, src_associations sa
- WHERE s.id = sa.source AND sa.suite = %s AND NOT EXISTS
- (SELECT 1 FROM override o WHERE o.suite = %s and o.package = s.source)"""
- % (suite_id, suite_id))
- print q
+ WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS
+ (SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)"""
+ % {'suiteid': suite.suite_id})
+ for j in q.fetchall():
+ print j[0]
################################################################################
# Not the most enterprising method, but hey...
broken_count = 0
- q = projectB.query("SELECT id FROM source;")
- for i in q.getresult():
- source_id = i[0]
- q2 = projectB.query("""
-SELECT l.path, f.filename FROM files f, dsc_files df, location l WHERE df.source = %s AND f.id = df.file AND l.id = f.location"""
- % (source_id))
+
+ session = DBConn().session()
+
+ q = session.query(DBSource)
+ for s in q.all():
first_path = ""
first_filename = ""
- broken = 0
- for j in q2.getresult():
- filename = j[0] + j[1]
+ broken = False
+
+ qf = session.query(PoolFile).join(Location).join(DSCFile).filter_by(source_id=s.source_id)
+ for f in qf.all():
+ # 0: path
+ # 1: filename
+ filename = os.path.join(f.location.path, f.filename)
path = os.path.dirname(filename)
+
if first_path == "":
first_path = path
first_filename = filename
elif first_path != path:
symlink = path + '/' + os.path.basename(first_filename)
if not os.path.exists(symlink):
- broken = 1
- print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, source_id, symlink)
+ broken = True
+ print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, s.source_id, symlink)
if broken:
broken_count += 1
+
print "Found %d source packages where the source is not all in one directory." % (broken_count)
################################################################################
-
def check_checksums():
"""
Validate all files
"""
print "Getting file information from database..."
- q = projectB.query("SELECT l.path, f.filename, f.md5sum, f.sha1sum, f.sha256sum, f.size FROM files f, location l WHERE f.location = l.id")
- ql = q.getresult()
+ q = DBConn().session().query(PoolFile)
print "Checking file checksums & sizes..."
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
- db_md5sum = i[2]
- db_sha1sum = i[3]
- db_sha256sum = i[4]
- db_size = int(i[5])
+ for f in q:
+ filename = os.path.abspath(os.path.join(f.location.path, f.filename))
+
try:
- f = utils.open_file(filename)
+ fi = utils.open_file(filename)
except:
utils.warn("can't open '%s'." % (filename))
continue
- md5sum = apt_pkg.md5sum(f)
+
size = os.stat(filename)[stat.ST_SIZE]
- if md5sum != db_md5sum:
- utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, db_md5sum))
- if size != db_size:
- utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, db_size))
- f.seek(0)
- sha1sum = apt_pkg.sha1sum(f)
- if sha1sum != db_sha1sum:
- utils.warn("**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha1sum, db_sha1sum))
-
- f.seek(0)
- sha256sum = apt_pkg.sha256sum(f)
- if sha256sum != db_sha256sum:
- utils.warn("**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha256sum, db_sha256sum))
+ if size != f.filesize:
+ utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, f.filesize))
+
+ md5sum = apt_pkg.md5sum(fi)
+ if md5sum != f.md5sum:
+ utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, f.md5sum))
+
+ fi.seek(0)
+ sha1sum = apt_pkg.sha1sum(fi)
+ if sha1sum != f.sha1sum:
+ utils.warn("**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha1sum, f.sha1sum))
+
+ fi.seek(0)
+ sha256sum = apt_pkg.sha256sum(fi)
+ if sha256sum != f.sha256sum:
+ utils.warn("**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha256sum, f.sha256sum))
print "Done."
global current_file
- q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.deb$'")
- ql = q.getresult()
+ q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like('.deb$'))
+
db_files.clear()
count = 0
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
+
+ for pf in q.all():
+ filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))
if os.access(filename, os.R_OK):
f = utils.open_file(filename)
current_file = filename
f.seek(0)
apt_inst.debExtract(f, Ent, "data.tar.gz")
count += 1
+
print "Checked %d files (out of %d)." % (count, len(db_files.keys()))
################################################################################
count = 0
print "Building list of database files..."
- q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.dsc$'")
- ql = q.getresult()
- if ql:
+ q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like('.dsc$'))
+
+ if q.count() > 0:
print "Checking %d files..." % len(ql)
else:
print "No files to check."
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
+
+ for pf in q.all():
+ filename = os.path.abspath(os.path.join(pf.location.path + pf.filename))
+
try:
# NB: don't enforce .dsc syntax
dsc = utils.parse_changes(filename)
except:
utils.fubar("error parsing .dsc file '%s'." % (filename))
+
dsc_files = utils.build_file_list(dsc, is_a_dsc=1)
has_tar = 0
+
for f in dsc_files.keys():
m = re_issource.match(f)
if not m:
ftype = m.group(3)
if ftype == "orig.tar.gz" or ftype == "tar.gz":
has_tar = 1
+
if not has_tar:
utils.warn("%s has no .tar.gz in the .dsc file." % (f))
count += 1
"""
print "Building list of database files... ",
before = time.time()
- q = projectB.query("SELECT l.path, f.filename, f.id FROM files f, location l WHERE f.location = l.id")
- print "done. (%d seconds)" % (int(time.time()-before))
- q_files = q.getresult()
+ q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like('.dsc$'))
- for i in q_files:
- filename = os.path.normpath(i[0] + i[1])
+ for pf in q.all():
+ filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))
if os.access(filename, os.R_OK) == 0:
utils.warn("%s: doesn't exist." % (filename))
else:
def check_build_depends():
""" Validate build-dependencies of .dsc files in the archive """
- os.path.walk(Cnf["Dir::Root"], chk_bd_process_dir, None)
+ os.path.walk(cnf["Dir::Root"], chk_bd_process_dir, None)
################################################################################
def main ():
- global Cnf, projectB, db_files, waste, excluded
+ global db_files, waste, excluded
+
+ cnf = Config()
- Cnf = utils.get_conf()
Arguments = [('h',"help","Check-Archive::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Check-Archive::Options::%s" % (i)):
- Cnf["Check-Archive::Options::%s" % (i)] = ""
+ if not cnf.has_key("Check-Archive::Options::%s" % (i)):
+ cnf["Check-Archive::Options::%s" % (i)] = ""
- args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ args = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Check-Archive::Options")
+ Options = cnf.SubTree("Check-Archive::Options")
if Options["Help"]:
usage()
usage(1)
mode = args[0].lower()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ # Initialize DB
+ DBConn()
if mode == "checksums":
check_checksums()
################################################################################
-import pg, sys, os
+import sys, os
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
from daklib import utils
################################################################################
Options = None
-projectB = None
Logger = None
sections = {}
priorities = {}
entry = entry.split('_')[0]
blacklist[entry] = 1
-def process(osuite, affected_suites, originosuite, component, type):
- global Logger, Options, projectB, sections, priorities
+def process(osuite, affected_suites, originosuite, component, otype, session):
+ global Logger, Options, sections, priorities
- osuite_id = database.get_suite_id(osuite)
- if osuite_id == -1:
+ o = get_suite(osuite, session)
+ if o is None:
utils.fubar("Suite '%s' not recognised." % (osuite))
+ osuite_id = o.suite_id
+
originosuite_id = None
if originosuite:
- originosuite_id = database.get_suite_id(originosuite)
- if originosuite_id == -1:
+ oo = get_suite(originosuite, session)
+ if oo is None:
utils.fubar("Suite '%s' not recognised." % (originosuite))
+ originosuite_id = oo.suite_id
- component_id = database.get_component_id(component)
- if component_id == -1:
+ c = get_component(component, session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
+ component_id = c.component_id
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
- dsc_type_id = database.get_override_type_id("dsc")
+ ot = get_override_type(otype, session)
+ if ot is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
+ type_id = ot.overridetype_id
+ dsc_type_id = get_override_type("dsc", session).overridetype_id
- source_priority_id = database.get_priority_id("source")
+ source_priority_id = get_priority("source", session).priority_id
- if type == "deb" or type == "udeb":
+ if otype == "deb" or otype == "udeb":
packages = {}
- q = projectB.query("""
+ # TODO: Fix to use placeholders (check how to with arrays)
+ q = session.execute("""
SELECT b.package FROM binaries b, bin_associations ba, files f,
location l, component c
- WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
- AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s
-""" % (type, ",".join([ str(i) for i in affected_suites ]), component_id))
- for i in q.getresult():
+ WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
+ AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
+""" % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
+ for i in q.fetchall():
packages[i[0]] = 0
src_packages = {}
- q = projectB.query("""
+ q = session.execute("""
SELECT s.source FROM source s, src_associations sa, files f, location l,
component c
WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
- AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s
-""" % (",".join([ str(i) for i in affected_suites]), component_id))
- for i in q.getresult():
+ AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
+""" % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
+ for i in q.fetchall():
src_packages[i[0]] = 0
# -----------
# Drop unused overrides
- q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id))
- projectB.query("BEGIN WORK")
- if type == "dsc":
- for i in q.getresult():
+ q = session.execute("""SELECT package, priority, section, maintainer
+ FROM override WHERE suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'suite_id': osuite_id, 'component_id': component_id,
+ 'type_id': type_id})
+ # We're already within a transaction
+ if otype == "dsc":
+ for i in q.fetchall():
package = i[0]
if src_packages.has_key(package):
src_packages[package] = 1
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(["removing unused override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""DELETE FROM override WHERE package =
- '%s' AND suite = %s AND component = %s AND type =
- %s""" % (package, osuite_id, component_id, type_id))
+ session.execute("""DELETE FROM override WHERE package = :package
+ AND suite = :suite_id AND component = :component_id
+ AND type = :type_id""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': type_id})
# create source overrides based on binary overrides, as source
# overrides not always get created
- q = projectB.query(""" SELECT package, priority, section,
- maintainer FROM override WHERE suite = %s AND component = %s
- """ % (osuite_id, component_id))
- for i in q.getresult():
+ q = session.execute("""SELECT package, priority, section, maintainer
+ FROM override WHERE suite = :suite_id AND component = :component_id""",
+ {'suite_id': osuite_id, 'component_id': component_id})
+ for i in q.fetchall():
package = i[0]
if not src_packages.has_key(package) or src_packages[package]:
continue
src_packages[package] = 1
Logger.log(["add missing override", osuite, component,
- type, package, "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
- osuite_id, component_id, source_priority_id, i[2],
- dsc_type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id, :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': source_priority_id,
+ 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
# Check whether originosuite has an override for us we can
# copy
if originosuite:
- q = projectB.query("""SELECT origin.package, origin.priority,
- origin.section, origin.maintainer, target.priority,
- target.section, target.maintainer FROM override origin LEFT
- JOIN override target ON (origin.package = target.package AND
- target.suite=%s AND origin.component = target.component AND origin.type =
- target.type) WHERE origin.suite = %s AND origin.component = %s
- AND origin.type = %s""" %
- (osuite_id, originosuite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT origin.package, origin.priority, origin.section,
+ origin.maintainer, target.priority, target.section,
+ target.maintainer
+ FROM override origin
+ LEFT JOIN override target ON (origin.package = target.package
+ AND target.suite = :suite_id
+ AND origin.component = target.component
+ AND origin.type = target.type)
+ WHERE origin.suite = :originsuite_id
+ AND origin.component = :component_id
+ AND origin.type = :type_id""",
+ {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
+ 'component_id': component_id, 'type_id': type_id})
+ for i in q.fetchall():
package = i[0]
if not src_packages.has_key(package) or src_packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(["syncing override", osuite, component,
- type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""UPDATE override SET section=%s,
- maintainer='%s' WHERE package='%s' AND
- suite=%s AND component=%s AND type=%s""" %
- (i[2], i[3], package, osuite_id, component_id,
- dsc_type_id))
+ session.execute("""UPDATE override
+ SET section = :section,
+ maintainer = :maintainer
+ WHERE package = :package AND suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'section': i[2], 'maintainer': i[3],
+ 'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': dsc_type_id})
continue
+
# we can copy
src_packages[package] = 1
Logger.log(["copying missing override", osuite, component,
- type, package, "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
- osuite_id, component_id, source_priority_id, i[2],
- dsc_type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id,
+ :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': source_priority_id,
+ 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
for package, hasoverride in src_packages.items():
if not hasoverride:
utils.warn("%s has no override!" % package)
else: # binary override
- for i in q.getresult():
+ for i in q.fetchall():
package = i[0]
if packages.has_key(package):
packages[package] = 1
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(["removing unused override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""DELETE FROM override WHERE package =
- '%s' AND suite = %s AND component = %s AND type =
- %s""" % (package, osuite_id, component_id, type_id))
+ session.execute("""DELETE FROM override
+ WHERE package = :package AND suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': type_id})
# Check whether originosuite has an override for us we can
# copy
if originosuite:
- q = projectB.query("""SELECT origin.package, origin.priority,
- origin.section, origin.maintainer, target.priority,
- target.section, target.maintainer FROM override origin LEFT
- JOIN override target ON (origin.package = target.package AND
- target.suite=%s AND origin.component = target.component AND
- origin.type = target.type) WHERE origin.suite = %s AND
- origin.component = %s AND origin.type = %s""" % (osuite_id,
- originosuite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT origin.package, origin.priority, origin.section,
+ origin.maintainer, target.priority, target.section,
+ target.maintainer
+ FROM override origin LEFT JOIN override target
+ ON (origin.package = target.package
+ AND target.suite = :suite_id
+ AND origin.component = target.component
+ AND origin.type = target.type)
+ WHERE origin.suite = :originsuite_id
+ AND origin.component = :component_id
+ AND origin.type = :type_id""",
+ {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
+ 'component_id': component_id, 'type_id': type_id})
+ for i in q.fetchall():
package = i[0]
if not packages.has_key(package) or packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(["syncing override", osuite, component,
- type, package, priorities[i[4]], sections[i[5]],
+ otype, package, priorities[i[4]], sections[i[5]],
i[6], priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""UPDATE override SET priority=%s, section=%s,
- maintainer='%s' WHERE package='%s' AND
- suite=%s AND component=%s AND type=%s""" %
- (i[1], i[2], i[3], package, osuite_id,
- component_id, type_id))
+ session.execute("""UPDATE override
+ SET priority = :priority_id,
+ section = :section_id,
+ maintainer = :maintainer
+ WHERE package = :package
+ AND suite = :suite_id
+ AND component = :component_id
+ AND type = :type_id""",
+ {'priority_id': i[1], 'section_id': i[2],
+ 'maintainer': i[3], 'package': package,
+ 'suite_id': osuite_id, 'component_id': component_id,
+ 'type_id': type_id})
continue
# we can copy
packages[package] = 1
Logger.log(["copying missing override", osuite, component,
type, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id, :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': i[1],
+ 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
for package, hasoverride in packages.items():
if not hasoverride:
utils.warn("%s has no override!" % package)
- projectB.query("COMMIT WORK")
+ session.commit()
sys.stdout.flush()
################################################################################
def main ():
- global Logger, Options, projectB, sections, priorities
+ global Logger, Options, sections, priorities
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Check-Overrides::Options::Help"),
('n',"no-action", "Check-Overrides::Options::No-Action")]
for i in [ "help", "no-action" ]:
- if not Cnf.has_key("Check-Overrides::Options::%s" % (i)):
- Cnf["Check-Overrides::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Check-Overrides::Options")
+ if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
+ cnf["Check-Overrides::Options::%s" % (i)] = ""
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Check-Overrides::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# init sections, priorities:
- q = projectB.query("SELECT id, section FROM section")
- for i in q.getresult():
- sections[i[0]] = i[1]
- q = projectB.query("SELECT id, priority FROM priority")
- for i in q.getresult():
- priorities[i[0]] = i[1]
+
+ # We need forward and reverse
+ sections = get_sections(session)
+ for name, entry in sections.items():
+ sections[entry] = name
+
+ priorities = get_priorities(session)
+ for name, entry in priorities.items():
+ priorities[entry] = name
if not Options["No-Action"]:
- Logger = logging.Logger(Cnf, "check-overrides")
+ Logger = daklog.Logger(cnf, "check-overrides")
else:
- Logger = logging.Logger(Cnf, "check-overrides", 1)
+ Logger = daklog.Logger(cnf, "check-overrides", 1)
- gen_blacklist(Cnf["Dir::Queue::Accepted"])
+ gen_blacklist(cnf["Dir::Queue::Accepted"])
- for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
- if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
+ for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
+ if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
continue
osuite = osuite.lower()
originosuite = None
originremark = ""
try:
- originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
+ originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
originosuite = originosuite.lower()
originremark = " taking missing from %s" % originosuite
except KeyError:
print "Processing %s%s..." % (osuite, originremark)
# Get a list of all suites that use the override file of 'osuite'
- ocodename = Cnf["Suite::%s::codename" % osuite].lower()
+ ocodename = cnf["Suite::%s::codename" % osuite].lower()
suites = []
- for suite in Cnf.SubTree("Suite").List():
- if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite].lower():
- suites.append(suite)
-
- q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \
- % ", ".join([ repr(i) for i in suites ]).lower())
-
suiteids = []
- for i in q.getresult():
- suiteids.append(i[0])
+ for suite in cnf.SubTree("Suite").List():
+ if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower():
+ suites.append(suite)
+ s = get_suite(suite.lower(), session)
+ if s is not None:
+ suiteids.append(s.suite_id)
if len(suiteids) != len(suites) or len(suiteids) < 1:
utils.fubar("Couldn't find id's of all suites: %s" % suites)
- for component in Cnf.SubTree("Component").List():
+ for component in cnf.SubTree("Component").List():
# It is crucial for the dsc override creation based on binary
# overrides that 'dsc' goes first
- otypes = Cnf.ValueList("OverrideType")
+ otypes = cnf.ValueList("OverrideType")
otypes.remove("dsc")
otypes = ["dsc"] + otypes
for otype in otypes:
print "Processing %s [%s - %s] using %s..." \
% (osuite, component, otype, suites)
sys.stdout.flush()
- process(osuite, suiteids, originosuite, component, otype)
+ process(osuite, suiteids, originosuite, component, otype, session)
Logger.close()
################################################################################
-import pg, sys, os
+import sys, os
import apt_pkg, apt_inst
-from daklib import database
+
+from daklib.dbconn import *
+from daklib.config import Config
from daklib import utils
from daklib.regexes import re_no_epoch
################################################################################
-Cnf = None
-projectB = None
Options = None
stable = {}
stable_virtual = {}
################################################################################
def check_changes (filename):
+ cnf = Config()
+
try:
changes = utils.parse_changes(filename)
files = utils.build_file_list(changes)
# Move to the pool directory
cwd = os.getcwd()
f = files.keys()[0]
- pool_dir = Cnf["Dir::Pool"] + '/' + utils.poolify(changes["source"], files[f]["component"])
+ pool_dir = cnf["Dir::Pool"] + '/' + utils.poolify(changes["source"], files[f]["component"])
os.chdir(pool_dir)
changes_result = 0
################################################################################
def check_joey (filename):
+ cnf = Config()
+
f = utils.open_file(filename)
cwd = os.getcwd()
- os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"]))
+ os.chdir("%s/dists/proposed-updates" % (cnf["Dir::Root"]))
for line in f.readlines():
line = line.rstrip()
def parse_packages():
global stable, stable_virtual, architectures
+ cnf = Config()
+
# Parse the Packages files (since it's a sub-second operation on auric)
suite = "stable"
stable = {}
- components = Cnf.ValueList("Suite::%s::Components" % (suite))
- architectures = filter(utils.real_arch, database.get_suite_architectures(suite))
+ components = cnf.ValueList("Suite::%s::Components" % (suite))
+ architectures = [ a.arch_string for a in get_suite_architectures(suite, skipsrc=True, skipall=True) ]
for component in components:
for architecture in architectures:
- filename = "%s/dists/%s/%s/binary-%s/Packages" % (Cnf["Dir::Root"], suite, component, architecture)
+ filename = "%s/dists/%s/%s/binary-%s/Packages" % (cnf["Dir::Root"], suite, component, architecture)
packages = utils.open_file(filename, 'r')
Packages = apt_pkg.ParseTagFile(packages)
while Packages.Step():
################################################################################
def main ():
- global Cnf, projectB, Options
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('d', "debug", "Check-Proposed-Updates::Options::Debug"),
('q',"quiet","Check-Proposed-Updates::Options::Quiet"),
('v',"verbose","Check-Proposed-Updates::Options::Verbose"),
('h',"help","Check-Proposed-Updates::Options::Help")]
for i in [ "debug", "quiet", "verbose", "help" ]:
- if not Cnf.has_key("Check-Proposed-Updates::Options::%s" % (i)):
- Cnf["Check-Proposed-Updates::Options::%s" % (i)] = ""
+ if not cnf.has_key("Check-Proposed-Updates::Options::%s" % (i)):
+ cnf["Check-Proposed-Updates::Options::%s" % (i)] = ""
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Check-Proposed-Updates::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Check-Proposed-Updates::Options")
if Options["Help"]:
usage(0)
if not arguments:
utils.fubar("need at least one package name as an argument.")
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ DBConn()
print "Parsing packages files...",
parse_packages()
################################################################################
-import os, pg, sys
+import os, sys
import apt_pkg
-from daklib import database
+
+from daklib.dbconn import *
+from daklib.config import Config
from daklib import utils
from daklib.regexes import re_isdeb, re_isadeb, re_issource, re_no_epoch
################################################################################
-Cnf = None
-projectB = None
Options = None
pu = {}
################################################################################
def check_changes (filename):
+ cnf = Config()
+
try:
changes = utils.parse_changes(filename)
files = utils.build_file_list(changes)
new_num_files = len(files.keys())
if new_num_files == 0:
print "%s: no files left, superseded by %s" % (filename, pu_version)
- dest = Cnf["Dir::Morgue"] + "/misc/"
+ dest = cnf["Dir::Morgue"] + "/misc/"
if not Options["no-action"]:
utils.move(filename, dest)
elif new_num_files < num_files:
################################################################################
def check_joey (filename):
+ cnf = Config()
+
f = utils.open_file(filename)
cwd = os.getcwd()
- os.chdir("%s/dists/%s" % (Cnf["Dir::Root"]), Options["suite"])
+ os.chdir("%s/dists/%s" % (cnf["Dir::Root"]), Options["suite"])
for line in f.readlines():
line = line.rstrip()
def init_pu ():
global pu
- q = projectB.query("""
+ q = DBConn().session().execute("""
SELECT b.package, b.version, a.arch_string
FROM bin_associations ba, binaries b, suite su, architecture a
WHERE b.id = ba.bin AND ba.suite = su.id
- AND su.suite_name = '%s' AND a.id = b.architecture
+ AND su.suite_name = :suite_name AND a.id = b.architecture
UNION SELECT s.source, s.version, 'source'
FROM src_associations sa, source s, suite su
WHERE s.id = sa.source AND sa.suite = su.id
- AND su.suite_name = '%s'
+ AND su.suite_name = :suite_name
ORDER BY package, version, arch_string
-""" % (Options["suite"], Options["suite"]))
- ql = q.getresult()
- for i in ql:
+""" % {'suite_name': Options["suite"]})
+
+ for i in q.fetchall():
pkg = i[0]
version = i[1]
arch = i[2]
pu[pkg][arch] = version
def main ():
- global Cnf, projectB, Options
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('d', "debug", "Clean-Proposed-Updates::Options::Debug"),
('v', "verbose", "Clean-Proposed-Updates::Options::Verbose"),
('s', "suite", "Clean-Proposed-Updates::Options::Suite", "HasArg"),
('n', "no-action", "Clean-Proposed-Updates::Options::No-Action"),]
for i in [ "debug", "verbose", "help", "no-action" ]:
- if not Cnf.has_key("Clean-Proposed-Updates::Options::%s" % (i)):
- Cnf["Clean-Proposed-Updates::Options::%s" % (i)] = ""
+ if not cnf.has_key("Clean-Proposed-Updates::Options::%s" % (i)):
+ cnf["Clean-Proposed-Updates::Options::%s" % (i)] = ""
# suite defaults to proposed-updates
- if not Cnf.has_key("Clean-Proposed-Updates::Options::Suite"):
- Cnf["Clean-Proposed-Updates::Options::Suite"] = "proposed-updates"
+ if not cnf.has_key("Clean-Proposed-Updates::Options::Suite"):
+ cnf["Clean-Proposed-Updates::Options::Suite"] = "proposed-updates"
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Clean-Proposed-Updates::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Clean-Proposed-Updates::Options")
if Options["Help"]:
usage(0)
if not arguments:
utils.fubar("need at least one package name as an argument.")
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ DBConn()
init_pu()
################################################################################
-import os, pg, stat, sys, time
+import os, stat, sys, time
import apt_pkg
+from datetime import datetime, timedelta
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
################################################################################
-projectB = None
-Cnf = None
Options = None
-now_date = None; # mark newly "deleted" things as deleted "now"
-delete_date = None; # delete things marked "deleted" earler than this
-max_delete = None
################################################################################
################################################################################
-def check_binaries():
- global delete_date, now_date
-
+def check_binaries(now_date, delete_date, max_delete, session):
print "Checking for orphaned binary packages..."
# Get the list of binary packages not in a suite and mark them for
# deletion.
- q = projectB.query("""
+
+ # TODO: This can be a single SQL UPDATE statement
+ q = session.execute("""
SELECT b.file FROM binaries b, files f
WHERE f.last_used IS NULL AND b.file = f.id
AND NOT EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
- ql = q.getresult()
- projectB.query("BEGIN WORK")
- for i in ql:
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id))
- projectB.query("COMMIT WORK")
+ for i in q.fetchall():
+ session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid AND last_used IS NULL",
+ {'lastused': now_date, 'fileid': i[0]})
+ session.commit()
# Check for any binaries which are marked for eventual deletion
# but are now used again.
- q = projectB.query("""
+
+ # TODO: This can be a single SQL UPDATE statement
+ q = session.execute("""
SELECT b.file FROM binaries b, files f
WHERE f.last_used IS NOT NULL AND f.id = b.file
AND EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
- ql = q.getresult()
- projectB.query("BEGIN WORK")
- for i in ql:
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id))
- projectB.query("COMMIT WORK")
+ for i in q.fetchall():
+ session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid", {'fileid': i[0]})
+ session.commit()
########################################
-def check_sources():
- global delete_date, now_date
-
+def check_sources(now_date, delete_date, max_delete, session):
print "Checking for orphaned source packages..."
# Get the list of source packages not in a suite and not used by
# any binaries.
- q = projectB.query("""
+ q = session.execute("""
SELECT s.id, s.file FROM source s, files f
WHERE f.last_used IS NULL AND s.file = f.id
AND NOT EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id)
#### have been marked for deletion (so the delay between bins go
#### byebye and sources go byebye is 0 instead of StayOfExecution)
- ql = q.getresult()
-
- projectB.query("BEGIN WORK")
- for i in ql:
+ for i in q.fetchall():
source_id = i[0]
dsc_file_id = i[1]
# Mark the .dsc file for deletion
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, dsc_file_id))
+ session.execute("""UPDATE files SET last_used = :last_used
+ WHERE id = :dscfileid AND last_used IS NULL""",
+ {'last_used': now_date, 'dscfileid': dsc_file_id})
+
# Mark all other files references by .dsc too if they're not used by anyone else
- x = projectB.query("SELECT f.id FROM files f, dsc_files d WHERE d.source = %s AND d.file = f.id" % (source_id))
- for j in x.getresult():
+ x = session.execute("""SELECT f.id FROM files f, dsc_files d
+ WHERE d.source = :sourceid AND d.file = f.id""",
+ {'sourceid': source_id})
+ for j in x.fetchall():
file_id = j[0]
- y = projectB.query("SELECT id FROM dsc_files d WHERE d.file = %s" % (file_id))
- if len(y.getresult()) == 1:
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id))
- projectB.query("COMMIT WORK")
+ y = session.execute("SELECT id FROM dsc_files d WHERE d.file = :fileid", {'fileid': file_id})
+ if len(y.fetchall()) == 1:
+ session.execute("""UPDATE files SET last_used = :lastused
+ WHERE id = :fileid AND last_used IS NULL""",
+ {'lastused': now_date, 'fileid': file_id})
+
+ session.commit()
# Check for any sources which are marked for deletion but which
# are now used again.
- q = projectB.query("""
+ q = session.execute("""
SELECT f.id FROM source s, files f, dsc_files df
WHERE f.last_used IS NOT NULL AND s.id = df.source AND df.file = f.id
AND ((EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id))
#### XXX: this should also handle deleted binaries specially (ie, not
#### reinstate sources because of them
- ql = q.getresult()
# Could be done in SQL; but left this way for hysterical raisins
# [and freedom to innovate don'cha know?]
- projectB.query("BEGIN WORK")
- for i in ql:
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id))
- projectB.query("COMMIT WORK")
+ for i in q.fetchall():
+ session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid",
+ {'fileid': i[0]})
-########################################
+ session.commit()
-def check_files():
- global delete_date, now_date
+########################################
+def check_files(now_date, delete_date, max_delete, session):
# FIXME: this is evil; nothing should ever be in this state. if
# they are, it's a bug and the files should not be auto-deleted.
-
- return
+ # XXX: In that case, remove the stupid return to later and actually
+ # *TELL* us rather than silently ignore it - mhy
print "Checking for unused files..."
- q = projectB.query("""
-SELECT id FROM files f
+ q = session.execute("""
+SELECT id, filename FROM files f
WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = f.id)
- AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)""")
+ AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)
+ ORDER BY filename""")
- projectB.query("BEGIN WORK")
- for i in q.getresult():
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s" % (now_date, file_id))
- projectB.query("COMMIT WORK")
+ ql = q.fetchall()
+ if len(ql) > 0:
+ print "WARNING: check_files found something it shouldn't"
+ for x in ql:
+ print x
-def clean_binaries():
- global delete_date, now_date
+ # NOW return, the code below is left as an example of what was
+ # evidently done at some point in the past
+ return
+
+# for i in q.fetchall():
+# file_id = i[0]
+# session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid",
+# {'lastused': now_date, 'fileid': file_id})
+#
+# session.commit()
+def clean_binaries(now_date, delete_date, max_delete, session):
# We do this here so that the binaries we remove will have their
# source also removed (if possible).
# buys anything keeping this separate
print "Cleaning binaries from the DB..."
if not Options["No-Action"]:
- before = time.time()
- sys.stdout.write("[Deleting from binaries table... ")
- projectB.query("DELETE FROM binaries WHERE EXISTS (SELECT 1 FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')" % (delete_date))
- sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+ print "Deleting from binaries table... "
+ session.execute("""DELETE FROM binaries WHERE EXISTS
+ (SELECT 1 FROM files WHERE binaries.file = files.id
+ AND files.last_used <= :deldate)""",
+ {'deldate': delete_date})
########################################
-def clean():
- global delete_date, now_date, max_delete
+def clean(now_date, delete_date, max_delete, session):
+ cnf = Config()
+
count = 0
size = 0
print "Cleaning out packages..."
- date = time.strftime("%Y-%m-%d")
- dest = Cnf["Dir::Morgue"] + '/' + Cnf["Clean-Suites::MorgueSubDir"] + '/' + date
+ cur_date = now_date.strftime("%Y-%m-%d")
+ dest = os.path.join(cnf["Dir::Morgue"], cnf["Clean-Suites::MorgueSubDir"], cur_date)
if not os.path.exists(dest):
os.mkdir(dest)
# Delete from source
if not Options["No-Action"]:
- before = time.time()
- sys.stdout.write("[Deleting from source table... ")
- projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date))
- projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date))
- sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+ print "Deleting from source table... "
+ session.execute("""DELETE FROM dsc_files
+ WHERE EXISTS
+ (SELECT 1 FROM source s, files f, dsc_files df
+ WHERE f.last_used <= :deletedate
+ AND s.file = f.id AND s.id = df.source
+ AND df.id = dsc_files.id)""", {'deletedate': delete_date})
+ session.execute("""DELETE FROM source
+ WHERE EXISTS
+ (SELECT 1 FROM files
+ WHERE source.file = files.id
+ AND files.last_used <= :deletedate)""", {'deletedate': delete_date})
+
+ session.commit()
# Delete files from the pool
- query = "SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)
+ query = """SELECT l.path, f.filename FROM location l, files f
+ WHERE f.last_used <= :deletedate AND l.id = f.location"""
if max_delete is not None:
query += " LIMIT %d" % max_delete
- sys.stdout.write("Limiting removals to %d\n" % max_delete)
+ print "Limiting removals to %d" % max_delete
- q=projectB.query(query)
- for i in q.getresult():
+ q = session.execute(query, {'deletedate': delete_date})
+ for i in q.fetchall():
filename = i[0] + i[1]
if not os.path.exists(filename):
utils.warn("can not find '%s'." % (filename))
utils.fubar("%s is neither symlink nor file?!" % (filename))
# Delete from the 'files' table
+ # XXX: I've a horrible feeling that the max_delete stuff breaks here - mhy
+ # TODO: Change it so we do the DELETEs as we go; it'll be slower but
+ # more reliable
if not Options["No-Action"]:
- before = time.time()
- sys.stdout.write("[Deleting from files table... ")
- projectB.query("DELETE FROM files WHERE last_used <= '%s'" % (delete_date))
- sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+ print "Deleting from files table... "
+ session.execute("DELETE FROM files WHERE last_used <= :deletedate", {'deletedate': delete_date})
+ session.commit()
+
if count > 0:
- sys.stderr.write("Cleaned %d files, %s.\n" % (count, utils.size_type(size)))
+ print "Cleaned %d files, %s." % (count, utils.size_type(size))
################################################################################
-def clean_maintainers():
+def clean_maintainers(now_date, delete_date, max_delete, session):
print "Cleaning out unused Maintainer entries..."
- q = projectB.query("""
+ # TODO Replace this whole thing with one SQL statement
+ q = session.execute("""
SELECT m.id FROM maintainer m
WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id)
AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id OR s.changedby = m.id)
AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)""")
- ql = q.getresult()
count = 0
- projectB.query("BEGIN WORK")
- for i in ql:
+
+ for i in q.fetchall():
maintainer_id = i[0]
if not Options["No-Action"]:
- projectB.query("DELETE FROM maintainer WHERE id = %s" % (maintainer_id))
+ session.execute("DELETE FROM maintainer WHERE id = :maint", {'maint': maintainer_id})
count += 1
- projectB.query("COMMIT WORK")
+
+ if not Options["No-Action"]:
+ session.commit()
if count > 0:
- sys.stderr.write("Cleared out %d maintainer entries.\n" % (count))
+ print "Cleared out %d maintainer entries." % (count)
################################################################################
-def clean_fingerprints():
+def clean_fingerprints(now_date, delete_date, max_delete, session):
print "Cleaning out unused fingerprint entries..."
- q = projectB.query("""
+ # TODO Replace this whole thing with one SQL statement
+ q = session.execute("""
SELECT f.id FROM fingerprint f
WHERE f.keyring IS NULL
AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""")
- ql = q.getresult()
count = 0
- projectB.query("BEGIN WORK")
- for i in ql:
+
+ for i in q.fetchall():
fingerprint_id = i[0]
if not Options["No-Action"]:
- projectB.query("DELETE FROM fingerprint WHERE id = %s" % (fingerprint_id))
+ session.execute("DELETE FROM fingerprint WHERE id = :fpr", {'fpr': fingerprint_id})
count += 1
- projectB.query("COMMIT WORK")
+
+ if not Options["No-Action"]:
+ session.commit()
if count > 0:
- sys.stderr.write("Cleared out %d fingerprint entries.\n" % (count))
+ print "Cleared out %d fingerprint entries." % (count)
################################################################################
-def clean_queue_build():
- global now_date
+def clean_queue_build(now_date, delete_date, max_delete, session):
+
+ cnf = Config()
- if not Cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]:
+ if not cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]:
return
print "Cleaning out queue build symlinks..."
- our_delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Clean-Suites::QueueBuildStayOfExecution"])))
+ our_delete_date = now_date - timedelta(seconds = int(cnf["Clean-Suites::QueueBuildStayOfExecution"]))
count = 0
- q = projectB.query("SELECT filename FROM queue_build WHERE last_used <= '%s'" % (our_delete_date))
- for i in q.getresult():
+ q = session.execute("SELECT filename FROM queue_build WHERE last_used <= :deletedate",
+ {'deletedate': our_delete_date})
+ for i in q.fetchall():
filename = i[0]
if not os.path.exists(filename):
utils.warn("%s (from queue_build) doesn't exist." % (filename))
continue
- if not Cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(filename):
+
+ if not cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(filename):
utils.fubar("%s (from queue_build) should be a symlink but isn't." % (filename))
+
os.unlink(filename)
count += 1
- projectB.query("DELETE FROM queue_build WHERE last_used <= '%s'" % (our_delete_date))
+
+ session.execute("DELETE FROM queue_build WHERE last_used <= :deletedate",
+ {'deletedate': our_delete_date})
+
+ session.commit()
if count:
- sys.stderr.write("Cleaned %d queue_build files.\n" % (count))
+ print "Cleaned %d queue_build files." % (count)
################################################################################
def main():
- global Cnf, Options, projectB, delete_date, now_date, max_delete
+ global Options
+
+ cnf = Config()
- Cnf = utils.get_conf()
for i in ["Help", "No-Action", "Maximum" ]:
- if not Cnf.has_key("Clean-Suites::Options::%s" % (i)):
- Cnf["Clean-Suites::Options::%s" % (i)] = ""
+ if not cnf.has_key("Clean-Suites::Options::%s" % (i)):
+ cnf["Clean-Suites::Options::%s" % (i)] = ""
Arguments = [('h',"help","Clean-Suites::Options::Help"),
('n',"no-action","Clean-Suites::Options::No-Action"),
('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")]
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Clean-Suites::Options")
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Clean-Suites::Options")
- if Cnf["Clean-Suites::Options::Maximum"] != "":
+ if cnf["Clean-Suites::Options::Maximum"] != "":
try:
# Only use Maximum if it's an integer
- max_delete = int(Cnf["Clean-Suites::Options::Maximum"])
+ max_delete = int(cnf["Clean-Suites::Options::Maximum"])
if max_delete < 1:
utils.fubar("If given, Maximum must be at least 1")
except ValueError, e:
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ session = DBConn().session()
- now_date = time.strftime("%Y-%m-%d %H:%M")
- delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Clean-Suites::StayOfExecution"])))
+ now_date = datetime.now()
+ delete_date = now_date - timedelta(seconds=int(cnf['Clean-Suites::StayOfExecution']))
- check_binaries()
- clean_binaries()
- check_sources()
- check_files()
- clean()
- clean_maintainers()
- clean_fingerprints()
- clean_queue_build()
+ check_binaries(now_date, delete_date, max_delete, session)
+ clean_binaries(now_date, delete_date, max_delete, session)
+ check_sources(now_date, delete_date, max_delete, session)
+ check_files(now_date, delete_date, max_delete, session)
+ clean(now_date, delete_date, max_delete, session)
+ clean_maintainers(now_date, delete_date, max_delete, session)
+ clean_fingerprints(now_date, delete_date, max_delete, session)
+ clean_queue_build(now_date, delete_date, max_delete, session)
################################################################################
from daklib import utils
from daklib.binary import Binary
from daklib.config import Config
-from daklib.dbconn import DBConn
+from daklib.dbconn import *
+
################################################################################
def usage (exit_code=0):
################################################################################
-# get all the arches delivered for a given suite
-# this should probably exist somehere common
-arches_q = """PREPARE arches_q(int) as
- SELECT s.architecture, a.arch_string
- FROM suite_architectures s
- JOIN architecture a ON (s.architecture=a.id)
- WHERE suite = $1"""
-
-# find me the .deb for a given binary id
-debs_q = """PREPARE debs_q(int, int) as
- SELECT b.id, f.filename FROM bin_assoc_by_arch baa
- JOIN binaries b ON baa.bin=b.id
- JOIN files f ON b.file=f.id
- WHERE suite = $1
- AND arch = $2"""
-
-# ask if we already have contents associated with this binary
-olddeb_q = """PREPARE olddeb_q(int) as
- SELECT 1 FROM content_associations
- WHERE binary_pkg = $1
- LIMIT 1"""
-
-# find me all of the contents for a given .deb
-contents_q = """PREPARE contents_q(int,int) as
- SELECT (p.path||'/'||n.file) AS fn,
- s.section,
- b.package,
- b.architecture
- FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE o.suite = $1 AND o.type = $2
- AND b.type='deb'
- ORDER BY fn"""
-
-# find me all of the contents for a given .udeb
-udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as
- SELECT (p.path||'/'||n.file) AS fn,
- s.section,
- b.package,
- b.architecture
- FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE o.suite = $1 AND o.type = $2
- AND s.id = $3
- AND b.type='udeb'
- ORDER BY fn"""
-
-# FROM content_file_paths p join content_associations c ON (c.filepath=p.id)
-# JOIN content_file_names n ON (c.filename=n.id)
-# JOIN binaries b ON (b.id=c.binary_pkg)
-# JOIN override o ON (o.package=b.package)
-# JOIN section s ON (s.id=o.section)
-# WHERE o.suite = $1 AND o.type = $2
-# AND s.id = $3
-# AND b.id in (SELECT ba.bin from bin_associations ba join binaries b on b.id=ba.bin where (b.architecture=$3 or b.architecture=$4)and ba.suite=$1 and b.type='udeb')
-# GROUP BY fn
-# ORDER BY fn;"""
-
-
-
-# clear out all of the temporarily stored content associations
-# this should be run only after p-a has run. after a p-a
-# run we should have either accepted or rejected every package
-# so there should no longer be anything in the queue
-remove_pending_contents_cruft_q = """DELETE FROM pending_content_associations"""
-
-# delete any filenames we are storing which have no binary associated with them
-remove_filename_cruft_q = """DELETE FROM content_file_names
- WHERE id IN (SELECT cfn.id FROM content_file_names cfn
- LEFT JOIN content_associations ca
- ON ca.filename=cfn.id
- WHERE ca.id IS NULL)"""
-
-# delete any paths we are storing which have no binary associated with them
-remove_filepath_cruft_q = """DELETE FROM content_file_paths
- WHERE id IN (SELECT cfn.id FROM content_file_paths cfn
- LEFT JOIN content_associations ca
- ON ca.filepath=cfn.id
- WHERE ca.id IS NULL)"""
-
class EndOfContents(object):
"""
A sentry object for the end of the filename stream
class Contents(object):
"""
Class capable of generating Contents-$arch.gz files
-
- Usage GenerateContents().generateContents( ["main","contrib","non-free"] )
"""
def __init__(self):
def reject(self, message):
log.error("E: %s" % message)
- # goal column for section column
- _goal_column = 54
-
def cruft(self):
"""
remove files/paths from the DB which are no longer referenced
by binaries and clean the temporary table
"""
- cursor = DBConn().cursor();
- cursor.execute( "BEGIN WORK" )
- cursor.execute( remove_pending_contents_cruft_q )
- cursor.execute( remove_filename_cruft_q )
- cursor.execute( remove_filepath_cruft_q )
- cursor.execute( "COMMIT" )
+ s = DBConn().session()
+
+ # clear out all of the temporarily stored content associations
+ # this should be run only after p-a has run. after a p-a
+ # run we should have either accepted or rejected every package
+ # so there should no longer be anything in the queue
+ s.query(PendingContentAssociation).delete()
+
+ # delete any filenames we are storing which have no binary associated
+ # with them
+ cafq = s.query(ContentAssociation.filename_id).distinct()
+ cfq = s.query(ContentFilename)
+ cfq = cfq.filter(~ContentFilename.cafilename_id.in_(cafq))
+ cfq.delete()
+
+ # delete any paths we are storing which have no binary associated with
+ # them
+ capq = s.query(ContentAssociation.filepath_id).distinct()
+ cpq = s.query(ContentFilepath)
+ cpq = cpq.filter(~ContentFilepath.cafilepath_id.in_(capq))
+ cpq.delete()
+
+ s.commit()
def bootstrap(self):
"""
pooldir = Config()[ 'Dir::Pool' ]
- cursor = DBConn().cursor();
- DBConn().prepare("debs_q",debs_q)
- DBConn().prepare("olddeb_q",olddeb_q)
- DBConn().prepare("arches_q",arches_q)
-
- suites = self._suites()
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
-
- arch_list = self._arches(cursor, suite_id)
- arch_all_id = DBConn().get_architecture_id("all")
- for arch_id in arch_list:
- cursor.execute( "EXECUTE debs_q(%d, %d)" % ( suite_id, arch_id[0] ) )
-
- count = 0
- while True:
- deb = cursor.fetchone()
- if not deb:
- break
- count += 1
- cursor1 = DBConn().cursor();
- cursor1.execute( "EXECUTE olddeb_q(%d)" % (deb[0] ) )
- old = cursor1.fetchone()
- if old:
- log.debug( "already imported: %s" % (deb[1]) )
+ s = DBConn().session()
+
+ for suite in s.query(Suite).all():
+ for arch in get_suite_architectures(suite.suite_name, skipsrc=True, skipall=True, session=s):
+ q = s.query(BinAssociation).join(Suite)
+ q = q.join(Suite).filter_by(suite_name=suite.suite_name)
+ q = q.join(DBBinary).join(Architecture).filter_by(arch.arch_string)
+ for ba in q:
+ filename = ba.binary.poolfile.filename
+ # Check for existing contents
+ existingq = s.query(ContentAssociations).filter_by(binary_pkg=ba.binary_id).limit(1)
+ if existingq.count() > 0:
+ log.debug( "already imported: %s" % (filename))
else:
- log.debug( "scanning: %s" % (deb[1]) )
- debfile = os.path.join( pooldir, deb[1] )
- if os.path.exists( debfile ):
- Binary(debfile, self.reject).scan_package(deb[0],True)
+ # We don't have existing contents so import them
+ log.debug( "scanning: %s" % (filename) )
+ debfile = os.path.join(pooldir, filename)
+ if os.path.exists(debfile):
+ Binary(debfile, self.reject).scan_package(ba.binary_id, True)
else:
- log.error("missing .deb: %s" % deb[1])
+ log.error("missing .deb: %s" % filename)
+
def generate(self):
"""
Generate Contents-$arch.gz files for every available arch in each given suite.
"""
- cursor = DBConn().cursor()
-
- DBConn().prepare("arches_q", arches_q)
- DBConn().prepare("contents_q", contents_q)
- DBConn().prepare("udeb_contents_q", udeb_contents_q)
-
- debtype_id=DBConn().get_override_type_id("deb")
- udebtype_id=DBConn().get_override_type_id("udeb")
-
- arch_all_id = DBConn().get_architecture_id("all")
- suites = self._suites()
-
-
- # Get our suites, and the architectures
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
- arch_list = self._arches(cursor, suite_id)
-
- file_writers = {}
-
- try:
- for arch_id in arch_list:
- file_writers[arch_id[0]] = GzippedContentWriter("dists/%s/Contents-%s.gz" % (suite, arch_id[1]))
-
- cursor.execute("EXECUTE contents_q(%d,%d);" % (suite_id, debtype_id))
-
- while True:
- r = cursor.fetchone()
- if not r:
- break
-
- filename, section, package, arch = r
-
- if not file_writers.has_key( arch ):
- continue
-
- if arch == arch_all_id:
- ## its arch all, so all contents files get it
- for writer in file_writers.values():
- writer.write(filename, section, package)
-
- else:
- file_writers[arch].write(filename, section, package)
-
- finally:
- # close all the files
- for writer in file_writers.values():
- writer.finish()
-
+ session = DBConn().session()
- # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
- # This is HORRIBLY debian specific :-/
- for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"),
- ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
+ arch_all_id = get_architecture("all", session).arch_id
- section_id = DBConn().get_section_id(section) # all udebs should be here)
- if section_id != -1:
+ # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
+ # This is HORRIBLY debian specific :-/
+ for dtype, section, fn_pattern in \
+ [('deb', None, "dists/%s/Contents-%s.gz"),
+ ('udeb', "debian-installer", "dists/%s/Contents-udeb-%s.gz"),
+ ('udeb', "non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
- # Get our suites, and the architectures
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
- arch_list = self._arches(cursor, suite_id)
+ overridetype = get_override_type(dtype, session)
- file_writers = {}
+ # For udebs, we only look in certain sections (see the for loop above)
+ if section is not None:
+ section = get_section(section, session)
- try:
- for arch_id in arch_list:
- file_writers[arch_id[0]] = GzippedContentWriter(fn_pattern % (suite, arch_id[1]))
+ # Get our suites
+ for suite in which_suites():
+ # Which architectures do we need to work on
+ arch_list = get_suite_architectures(suite.suite_name, skipsrc=True, skipall=True, session=session)
- cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id))
-
- while True:
- r = cursor.fetchone()
- if not r:
- break
-
- filename, section, package, arch = r
-
- if not file_writers.has_key( arch ):
- continue
-
- if arch == arch_all_id:
- ## its arch all, so all contents files get it
- for writer in file_writers.values():
- writer.write(filename, section, package)
-
- else:
- file_writers[arch].write(filename, section, package)
- finally:
- # close all the files
- for writer in file_writers.values():
- writer.finish()
-
-
-
-################################################################################
-
- def _suites(self):
- """
- return a list of suites to operate on
- """
- if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
- suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
- else:
- suites = Config().SubTree("Suite").List()
-
- return suites
+ # Set up our file writer dictionary
+ file_writers = {}
+ try:
+ # One file writer per arch
+ for arch in arch_list:
+ file_writers[arch.arch_id] = GzippedContentWriter(fn_pattern % (suite, arch.arch_string))
- def _arches(self, cursor, suite):
- """
- return a list of archs to operate on
- """
- arch_list = []
- cursor.execute("EXECUTE arches_q(%d)" % (suite))
- while True:
- r = cursor.fetchone()
- if not r:
- break
+ for r in get_suite_contents(suite, overridetype, section, session=session).fetchall():
+ filename, section, package, arch_id = r
- if r[1] != "source" and r[1] != "all":
- arch_list.append((r[0], r[1]))
+ if arch_id == arch_all_id:
+ # It's arch all, so all contents files get it
+ for writer in file_writers.values():
+ writer.write(filename, section, package)
+ else:
+ if file_writers.has_key(arch_id):
+ file_writers[arch_id].write(filename, section, package)
- return arch_list
+ finally:
+ # close all the files
+ for writer in file_writers.values():
+ writer.finish()
################################################################################
-
def main():
cnf = Config()
commands[args[0]](Contents())
+def which_suites(session):
+ """
+ return a list of suites to operate on
+ """
+ if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
+ suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
+ else:
+ suites = Config().SubTree("Suite").List()
+
+ return [get_suite(s.lower(), session) for s in suites]
+
+
if __name__ == '__main__':
main()
################################################################################
-import pg, sys, time
+import sys, time
import apt_pkg
+
+from daklib.dbconn import *
+from daklib.config import Config
from daklib import utils
-from daklib import database
-from daklib import logging
+from daklib import daklog
from daklib.regexes import re_comments
################################################################################
-Cnf = None
-projectB = None
Logger = None
################################################################################
################################################################################
-def process_file (file, suite, component, type, action, noaction=0):
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
+def process_file(file, suite, component, otype, mode, action, session):
+ cnf = Config()
+
+ s = get_suite(suite, session=session)
+ if s is None:
utils.fubar("Suite '%s' not recognised." % (suite))
+ suite_id = s.suite_id
- component_id = database.get_component_id(component)
- if component_id == -1:
+ c = get_component(component, session=session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
+ component_id = c.component_id
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (type))
+ o = get_override_type(otype)
+ if o is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (otype))
+ type_id = o.overridetype_id
# --set is done mostly internal for performance reasons; most
# invocations of --set will be updates and making people wait 2-3
c_removed = 0
c_error = 0
- q = projectB.query("SELECT o.package, o.priority, o.section, o.maintainer, p.priority, s.section FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s and o.priority = p.id and o.section = s.id"
- % (suite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT o.package, o.priority, o.section, o.maintainer, p.priority, s.section
+ FROM override o, priority p, section s
+ WHERE o.suite = :suiteid AND o.component = :componentid AND o.type = :typeid
+ and o.priority = p.id and o.section = s.id""",
+ {'suiteid': suite_id, 'componentid': component_id, 'typeid': type_id})
+ for i in q.fetchall():
original[i[0]] = i[1:]
start_time = time.time()
- if not noaction:
- projectB.query("BEGIN WORK")
+
+ section_cache = get_sections(session)
+ priority_cache = get_priorities(session)
+
+ # Our session is already in a transaction
+
for line in file.readlines():
line = re_comments.sub('', line).strip()
if line == "":
continue
- maintainer_override = ""
- if type == "dsc":
+ maintainer_override = None
+ if otype == "dsc":
split_line = line.split(None, 2)
if len(split_line) == 2:
(package, section) = split_line
c_error += 1
continue
- section_id = database.get_section_id(section)
- if section_id == -1:
+ if not section_cache.has_key(section):
utils.warn("'%s' is not a valid section. ['%s' in suite %s, component %s]." % (section, package, suite, component))
c_error += 1
continue
- priority_id = database.get_priority_id(priority)
- if priority_id == -1:
+
+ section_id = section_cache[section]
+
+ if not priority_cache.has_key(priority):
utils.warn("'%s' is not a valid priority. ['%s' in suite %s, component %s]." % (priority, package, suite, component))
c_error += 1
continue
+ priority_id = priority_cache[priority]
+
if new.has_key(package):
utils.warn("Can't insert duplicate entry for '%s'; ignoring all but the first. [suite %s, component %s]" % (package, suite, component))
c_error += 1
continue
new[package] = ""
+
if original.has_key(package):
(old_priority_id, old_section_id, old_maintainer_override, old_priority, old_section) = original[package]
- if action == "add" or old_priority_id == priority_id and \
+ if mode == "add" or old_priority_id == priority_id and \
old_section_id == section_id and \
old_maintainer_override == maintainer_override:
# If it's unchanged or we're in 'add only' mode, ignore it
# If it's changed, delete the old one so we can
# reinsert it with the new information
c_updated += 1
- if not noaction:
- projectB.query("DELETE FROM override WHERE suite = %s AND component = %s AND package = '%s' AND type = %s"
- % (suite_id, component_id, package, type_id))
+ if action:
+ session.execute("""DELETE FROM override WHERE suite = :suite AND component = :component
+ AND package = :package AND type = :typeid""",
+ {'suite': suite_id, 'component': component_id,
+ 'package': package, 'typeid': type_id})
+
# Log changes
if old_priority_id != priority_id:
- Logger.log(["changed priority",package,old_priority,priority])
+ Logger.log(["changed priority", package, old_priority, priority])
if old_section_id != section_id:
- Logger.log(["changed section",package,old_section,section])
+ Logger.log(["changed section", package, old_section, section])
if old_maintainer_override != maintainer_override:
- Logger.log(["changed maintainer override",package,old_maintainer_override,maintainer_override])
+ Logger.log(["changed maintainer override", package, old_maintainer_override, maintainer_override])
update_p = 1
- elif action == "change":
+ elif mode == "change":
# Ignore additions in 'change only' mode
c_skipped += 1
continue
c_added += 1
update_p = 0
- if not noaction:
- if maintainer_override:
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '%s')"
- % (suite_id, component_id, type_id, package, priority_id, section_id, maintainer_override))
+ if action:
+ if not maintainer_override:
+ m_o = None
else:
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section,maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')"
- % (suite_id, component_id, type_id, package, priority_id, section_id))
+ m_o = maintainer_override
+ session.execute("""INSERT INTO override (suite, component, type, package,
+ priority, section, maintainer)
+ VALUES (:suiteid, :componentid, :typeid,
+ :package, :priorityid, :sectionid,
+ :maintainer)""",
+ {'suiteid': suite_id, 'componentid': component_id,
+ 'typeid': type_id, 'package': package,
+ 'priorityid': priority_id, 'sectionid': section_id,
+ 'maintainer': m_o})
if not update_p:
- Logger.log(["new override",suite,component,type,package,priority,section,maintainer_override])
+ Logger.log(["new override", suite, component, otype, package,priority,section,maintainer_override])
- if action == "set":
+ if mode == "set":
# Delete any packages which were removed
for package in original.keys():
if not new.has_key(package):
- if not noaction:
- projectB.query("DELETE FROM override WHERE suite = %s AND component = %s AND package = '%s' AND type = %s"
- % (suite_id, component_id, package, type_id))
+ if action:
+ session.execute("""DELETE FROM override
+ WHERE suite = :suiteid AND component = :componentid
+ AND package = :package AND type = :typeid""",
+ {'suiteid': suite_id, 'componentid': component_id,
+ 'package': package, 'typeid': type_id})
c_removed += 1
- Logger.log(["removed override",suite,component,type,package])
+ Logger.log(["removed override", suite, component, otype, package])
- if not noaction:
- projectB.query("COMMIT WORK")
- if not Cnf["Control-Overrides::Options::Quiet"]:
+ if action:
+ session.commit()
+
+ if not cnf["Control-Overrides::Options::Quiet"]:
print "Done in %d seconds. [Updated = %d, Added = %d, Removed = %d, Skipped = %d, Errors = %d]" % (int(time.time()-start_time), c_updated, c_added, c_removed, c_skipped, c_error)
- Logger.log(["set complete",c_updated, c_added, c_removed, c_skipped, c_error])
+
+ Logger.log(["set complete", c_updated, c_added, c_removed, c_skipped, c_error])
################################################################################
-def list_overrides(suite, component, type):
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
+def list_overrides(suite, component, otype, session):
+ dat = {}
+ s = get_suite(suite, session)
+ if s is None:
utils.fubar("Suite '%s' not recognised." % (suite))
- component_id = database.get_component_id(component)
- if component_id == -1:
+ dat['suiteid'] = s.suite_id
+
+ c = get_component(component, session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
+ dat['componentid'] = c.component_id
+
+ o = get_override_type(otype)
+ if o is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
- if type == "dsc":
- q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, type_id))
- for i in q.getresult():
+ dat['typeid'] = o.overridetype_id
+
+ if otype == "dsc":
+ q = session.execute("""SELECT o.package, s.section, o.maintainer FROM override o, section s
+ WHERE o.suite = :suiteid AND o.component = :componentid
+ AND o.type = :typeid AND o.section = s.id
+ ORDER BY s.section, o.package""", dat)
+ for i in q.fetchall():
print utils.result_join(i)
else:
- q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT o.package, p.priority, s.section, o.maintainer, p.level
+ FROM override o, priority p, section s
+ WHERE o.suite = :suiteid AND o.component = :componentid
+ AND o.type = :typeid AND o.priority = p.id AND o.section = s.id
+ ORDER BY s.section, p.level, o.package""", dat)
+ for i in q.fetchall():
print utils.result_join(i[:-1])
################################################################################
def main ():
- global Cnf, projectB, Logger
+ global Logger
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a', "add", "Control-Overrides::Options::Add"),
('c', "component", "Control-Overrides::Options::Component", "HasArg"),
('h', "help", "Control-Overrides::Options::Help"),
# Default arguments
for i in [ "add", "help", "list", "quiet", "set", "change", "no-action" ]:
- if not Cnf.has_key("Control-Overrides::Options::%s" % (i)):
- Cnf["Control-Overrides::Options::%s" % (i)] = ""
- if not Cnf.has_key("Control-Overrides::Options::Component"):
- Cnf["Control-Overrides::Options::Component"] = "main"
- if not Cnf.has_key("Control-Overrides::Options::Suite"):
- Cnf["Control-Overrides::Options::Suite"] = "unstable"
- if not Cnf.has_key("Control-Overrides::Options::Type"):
- Cnf["Control-Overrides::Options::Type"] = "deb"
-
- file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
-
- if Cnf["Control-Overrides::Options::Help"]:
+ if not cnf.has_key("Control-Overrides::Options::%s" % (i)):
+ cnf["Control-Overrides::Options::%s" % (i)] = ""
+ if not cnf.has_key("Control-Overrides::Options::Component"):
+ cnf["Control-Overrides::Options::Component"] = "main"
+ if not cnf.has_key("Control-Overrides::Options::Suite"):
+ cnf["Control-Overrides::Options::Suite"] = "unstable"
+ if not cnf.has_key("Control-Overrides::Options::Type"):
+ cnf["Control-Overrides::Options::Type"] = "deb"
+
+ file_list = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+
+ if cnf["Control-Overrides::Options::Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
- action = None
+ mode = None
for i in [ "add", "list", "set", "change" ]:
- if Cnf["Control-Overrides::Options::%s" % (i)]:
- if action:
+ if cnf["Control-Overrides::Options::%s" % (i)]:
+ if mode:
utils.fubar("Can not perform more than one action at once.")
- action = i
+ mode = i
+
+ # Need an action...
+ if mode is None:
+ utils.fubar("No action specified.")
- (suite, component, otype) = (Cnf["Control-Overrides::Options::Suite"],
- Cnf["Control-Overrides::Options::Component"],
- Cnf["Control-Overrides::Options::Type"])
+ (suite, component, otype) = (cnf["Control-Overrides::Options::Suite"],
+ cnf["Control-Overrides::Options::Component"],
+ cnf["Control-Overrides::Options::Type"])
- if action == "list":
- list_overrides(suite, component, otype)
+ if mode == "list":
+ list_overrides(suite, component, otype, session)
else:
- if database.get_suite_untouchable(suite):
+ if get_suite(suite).untouchable:
utils.fubar("%s: suite is untouchable" % suite)
- noaction = 0
- if Cnf["Control-Overrides::Options::No-Action"]:
+ action = True
+ if cnf["Control-Overrides::Options::No-Action"]:
utils.warn("In No-Action Mode")
- noaction = 1
+ action = False
- Logger = logging.Logger(Cnf, "control-overrides", noaction)
+ Logger = daklog.Logger(cnf.Cnf, "control-overrides", mode)
if file_list:
for f in file_list:
- process_file(utils.open_file(f), suite, component, otype, action, noaction)
+ process_file(utils.open_file(f), suite, component, otype, mode, action, session)
else:
- process_file(sys.stdin, suite, component, otype, action, noaction)
+ process_file(sys.stdin, suite, component, otype, mode, action, session)
Logger.close()
#######################################################################################
#######################################################################################
-import pg, sys
+import sys
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
from daklib import utils
#######################################################################################
-Cnf = None
-projectB = None
Logger = None
################################################################################
#######################################################################################
-def get_id (package, version, architecture):
+def get_id(package, version, architecture, session):
if architecture == "source":
- q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
+ q = session.execute("SELECT id FROM source WHERE source = :package AND version = :version",
+ {'package': package, 'version': version})
else:
- q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
-
- ql = q.getresult()
- if not ql:
+ q = session.execute("""SELECT b.id FROM binaries b, architecture a
+ WHERE b.package = :package AND b.version = :version
+ AND (a.arch_string = :arch OR a.arch_string = 'all')
+ AND b.architecture = a.id""",
+ {'package': package, 'version': version, 'arch': architecture})
+
+ ql = q.fetchall()
+ if len(ql) < 1:
utils.warn("Couldn't find '%s_%s_%s'." % (package, version, architecture))
return None
+
if len(ql) > 1:
utils.warn("Found more than one match for '%s_%s_%s'." % (package, version, architecture))
return None
+
return ql[0][0]
#######################################################################################
-def set_suite (file, suite_id):
+def set_suite(file, suite, session):
+ suite_id = suite.suite_id
lines = file.readlines()
- projectB.query("BEGIN WORK")
+ # Our session is already in a transaction
# Build up a dictionary of what is currently in the suite
current = {}
- q = projectB.query("SELECT b.package, b.version, a.arch_string, ba.id FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+ q = session.execute("""SELECT b.package, b.version, a.arch_string, ba.id
+ FROM binaries b, bin_associations ba, architecture a
+ WHERE ba.suite = :suiteid
+ AND ba.bin = b.id AND b.architecture = a.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
key = " ".join(i[:3])
current[key] = i[3]
- q = projectB.query("SELECT s.source, s.version, sa.id FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+
+ q = session.execute("""SELECT s.source, s.version, sa.id
+ FROM source s, src_associations sa
+ WHERE sa.suite = :suiteid
+ AND sa.source = s.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
key = " ".join(i[:2]) + " source"
current[key] = i[2]
(package, version, architecture) = key.split()
pkid = current[key]
if architecture == "source":
- q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (pkid))
+ session.execute("""DELETE FROM src_associations WHERE id = :pkid""", {'pkid': pkid})
else:
- q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (pkid))
+ session.execute("""DELETE FROM bin_associations WHERE id = :pkid""", {'pkid': pkid})
Logger.log(["removed", key, pkid])
# Check to see which packages need added and add them
for key in desired.keys():
if not current.has_key(key):
(package, version, architecture) = key.split()
- pkid = get_id (package, version, architecture)
+ pkid = get_id (package, version, architecture, session)
if not pkid:
continue
if architecture == "source":
- q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO src_associations (suite, source)
+ VALUES (:suiteid, :pkid)""", {'suiteid': suite_id, 'pkid': pkid})
else:
- q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO bin_associations (suite, bin)
+ VALUES (:suiteid, :pkid)""", {'suiteid': suite_id, 'pkid': pkid})
Logger.log(["added", key, pkid])
- projectB.query("COMMIT WORK")
+ session.commit()
#######################################################################################
-def process_file (file, suite, action):
-
- suite_id = database.get_suite_id(suite)
-
+def process_file(file, suite, action, session):
if action == "set":
- set_suite (file, suite_id)
+ set_suite(file, suite, session)
return
- lines = file.readlines()
+ suite_id = suite.suite_id
- projectB.query("BEGIN WORK")
+ lines = file.readlines()
+ # Our session is already in a transaction
for line in lines:
split_line = line.strip().split()
if len(split_line) != 3:
(package, version, architecture) = split_line
- pkid = get_id(package, version, architecture)
+ pkid = get_id(package, version, architecture, session)
if not pkid:
continue
if architecture == "source":
- # Find the existing assoications ID, if any
- q = projectB.query("SELECT id FROM src_associations WHERE suite = %s and source = %s" % (suite_id, pkid))
- ql = q.getresult()
- if not ql:
- assoication_id = None
+ # Find the existing association ID, if any
+ q = session.execute("""SELECT id FROM src_associations
+ WHERE suite = :suiteid and source = :pkid""",
+ {'suiteid': suite_id, 'pkid': pkid})
+ ql = q.fetchall()
+ if len(ql) < 1:
+ association_id = None
else:
- assoication_id = ql[0][0]
+ association_id = ql[0][0]
+
# Take action
if action == "add":
- if assoication_id:
+ if association_id:
utils.warn("'%s_%s_%s' already exists in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO src_associations (suite, source)
+ VALUES (:suiteid, :pkid)""",
+ {'suiteid': suite_id, 'pkid': pkid})
elif action == "remove":
- if assoication_id == None:
+ if association_id == None:
utils.warn("'%s_%s_%s' doesn't exist in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (assoication_id))
+ session.execute("""DELETE FROM src_associations WHERE id = :pkid""", {'pkid': association_id})
else:
- # Find the existing assoications ID, if any
- q = projectB.query("SELECT id FROM bin_associations WHERE suite = %s and bin = %s" % (suite_id, pkid))
- ql = q.getresult()
- if not ql:
- assoication_id = None
+ # Find the existing associations ID, if any
+ q = session.execute("""SELECT id FROM bin_associations
+ WHERE suite = :suiteid and bin = :pkid""",
+ {'suiteid': suite_id, 'pkid': pkid})
+ ql = q.fetchall()
+ if len(ql) < 1:
+ association_id = None
else:
- assoication_id = ql[0][0]
+ association_id = ql[0][0]
+
# Take action
if action == "add":
- if assoication_id:
+ if association_id:
utils.warn("'%s_%s_%s' already exists in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO bin_associations (suite, bin)
+ VALUES (%s, %s)""",
+ {'suiteid': suite_id, 'pkid': pkid})
elif action == "remove":
- if assoication_id == None:
+ if association_id == None:
utils.warn("'%s_%s_%s' doesn't exist in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (assoication_id))
+ session.execute("""DELETE FROM bin_associations WHERE id = :pkid""", {'pkid': association_id})
- projectB.query("COMMIT WORK")
+ session.commit()
#######################################################################################
-def get_list (suite):
- suite_id = database.get_suite_id(suite)
+def get_list(suite, session):
+ suite_id = suite.suite_id
# List binaries
- q = projectB.query("SELECT b.package, b.version, a.arch_string FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+ q = session.execute("""SELECT b.package, b.version, a.arch_string
+ FROM binaries b, bin_associations ba, architecture a
+ WHERE ba.suite = :suiteid
+ AND ba.bin = b.id AND b.architecture = a.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
print " ".join(i)
# List source
- q = projectB.query("SELECT s.source, s.version FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+ q = session.execute("""SELECT s.source, s.version
+ FROM source s, src_associations sa
+ WHERE sa.suite = :suiteid
+ AND sa.source = s.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
print " ".join(i) + " source"
#######################################################################################
def main ():
- global Cnf, projectB, Logger
+ global Logger
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a',"add","Control-Suite::Options::Add", "HasArg"),
('h',"help","Control-Suite::Options::Help"),
('s',"set", "Control-Suite::Options::Set", "HasArg")]
for i in ["add", "help", "list", "remove", "set", "version" ]:
- if not Cnf.has_key("Control-Suite::Options::%s" % (i)):
- Cnf["Control-Suite::Options::%s" % (i)] = ""
+ if not cnf.has_key("Control-Suite::Options::%s" % (i)):
+ cnf["Control-Suite::Options::%s" % (i)] = ""
try:
- file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
+ file_list = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv);
except SystemError, e:
print "%s\n" % e
usage(1)
- Options = Cnf.SubTree("Control-Suite::Options")
+ Options = cnf.SubTree("Control-Suite::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"],int(Cnf["DB::Port"]))
-
- database.init(Cnf, projectB)
+ session = DBConn().session()
action = None
for i in ("add", "list", "remove", "set"):
- if Cnf["Control-Suite::Options::%s" % (i)] != "":
- suite = Cnf["Control-Suite::Options::%s" % (i)]
- if database.get_suite_id(suite) == -1:
- utils.fubar("Unknown suite '%s'." %(suite))
+ if cnf["Control-Suite::Options::%s" % (i)] != "":
+ suite_name = cnf["Control-Suite::Options::%s" % (i)]
+ suite = get_suite(suite_name, session=session)
+ if suite is None:
+ utils.fubar("Unknown suite '%s'." % (suite_name))
else:
if action:
utils.fubar("Can only perform one action at a time.")
utils.fubar("No action specified.")
# Safety/Sanity check
- if action == "set" and suite not in ["testing", "etch-m68k"]:
- utils.fubar("Will not reset suite %s" % (suite))
+ # XXX: This should be stored in the database
+ if action == "set" and suite_name not in ["testing", "etch-m68k"]:
+ utils.fubar("Will not reset suite %s" % (suite_name))
if action == "list":
- get_list(suite)
+ get_list(suite, session)
else:
- Logger = logging.Logger(Cnf, "control-suite")
+ Logger = daklog.Logger(cnf.Cnf, "control-suite")
if file_list:
for f in file_list:
- process_file(utils.open_file(f), suite, action)
+ process_file(utils.open_file(f), suite, action, session)
else:
- process_file(sys.stdin, suite, action)
+ process_file(sys.stdin, suite, action, session)
Logger.close()
#######################################################################################
# you might as well write some letters to God about how unfair entropy
# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au
-## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add overrides, avoid ANAIS for duplicated packages
+## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of
+## duplicate source package stuff, improve experimental ?, add overrides,
+## avoid ANAIS for duplicated packages
################################################################################
-import commands, pg, os, sys, time, re
+import commands, os, sys, time, re
import apt_pkg
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_extract_src_version
################################################################################
-Cnf = None
-projectB = None
-suite = "unstable" # Default
-suite_id = None
no_longer_in_suite = {}; # Really should be static to add_nbs, but I'm lazy
source_binaries = {}
################################################################################
-def add_nbs(nbs_d, source, version, package):
+def add_nbs(nbs_d, source, version, package, suite_id, session):
# Ensure the package is still in the suite (someone may have already removed it)
if no_longer_in_suite.has_key(package):
return
else:
- q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package))
- if not q.getresult():
+ q = session.execute("""SELECT b.id FROM binaries b, bin_associations ba
+ WHERE ba.bin = b.id AND ba.suite = :suite_id
+ AND b.package = suite_id LIMIT 1""" % (suite_id, package))
+ if not q.fetchall():
no_longer_in_suite[package] = ""
return
################################################################################
# Check for packages built on architectures they shouldn't be.
-def do_anais(architecture, binaries_list, source):
+def do_anais(architecture, binaries_list, source, session):
if architecture == "any" or architecture == "all":
return ""
for arch in architecture.split():
architectures[arch.strip()] = ""
for binary in binaries_list:
- q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary))
- ql = q.getresult()
+ q = session.execute("""SELECT a.arch_string, b.version
+ FROM binaries b, bin_associations ba, architecture a
+ WHERE ba.suite = :suiteid AND ba.bin = b.id
+ AND b.architecture = a.id AND b.package = :package""",
+ {'suiteid': suite_id, 'package': binary})
versions = []
- for i in ql:
+ for i in q.fetchall():
arch = i[0]
version = i[1]
if architectures.has_key(arch):
print
def parse_nfu(architecture):
+ cnf = Config()
# utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
r = re.compile("^\w+/([^_]+)_.*: Not-For-Us")
ret = set()
- filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
+ filename = "%s/%s-all.txt" % (cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
# Not all architectures may have a wanna-build dump, so we want to ignore missin
# files
################################################################################
-def do_nviu():
- experimental_id = database.get_suite_id("experimental")
- if experimental_id == -1:
+def do_newer_version(lowersuite_name, highersuite_name, code, session):
+ lowersuite = get_suite(lowersuite_name, session)
+ if not lowersuite:
+ return
+
+ highersuite = get_suite(highersuite_name, session)
+ if not highersuite:
return
- # Check for packages in experimental obsoleted by versions in unstable
- q = projectB.query("""
-SELECT s.source, s.version AS experimental, s2.version AS unstable
+
+ # Check for packages in $highersuite obsoleted by versions in $lowersuite
+ q = session.execute("""
+SELECT s.source, s.version AS lower, s2.version AS higher
FROM src_associations sa, source s, source s2, src_associations sa2
- WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id
+ WHERE sa.suite = :highersuite_id AND sa2.suite = :lowersuite_id AND sa.source = s.id
AND sa2.source = s2.id AND s.source = s2.source
- AND s.version < s2.version""" % (experimental_id,
- database.get_suite_id("unstable")))
- ql = q.getresult()
+ AND s.version < s2.version""" % {'lowersuite_id': lowersuite.suite_id,
+ 'highersuite_id': highersuite.suite_id})
+ ql = q.fetchall()
if ql:
- nviu_to_remove = []
- print "Newer version in unstable"
- print "-------------------------"
+ nv_to_remove = []
+ print "Newer version in %s" % lowersuite.suite_name
+ print "-----------------" + "-" * len(lowersuite.suite_name)
print
for i in ql:
- (source, experimental_version, unstable_version) = i
- print " o %s (%s, %s)" % (source, experimental_version, unstable_version)
- nviu_to_remove.append(source)
+ (source, higher_version, lower_version) = i
+ print " o %s (%s, %s)" % (source, higher_version, lower_version)
+ nv_to_remove.append(source)
print
print "Suggested command:"
- print " dak rm -m \"[auto-cruft] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove))
+ print " dak rm -m \"[auto-cruft] %s\" -s %s %s" % (code, highersuite.suite_name,
+ " ".join(nv_to_remove))
print
################################################################################
print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
print
-def get_suite_binaries():
+def get_suite_binaries(suite, session):
# Initalize a large hash table of all binary packages
binaries = {}
- before = time.time()
- sys.stderr.write("[Getting a list of binary packages in %s..." % (suite))
- q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id))
- ql = q.getresult()
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
- for i in ql:
+ print "Getting a list of binary packages in %s..." % suite.suite_name
+ q = session.execute("""SELECT distinct b.package
+ FROM binaries b, bin_associations ba
+ WHERE ba.suite = :suiteid AND ba.bin = b.id""",
+ {'suiteid': suite.suite_id})
+ for i in q.fetchall():
binaries[i[0]] = ""
return binaries
################################################################################
def main ():
- global Cnf, projectB, suite, suite_id, source_binaries, source_versions
+ global suite, suite_id, source_binaries, source_versions
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Cruft-Report::Options::Help"),
('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
('s',"suite","Cruft-Report::Options::Suite","HasArg"),
('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
for i in [ "help" ]:
- if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
- Cnf["Cruft-Report::Options::%s" % (i)] = ""
- Cnf["Cruft-Report::Options::Suite"] = Cnf["Dinstall::DefaultSuite"]
+ if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
+ cnf["Cruft-Report::Options::%s" % (i)] = ""
+ cnf["Cruft-Report::Options::Suite"] = cnf["Dinstall::DefaultSuite"]
- if not Cnf.has_key("Cruft-Report::Options::Mode"):
- Cnf["Cruft-Report::Options::Mode"] = "daily"
+ if not cnf.has_key("Cruft-Report::Options::Mode"):
+ cnf["Cruft-Report::Options::Mode"] = "daily"
- if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
- Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
+ if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
+ cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Cruft-Report::Options")
+ Options = cnf.SubTree("Cruft-Report::Options")
if Options["Help"]:
usage()
utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
usage(1)
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
nfu_packages = {}
- suite = Options["Suite"]
- suite_id = database.get_suite_id(suite)
+ suite = get_suite(Options["Suite"].lower(), session)
+ suite_id = suite.suite_id
bin_not_built = {}
if "bnb" in checks:
- bins_in_suite = get_suite_binaries()
+ bins_in_suite = get_suite_binaries(suite)
# Checks based on the Sources files
- components = Cnf.ValueList("Suite::%s::Components" % (suite))
+ components = cnf.ValueList("Suite::%s::Components" % (suite))
for component in components:
- filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component)
+ filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite.suite_name, component)
# apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
bin_not_built[source][binary] = ""
if "anais" in checks:
- anais_output += do_anais(architecture, binaries_list, source)
+ anais_output += do_anais(architecture, binaries_list, source, session)
# Check for duplicated packages and build indices for checking "no source" later
source_index = component + '/' + source
# Checks based on the Packages files
check_components = components[:]
- if suite != "experimental":
+ if suite.suite_name != "experimental":
check_components.append('main/debian-installer');
+
for component in check_components:
- architectures = filter(utils.real_arch, database.get_suite_architectures(suite))
+ architectures = [ a.arch_string for a in get_suite_architectures(suite.suite_name,
+ skipsrc=True, skipall=True,
+ session=session) ]
for architecture in architectures:
- if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
- continue
- filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture)
+ if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
+ continue
+ filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suite.suite_name, component, architecture)
# apt_pkg.ParseTagFile needs a real file handle
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
latest_version = versions.pop()
source_version = source_versions.get(source,"0")
if apt_pkg.VersionCompare(latest_version, source_version) == 0:
- add_nbs(dubious_nbs, source, latest_version, package)
+ add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
else:
- add_nbs(real_nbs, source, latest_version, package)
+ add_nbs(real_nbs, source, latest_version, package, suite_id, session)
if "nviu" in checks:
- do_nviu()
+ do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nbs" in checks:
do_nbs(real_nbs)
"Syncs fingerprint and uid tables with Debian LDAP db"),
("import-users-from-passwd",
"Sync PostgreSQL users with passwd file"),
+ ("admin",
+ "Perform administration on the dak database"),
("init-db",
"Update the database to match the conf file"),
("update-db",
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Make sure we always have primary keys
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+
+################################################################################
+
+def do_update(self):
+ print "Adding primary keys to various tables"
+
+ try:
+ c = self.db.cursor()
+ c.execute("ALTER TABLE content_associations ADD PRIMARY KEY (id)")
+ c.execute("ALTER TABLE override ADD PRIMARY KEY (suite, component, package, type)")
+ c.execute("ALTER TABLE pending_content_associations ADD PRIMARY KEY (id)")
+ c.execute("ALTER TABLE queue_build ADD PRIMARY KEY (suite, queue, filename)")
+ c.execute("ALTER TABLE suite_architectures ADD PRIMARY KEY (suite, architecture)")
+
+ c.execute("UPDATE config SET value = '14' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply process-new update 14, rollback issued. Error message : %s" % (str(msg))
import sys
import apt_pkg
-from daklib import queue
+from daklib.changes import Changes
from daklib import utils
################################################################################
if Options["Help"]:
usage()
- k = queue.Upload(Cnf)
+
for arg in sys.argv[1:]:
arg = utils.validate_changes_file_arg(arg,require_changes=-1)
- k.pkg.changes_file = arg
- print "%s:" % (arg)
- k.init_vars()
- k.update_vars()
-
- changes = k.pkg.changes
- print " Changes:"
- # Mandatory changes fields
- for i in [ "source", "version", "maintainer", "urgency", "changedby822",
- "changedby2047", "changedbyname", "maintainer822",
- "maintainer2047", "maintainername", "maintaineremail",
- "fingerprint", "changes" ]:
- print " %s: %s" % (i.capitalize(), changes[i])
- del changes[i]
- # Mandatory changes lists
- for i in [ "distribution", "architecture", "closes" ]:
- print " %s: %s" % (i.capitalize(), " ".join(changes[i].keys()))
- del changes[i]
- # Optional changes fields
- for i in [ "changed-by", "filecontents", "format", "adv id" ]:
- if changes.has_key(i):
- print " %s: %s" % (i.capitalize(), changes[i])
- del changes[i]
- print
- if changes:
- utils.warn("changes still has following unrecognised keys: %s" % (changes.keys()))
-
- dsc = k.pkg.dsc
- print " Dsc:"
- for i in [ "source", "version", "maintainer", "fingerprint", "uploaders",
- "bts changelog" ]:
- if dsc.has_key(i):
- print " %s: %s" % (i.capitalize(), dsc[i])
- del dsc[i]
- print
- if dsc:
- utils.warn("dsc still has following unrecognised keys: %s" % (dsc.keys()))
-
- files = k.pkg.files
- print " Files:"
- for f in files.keys():
- print " %s:" % (f)
- for i in [ "package", "version", "architecture", "type", "size",
- "md5sum", "sha1sum", "sha256sum", "component", "location id",
- "source package", "source version", "maintainer", "dbtype",
- "files id", "new", "section", "priority", "pool name" ]:
- if files[f].has_key(i):
- print " %s: %s" % (i.capitalize(), files[f][i])
- del files[f][i]
- if files[f]:
- utils.warn("files[%s] still has following unrecognised keys: %s" % (f, files[f].keys()))
- print
-
- dsc_files = k.pkg.dsc_files
- print " Dsc Files:"
- for f in dsc_files.keys():
- print " %s:" % (f)
- # Mandatory fields
- for i in [ "size", "md5sum" ]:
- print " %s: %s" % (i.capitalize(), dsc_files[f][i])
- del dsc_files[f][i]
- # Optional fields
- for i in [ "files id" ]:
- if dsc_files[f].has_key(i):
- print " %s: %s" % (i.capitalize(), dsc_files[f][i])
- del dsc_files[f][i]
- if dsc_files[f]:
- utils.warn("dsc_files[%s] still has following unrecognised keys: %s" % (f, dsc_files[f].keys()))
+ k = Changes()
+ k.load_dot_dak(arg)
+ print arg
+ print k
################################################################################
import errno
import os
-import pg
import re
import sys
import md5
import apt_inst
import shutil
import commands
-from daklib import database
+
from daklib import utils
+from daklib.dbconn import DBConn, get_binary_from_name_suite
from daklib.regexes import html_escaping, re_html_escaping, re_version, re_spacestrip, \
re_contrib, re_nonfree, re_localhost, re_newlinespace, \
re_package, re_doc_directory
################################################################################
Cnf = None
-projectB = None
-
Cnf = utils.get_conf()
-projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
-database.init(Cnf, projectB)
printed_copyrights = {}
package_relations = {} #: Store relations of packages for later output
suite_where = " ='%s'" % suite
comma_count = 1
+ session = DBConn().session()
for l in depends_tree:
if (comma_count >= 2):
result += ", "
result += " | "
# doesn't do version lookup yet.
- q = projectB.query("SELECT DISTINCT(b.package), b.version, c.name, su.suite_name FROM binaries b, files fi, location l, component c, bin_associations ba, suite su WHERE b.package='%s' AND b.file = fi.id AND fi.location = l.id AND l.component = c.id AND ba.bin=b.id AND ba.suite = su.id AND su.suite_name %s ORDER BY b.version desc" % (d['name'], suite_where))
- ql = q.getresult()
- if ql:
- i = ql[0]
+ res = get_binary_from_name_suite(d['name'], suite_where)
+ if res.rowcount > 0:
+ i = res.fetchone()
adepends = d['name']
if d['version'] != '' :
# else: => byhand
def main ():
- global Cnf, projectB, db_files, waste, excluded
+ global Cnf, db_files, waste, excluded
# Cnf = utils.get_conf()
################################################################################
-import ldap, pg, sys, time
+import ldap, sys, time
import apt_pkg
-from daklib import utils
-################################################################################
-
-Cnf = None
-projectB = None
+from daklib.dbconn import *
+from daklib.config import Config
################################################################################
return ret[0]
def main():
- global Cnf, projectB
+ cnf = Config()
- Cnf = utils.get_conf()
Arguments = [('h',"help","Find-Null-Maintainers::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Find-Null-Maintainers::Options::%s" % (i)):
- Cnf["Find-Null-Maintainers::Options::%s" % (i)] = ""
+ if not cnf.has_key("Find-Null-Maintainers::Options::%s" % (i)):
+ cnf["Find-Null-Maintainers::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Find-Null-Maintainers::Options")
+ Options = cnf.SubTree("Find-Null-Maintainers::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ session = DBConn().session()
- before = time.time()
- sys.stderr.write("[Getting info from the LDAP server...")
- LDAPDn = Cnf["Import-LDAP-Fingerprints::LDAPDn"]
- LDAPServer = Cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ print "Getting info from the LDAP server..."
+ LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
+ LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
- "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Import-Users-From-Passwd::ValidGID"]),
+ "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "cn", "mn", "sn", "createTimestamp"])
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
db_uid = {}
db_unstable_uid = {}
- before = time.time()
- sys.stderr.write("[Getting UID info for entire archive...")
- q = projectB.query("SELECT DISTINCT u.uid FROM uid u, fingerprint f WHERE f.uid = u.id;")
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
- for i in q.getresult():
+ print "Getting UID info for entire archive..."
+ q = session.execute("SELECT DISTINCT u.uid FROM uid u, fingerprint f WHERE f.uid = u.id")
+ for i in q.fetchall():
db_uid[i[0]] = ""
- before = time.time()
- sys.stderr.write("[Getting UID info for unstable...")
- q = projectB.query("""
+ print "Getting UID info for unstable..."
+ q = session.execute("""
SELECT DISTINCT u.uid FROM suite su, src_associations sa, source s, fingerprint f, uid u
WHERE f.uid = u.id AND sa.source = s.id AND sa.suite = su.id
AND su.suite_name = 'unstable' AND s.sig_fpr = f.id
SELECT DISTINCT u.uid FROM suite su, bin_associations ba, binaries b, fingerprint f, uid u
WHERE f.uid = u.id AND ba.bin = b.id AND ba.suite = su.id
AND su.suite_name = 'unstable' AND b.sig_fpr = f.id""")
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
- for i in q.getresult():
+ for i in q.fetchall():
db_unstable_uid[i[0]] = ""
now = time.time()
import subprocess
import time
import apt_pkg
-import pg
+
from daklib import utils
-from daklib import database
+from daklib.dbconn import get_suite
################################################################################
-projectB = None
Cnf = None
Logger = None
Options = None
def main():
- global Cnf, Options, Logger, projectB
+ global Cnf, Options, Logger
os.umask(0002)
if Options.has_key("RootDir"): Cnf["Dir::Root"] = Options["RootDir"]
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
if not suites:
suites = Cnf.SubTree("Suite").List()
- for suite in suites:
+ for suitename in suites:
print "Processing: " + suite
- SuiteBlock = Cnf.SubTree("Suite::" + suite)
+ SuiteBlock = Cnf.SubTree("Suite::" + suitename)
+
+ suiteobj = get_suite(suitename.lower())
- if database.get_suite_untouchable(suite):
+ # Use the canonical version of the suite name
+ suite = suite.suite_name
+
+ if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
- suite = suite.lower()
-
- architectures = database.get_suite_architectures(suite)
- if architectures == None:
- architectures = []
+ architectures = get_suite_architectures(suite, skipall=True)
if SuiteBlock.has_key("Components"):
components = SuiteBlock.ValueList("Components")
print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename)
continue
- for architecture in architectures:
- if architecture == "all":
- continue
+ for archobj in architectures:
+ architecture = archobj.arch_string
if architecture != "source":
# Process Contents
################################################################################
-import sys, os, stat, time, pg
+import sys, os, stat, time
import gzip, bz2
import apt_pkg
+
from daklib import utils
-from daklib import database
from daklib.dak_exceptions import *
+from daklib.dbconn import *
################################################################################
Cnf = None
-projectB = None
out = None
AptCnf = None
################################################################################
def main ():
- global Cnf, AptCnf, projectB, out
+ global Cnf, AptCnf, out
out = sys.stdout
Cnf = utils.get_conf()
AptCnf = apt_pkg.newConfiguration()
apt_pkg.ReadConfigFileISC(AptCnf, Options["Apt-Conf"])
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
if not suites:
suites = Cnf.SubTree("Suite").List()
- for suite in suites:
- print "Processing: " + suite
+ for suitename in suites:
+ print "Processing: " + suitename
SuiteBlock = Cnf.SubTree("Suite::" + suite)
+ suiteobj = get_suite(suitename)
+
+ suite = suite.suite_name.lower()
- if database.get_suite_untouchable(suite) and not Options["Force-Touch"]:
+ if suite.untouchable and not Options["Force-Touch"]:
print "Skipping: " + suite + " (untouchable)"
continue
- suite = suite.lower()
-
- origin = SuiteBlock["Origin"]
- label = SuiteBlock.get("Label", origin)
- codename = SuiteBlock.get("CodeName", "")
-
+ origin = suite.origin
+ label = suite.label or suite.origin
+ codename = suite.codename or ""
version = ""
- description = ""
-
- q = projectB.query("SELECT version, description FROM suite WHERE suite_name = '%s'" % (suite))
- qs = q.getresult()
- if len(qs) == 1:
- if qs[0][0] != "-": version = qs[0][0]
- if qs[0][1]: description = qs[0][1]
+ if suite.version and suite.version != '-':
+ version = suite.version
+ description = suite.description or ""
- architectures = database.get_suite_architectures(suite)
- if architectures == None:
- architectures = []
+ architectures = get_suite_architectures(suite, skipall=True, skipsrc=True)
if SuiteBlock.has_key("NotAutomatic"):
notautomatic = "yes"
print Cnf["Dir::Root"] + tree + "/Release"
out = open(Cnf["Dir::Root"] + tree + "/Release", "w")
- out.write("Origin: %s\n" % (origin))
+ out.write("Origin: %s\n" % (suiteobj.origin))
out.write("Label: %s\n" % (label))
out.write("Suite: %s\n" % (suite))
if version != "":
if notautomatic != "":
out.write("NotAutomatic: %s\n" % (notautomatic))
- out.write("Architectures: %s\n" % (" ".join(filter(utils.real_arch, architectures))))
+ out.write("Architectures: %s\n" % (" ".join([a.arch_string for a in architectures])))
if components:
out.write("Components: %s\n" % (" ".join(components)))
################################################################################
-from daklib import database
-from daklib import utils
import sys, os, re
-import apt_pkg, pg, ldap, email.Utils
+import apt_pkg, ldap, email.Utils
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import utils
+
# Globals
-Cnf = None
Options = None
-projectB = None
################################################################################
-def get_uid_info():
+def get_uid_info(session):
byname = {}
byid = {}
- q = projectB.query("SELECT id, uid, name FROM uid")
- for (keyid, uid, name) in q.getresult():
+ q = session.execute("SELECT id, uid, name FROM uid")
+ for (keyid, uid, name) in q.fetchall():
byname[uid] = (keyid, name)
byid[keyid] = (uid, name)
return (byname, byid)
-def get_fingerprint_info():
+def get_fingerprint_info(session):
fins = {}
- q = projectB.query("SELECT f.fingerprint, f.id, f.uid, f.keyring FROM fingerprint f")
- for (fingerprint, fingerprint_id, uid, keyring) in q.getresult():
+ q = session.execute("SELECT f.fingerprint, f.id, f.uid, f.keyring FROM fingerprint f")
+ for (fingerprint, fingerprint_id, uid, keyring) in q.fetchall():
fins[fingerprint] = (uid, fingerprint_id, keyring)
return fins
################################################################################
-class Keyring:
+class Keyring(object):
gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
" --with-colons --fingerprint --fingerprint"
keys = {}
return "".join(esclist)
def __init__(self, keyring):
+ self.cnf = Config()
k = os.popen(self.gpg_invocation % keyring, "r")
keys = self.keys
key = None
return ({}, {})
def import_users_from_ldap(self):
- LDAPDn = Cnf["Import-LDAP-Fingerprints::LDAPDn"]
- LDAPServer = Cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ LDAPDn = self.cnf["Import-LDAP-Fingerprints::LDAPDn"]
+ LDAPServer = self.cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
- "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Import-Users-From-Passwd::ValidGID"]),
+ "(&(keyfingerprint=*)(gidnumber=%s))" % (self.cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "keyfingerprint", "cn", "mn", "sn"])
ldap_fin_uid_id = {}
################################################################################
def main():
- global Cnf, projectB, Options
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Import-Keyring::Options::Help"),
('L',"import-ldap-users","Import-Keyring::Options::Import-Ldap-Users"),
('U',"generate-users","Import-Keyring::Options::Generate-Users", "HasArg"),
]
for i in [ "help", "report-changes", "generate-users", "import-ldap-users" ]:
- if not Cnf.has_key("Import-Keyring::Options::%s" % (i)):
- Cnf["Import-Keyring::Options::%s" % (i)] = ""
+ if not cnf.has_key("Import-Keyring::Options::%s" % (i)):
+ cnf["Import-Keyring::Options::%s" % (i)] = ""
- keyring_names = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ keyring_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
### Parse options
- Options = Cnf.SubTree("Import-Keyring::Options")
+ Options = cnf.SubTree("Import-Keyring::Options")
if Options["Help"]:
usage()
changes = [] # (uid, changes strings)
### Initialise
-
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
- projectB.query("BEGIN WORK")
+ session = DBConn().session()
### Cache all the existing fingerprint entries
-
- db_fin_info = get_fingerprint_info()
+ db_fin_info = get_fingerprint_info(session)
### Parse the keyring
keyring = Keyring(keyringname)
is_dm = "false"
- if Cnf.has_key("Import-Keyring::"+keyringname+"::Debian-Maintainer"):
- projectB.query("UPDATE keyrings SET debian_maintainer = '%s' WHERE name = '%s'" % (Cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"], keyringname.split("/")[-1]))
- is_dm = Cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"]
+ if cnf.has_key("Import-Keyring::"+keyringname+"::Debian-Maintainer"):
+ session.execute("UPDATE keyrings SET debian_maintainer = :dm WHERE name = :name",
+ {'dm': cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"],
+ 'name': keyringname.split("/")[-1]})
+
+ is_dm = cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"]
keyring_id = database.get_or_set_keyring_id(
keyringname.split("/")[-1])
(desuid_byname, desuid_byid) = keyring.generate_desired_users()
### Cache all the existing uid entries
- (db_uid_byname, db_uid_byid) = get_uid_info()
+ (db_uid_byname, db_uid_byid) = get_uid_info(session)
### Update full names of applicable users
for keyid in desuid_byid.keys():
oname = db_uid_byid[keyid][1]
if name and oname != name:
changes.append((uid[1], "Full name: %s" % (name)))
- projectB.query("UPDATE uid SET name = '%s' WHERE id = %s" %
- (pg.escape_string(name), keyid))
+ session.execute("UPDATE uid SET name = :name WHERE id = :keyid",
+ {'name': name, 'keyid': keyid})
# The fingerprint table (fpr) points to a uid and a keyring.
# If the uid is being decided here (ldap/generate) we set it to it.
if kr != keyring_id: continue
if f in fpr: continue
changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s" % (f)))
- projectB.query("UPDATE fingerprint SET keyring = NULL WHERE id = %d" % (fid))
+ session.execute("UPDATE fingerprint SET keyring = NULL WHERE id = :fprid", {'fprid': fid})
# For the keys in this keyring, add/update any fingerprints that've
# changed.
if oldfid == -1:
changes.append((newuiduid, "Added key: %s" % (f)))
if newuid:
- projectB.query("INSERT INTO fingerprint (fingerprint, uid, keyring) VALUES ('%s', %d, %d)" % (f, newuid, keyring_id))
+ session.execute("""INSERT INTO fingerprint (fingerprint, uid, keyring)
+ VALUES (:fpr, :uid, :keyring)""",
+ {'fpr': f, 'uid': uid, 'keyring': keyring_id})
else:
- projectB.query("INSERT INTO fingerprint (fingerprint, keyring) VALUES ('%s', %d)" % (f, keyring_id))
+ session.execute("""INSERT INTO fingerprint (fingerprint, keyring)
+ VALUES (:fpr, :keyring)""",
+ {'fpr': f, 'keyring': keyring_id})
else:
if newuid and olduid != newuid:
if olduid != -1:
else:
changes.append((newuiduid, "Linked key: %s" % f))
changes.append((newuiduid, " (formerly unowned)"))
- projectB.query("UPDATE fingerprint SET uid = %d WHERE id = %d" % (newuid, oldfid))
+ session.execute("UPDATE fingerprint SET uid = :uid WHERE id = :fpr",
+ {'uid': newuid, 'fpr': oldfid})
if oldkid != keyring_id:
# Only change the keyring if it won't result in a loss of permissions
- q = projectB.query("SELECT debian_maintainer FROM keyrings WHERE id = '%d'" % (keyring_id))
- if is_dm == "false" and q.getresult()[0][0] == 'f':
- projectB.query("UPDATE fingerprint SET keyring = %d WHERE id = %d" % (keyring_id, oldfid))
+ q = session.execute("SELECT debian_maintainer FROM keyrings WHERE id = :keyring",
+ {'keyring': keyring_id})
+ if is_dm == "false" and not q.fetchall()[0][0]:
+ session.execute("UPDATE fingerprint SET keyring = :keyring WHERE id = :fpr",
+ {'keyring': keyring_id, 'fpr': oldfid})
else:
print "Key %s exists in both DM and DD keyrings. Not demoting." % (f)
# All done!
-
- projectB.query("COMMIT WORK")
+ session.commit()
changesd = {}
for (k, v) in changes:
################################################################################
-import commands, ldap, pg, re, sys
+import commands, ldap, re, sys
import apt_pkg
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_gpg_fingerprint, re_debian_address
################################################################################
-Cnf = None
-projectB = None
-
-################################################################################
-
def usage(exit_code=0):
print """Usage: dak import-ldap-fingerprints
Syncs fingerprint and uid tables with a debian.org LDAP DB
name += get_ldap_value(entry, "sn")
return name.rstrip()
-def escape_string(str):
- return str.replace("'", "\\'")
-
def main():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Import-LDAP-Fingerprints::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Import-LDAP-Fingerprints::Options::%s" % (i)):
- Cnf["Import-LDAP-Fingerprints::Options::%s" % (i)] = ""
+ if not cnf.has_key("Import-LDAP-Fingerprints::Options::%s" % (i)):
+ cnf["Import-LDAP-Fingerprints::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Import-LDAP-Fingerprints::Options")
+ Options = cnf.SubTree("Import-LDAP-Fingerprints::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
- LDAPDn = Cnf["Import-LDAP-Fingerprints::LDAPDn"]
- LDAPServer = Cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
+ LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
- "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Import-Users-From-Passwd::ValidGID"]),
+ "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "keyfingerprint", "cn", "mn", "sn"])
- projectB.query("BEGIN WORK")
-
+ # Our database session is already in a transaction
# Sync LDAP with DB
db_fin_uid = {}
db_uid_name = {}
ldap_fin_uid_id = {}
- q = projectB.query("""
+ q = session.execute("""
SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
UNION SELECT f.fingerprint, f.id, null FROM fingerprint f where f.uid is null""")
- for i in q.getresult():
+ for i in q.fetchall():
(fingerprint, fingerprint_id, uid) = i
db_fin_uid[fingerprint] = (uid, fingerprint_id)
- q = projectB.query("SELECT id, name FROM uid")
- for i in q.getresult():
+ q = session.execute("SELECT id, name FROM uid")
+ for i in q.fetchall():
(uid, name) = i
db_uid_name[uid] = name
for i in Attrs:
entry = i[1]
fingerprints = entry["keyFingerPrint"]
- uid = entry["uid"][0]
+ uid_name = entry["uid"][0]
name = get_ldap_name(entry)
- uid_id = database.get_or_set_uid_id(uid)
+ uid = get_or_set_uid(uid_name, session)
+ uid_id = uid.uid_id
if not db_uid_name.has_key(uid_id) or db_uid_name[uid_id] != name:
- q = projectB.query("UPDATE uid SET name = '%s' WHERE id = %d" % (escape_string(name), uid_id))
- print "Assigning name of %s as %s" % (uid, name)
+ session.execute("UPDATE uid SET name = :name WHERE id = :uidid", {'name': name, 'uidid': uid_id})
+ print "Assigning name of %s as %s" % (uid_name, name)
for fingerprint in fingerprints:
- ldap_fin_uid_id[fingerprint] = (uid, uid_id)
+ ldap_fin_uid_id[fingerprint] = (uid_name, uid_id)
if db_fin_uid.has_key(fingerprint):
(existing_uid, fingerprint_id) = db_fin_uid[fingerprint]
if not existing_uid:
- q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
- print "Assigning %s to 0x%s." % (uid, fingerprint)
- elif existing_uid == uid:
+ session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
+ {'uidid': uid_id, 'fprid': fingerprint_id})
+ print "Assigning %s to 0x%s." % (uid_name, fingerprint)
+ elif existing_uid == uid_name:
pass
elif '@' not in existing_uid:
- q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
- print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid, fingerprint)
+ session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
+ {'uidid': uid_id, 'fprid': fingerprint_id})
+ print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid_name, fingerprint)
else:
- utils.warn("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid))
+ utils.warn("%s has %s in LDAP, but database says it should be %s." % \
+ (uid_name, fingerprint, existing_uid))
# Try to update people who sign with non-primary key
- q = projectB.query("SELECT fingerprint, id FROM fingerprint WHERE uid is null")
- for i in q.getresult():
+ q = session.execute("SELECT fingerprint, id FROM fingerprint WHERE uid is null")
+ for i in q.fetchall():
(fingerprint, fingerprint_id) = i
cmd = "gpg --no-default-keyring %s --fingerprint %s" \
% (utils.gpg_keyring_args(), fingerprint)
m = re_gpg_fingerprint.search(output)
if not m:
print output
- utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % (fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] ")))
+ utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % \
+ (fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ","")
if not ldap_fin_uid_id.has_key(primary_key):
utils.warn("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint))
else:
(uid, uid_id) = ldap_fin_uid_id[primary_key]
- q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
+ session.execute("UPDATE fingerprint SET uid = :uid WHERE id = :fprid",
+ {'uid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid, fingerprint)
else:
extra_keyrings = ""
- for keyring in Cnf.ValueList("Import-LDAP-Fingerprints::ExtraKeyrings"):
+ for keyring in cnf.ValueList("Import-LDAP-Fingerprints::ExtraKeyrings"):
extra_keyrings += " --keyring=%s" % (keyring)
cmd = "gpg %s %s --list-key %s" \
% (utils.gpg_keyring_args(), extra_keyrings, fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
- cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (Cnf["Import-LDAP-Fingerprints::KeyServer"], fingerprint)
+ cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (cnf["Import-LDAP-Fingerprints::KeyServer"], fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
print "0x%s: NOT found on keyserver." % (fingerprint)
prompt = "Map to %s - %s (y/N) ? " % (uid, name.replace(" "," "))
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
- uid_id = database.get_or_set_uid_id(uid)
- projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
+ uid_o = get_or_set_uid(uid, session=session)
+ uid_id = uid_o.uid_id
+ session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
+ {'uidid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid, fingerprint)
else:
uid = None
- projectB.query("COMMIT WORK")
+
+ # Commit it all
+ session.commit()
############################################################
################################################################################
-import pg, pwd, sys
+import pwd
+import sys
+import re
import apt_pkg
-from daklib import utils
-################################################################################
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import utils
-Cnf = None
-projectB = None
################################################################################
def usage (exit_code=0):
################################################################################
def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('n', "no-action", "Import-Users-From-Passwd::Options::No-Action"),
('q', "quiet", "Import-Users-From-Passwd::Options::Quiet"),
('v', "verbose", "Import-Users-From-Passwd::Options::Verbose"),
('h', "help", "Import-Users-From-Passwd::Options::Help")]
for i in [ "no-action", "quiet", "verbose", "help" ]:
- if not Cnf.has_key("Import-Users-From-Passwd::Options::%s" % (i)):
- Cnf["Import-Users-From-Passwd::Options::%s" % (i)] = ""
+ if not cnf.has_key("Import-Users-From-Passwd::Options::%s" % (i)):
+ cnf["Import-Users-From-Passwd::Options::%s" % (i)] = ""
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Import-Users-From-Passwd::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Import-Users-From-Passwd::Options")
if Options["Help"]:
usage()
utils.warn("dak import-users-from-passwd takes no non-option arguments.")
usage(1)
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- valid_gid = int(Cnf.get("Import-Users-From-Passwd::ValidGID",""))
+ session = DBConn().session()
+ valid_gid = int(cnf.get("Import-Users-From-Passwd::ValidGID",""))
passwd_unames = {}
for entry in pwd.getpwall():
passwd_unames[uname] = ""
postgres_unames = {}
- q = projectB.query("SELECT usename FROM pg_user")
- ql = q.getresult()
- for i in ql:
+ q = session.execute("SELECT usename FROM pg_user")
+ for i in q.fetchall():
uname = i[0]
postgres_unames[uname] = ""
known_postgres_unames = {}
- for i in Cnf.get("Import-Users-From-Passwd::KnownPostgres","").split(","):
+ for i in cnf.get("Import-Users-From-Passwd::KnownPostgres","").split(","):
uname = i.strip()
known_postgres_unames[uname] = ""
keys = passwd_unames.keys()
keys.sort()
+ safe_name = re.compile('^[A-Za-z0-9]+$')
for uname in keys:
if not postgres_unames.has_key(uname):
if not Options["Quiet"]:
print "Creating %s user in Postgres." % (uname)
if not Options["No-Action"]:
- q = projectB.query('CREATE USER "%s"' % (uname))
+ if safe_name.match(uname):
+ # NB: I never figured out how to use a bind parameter for this query
+ # XXX: Fix this as it looks like a potential SQL injection attack to me
+ # (hence the safe_name match we do)
+ q = session.execute('CREATE USER "%s"' % (uname))
+ else:
+ print "NOT CREATING USER %s. Doesn't match safety regex" % uname
+
+ session.commit()
#######################################################################################
################################################################################
-import psycopg2, sys
+import sys
import apt_pkg
from daklib import utils
-from daklib.dbconn import DBConn
+from daklib.dbconn import *
from daklib.config import Config
################################################################################
################################################################################
-def sql_get (config, key):
- """Return the value of config[key] or None if it doesn't exist."""
-
- try:
- return config[key]
- except KeyError:
- return None
-
-################################################################################
-
class InitDB(object):
def __init__(self, Cnf, projectB):
self.Cnf = Cnf
def do_archive(self):
"""initalize the archive table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM archive")
- archive_add = "INSERT INTO archive (name, origin_server, description) VALUES (%s, %s, %s)"
+ # Remove existing archives
+ s = self.projectB.session()
+ s.query(Archive).delete()
+
for name in self.Cnf.SubTree("Archive").List():
- archive_config = self.Cnf.SubTree("Archive::%s" % (name))
- origin_server = sql_get(archive_config, "OriginServer")
- description = sql_get(archive_config, "Description")
- c.execute(archive_add, [name, origin_server, description])
- self.projectB.commit()
+ a = Archive()
+ a.archive_name = name
+ a.origin_server = self.Cnf.get("Archive::%s::OriginServer" % name, "")
+ a.description = self.Cnf.get("Archive::%s::Description" % name, "")
+ s.add(a)
+
+ s.commit()
def do_architecture(self):
"""Initalize the architecture table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM architecture")
- arch_add = "INSERT INTO architecture (arch_string, description) VALUES (%s, %s)"
+ # Remove existing architectures
+ s = self.projectB.session()
+ s.query(Architecture).delete()
+
for arch in self.Cnf.SubTree("Architectures").List():
- description = self.Cnf["Architectures::%s" % (arch)]
- c.execute(arch_add, [arch, description])
- self.projectB.commit()
+ a = Architecture()
+ a.arch_string = arch
+ a.description = self.Cnf.get("Architecture::%s" % arch, "")
+ s.add(a)
+
+ s.commit()
def do_component(self):
"""Initalize the component table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM component")
-
- comp_add = "INSERT INTO component (name, description, meets_dfsg) " + \
- "VALUES (%s, %s, %s)"
+ # Remove existing components
+ s = self.projectB.session()
+ s.query(Component).delete()
for name in self.Cnf.SubTree("Component").List():
- component_config = self.Cnf.SubTree("Component::%s" % (name))
- description = sql_get(component_config, "Description")
- meets_dfsg = (component_config.get("MeetsDFSG").lower() == "true")
- c.execute(comp_add, [name, description, meets_dfsg])
+ c = Component()
+ c.component_name = name
+ c.description = self.Cnf.get("Component::%s::Description" % name, "")
+ c.meets_dfsg = False
+ if self.Cnf.get("Component::%s::MeetsDFSG" % name, "false").lower() == 'true':
+ c.meets_dfsg = True
+ s.add(c)
- self.projectB.commit()
+ s.commit()
def do_location(self):
"""Initalize the location table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM location")
-
- loc_add = "INSERT INTO location (path, component, archive, type) " + \
- "VALUES (%s, %s, %s, %s)"
+ # Remove existing locations
+ s = self.projectB.session()
+ s.query(Location).delete()
for location in self.Cnf.SubTree("Location").List():
- location_config = self.Cnf.SubTree("Location::%s" % (location))
- archive_id = self.projectB.get_archive_id(location_config["Archive"])
- if archive_id == -1:
- utils.fubar("Archive '%s' for location '%s' not found."
- % (location_config["Archive"], location))
- location_type = location_config.get("type")
- if location_type == "pool":
- for component in self.Cnf.SubTree("Component").List():
- component_id = self.projectB.get_component_id(component)
- c.execute(loc_add, [location, component_id, archive_id, location_type])
- else:
- utils.fubar("E: type '%s' not recognised in location %s."
- % (location_type, location))
-
- self.projectB.commit()
+ archive_name = self.Cnf.get("Location::%s::Archive" % location, "")
+ a = s.query(Archive).filter_by(archive_name=archive_name)
+ if a.count() < 1:
+ utils.fubar("E: Archive '%s' for location '%s' not found" % (archive_name, location))
+ archive_id = a.one().archive_id
+
+ location_type = self.Cnf.get("Location::%s::Type" % location, "")
+ if location_type != 'pool':
+ utils.fubar("E: type %s not recognised for location %s" % (location_type, location))
+
+ for component in self.Cnf.SubTree("Component").List():
+ c = s.query(Component).filter_by(component_name=component)
+ if c.count() < 1:
+ utils.fubar("E: Can't find component %s for location %s" % (component, location))
+ component_id = c.one().component_id
+
+ l = Location()
+ l.path = location
+ l.archive_id = archive_id
+ l.component_id = component_id
+ l.archive_type = location_type
+ s.add(l)
+
+ s.commit()
def do_suite(self):
"""Initalize the suite table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM suite")
-
- suite_add = "INSERT INTO suite (suite_name, version, origin, description) " + \
- "VALUES (%s, %s, %s, %s)"
-
- sa_add = "INSERT INTO suite_architectures (suite, architecture) " + \
- "VALUES (currval('suite_id_seq'), %s)"
+ s = self.projectB.session()
+ s.query(Suite).delete()
for suite in self.Cnf.SubTree("Suite").List():
- suite_config = self.Cnf.SubTree("Suite::%s" %(suite))
- version = sql_get(suite_config, "Version")
- origin = sql_get(suite_config, "Origin")
- description = sql_get(suite_config, "Description")
- c.execute(suite_add, [suite.lower(), version, origin, description])
- for architecture in self.Cnf.SubTree("Architectures").List():
- architecture_id = self.projectB.get_architecture_id (architecture)
- if architecture_id < 0:
- utils.fubar("architecture '%s' not found in architecture"
- " table for suite %s."
- % (architecture, suite))
- c.execute(sa_add, [architecture_id])
-
- self.projectB.commit()
+ su = Suite()
+ su.version = self.Cnf.get("Suite::%s::Version" % suite, "-")
+ su.origin = self.Cnf.get("Suite::%s::Origin" % suite, "")
+ su.description = self.Cnf.get("Suite::%s::Description" % suite, "")
+ s.add(su)
+
+ for architecture in self.Cnf.ValueList("Suite::%s::Architectures" % (suite)):
+ sa = SuiteArchitecture()
+ sa.suite_id = su.suite_id
+ a = s.query(Architecture).filter_by(arch_string=architecture)
+ if a.count() < 1:
+ utils.fubar("E: Architecture %s not found for suite %s" % (architecture, suite))
+ sa.arch_id = a.one().arch_id
+ s.add(sa)
+
+ s.commit()
def do_override_type(self):
"""Initalize the override_type table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM override_type")
-
- over_add = "INSERT INTO override_type (type) VALUES (%s)"
+ s = self.projectB.session()
+ s.query(OverrideType).delete()
for override_type in self.Cnf.ValueList("OverrideType"):
- c.execute(over_add, [override_type])
+ ot = OverrideType()
+ ot.overridetype = override_type
+ s.add(ot)
- self.projectB.commit()
+ s.commit()
def do_priority(self):
"""Initialize the priority table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM priority")
-
- prio_add = "INSERT INTO priority (priority, level) VALUES (%s, %s)"
+ s = self.projectB.session()
+ s.query(Priority).delete()
for priority in self.Cnf.SubTree("Priority").List():
- c.execute(prio_add, [priority, self.Cnf["Priority::%s" % (priority)]])
+ p = Priority()
+ p.priority = priority
+ p.level = self.Cnf.get("Priority::%s", 0)
+ s.add(p)
- self.projectB.commit()
+ s.commit()
def do_section(self):
"""Initalize the section table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM section")
-
- sect_add = "INSERT INTO section (section) VALUES (%s)"
+ s = self.projectB.session()
+ s.query(Section).delete()
for component in self.Cnf.SubTree("Component").List():
if self.Cnf["Control-Overrides::ComponentPosition"] == "prefix":
suffix = '/' + component
else:
suffix = ""
+
for section in self.Cnf.ValueList("Section"):
- c.execute(sect_add, [prefix + section + suffix])
+ sec = Section()
+ sec.section = prefix + section + suffix
+ s.add(sec)
- self.projectB.commit()
+ s.commit()
def do_all(self):
self.do_archive()
################################################################################
import os
-import pg
import sys
import apt_pkg
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import utils
################################################################################
################################################################################
def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a', "architecture", "Ls::Options::Architecture", "HasArg"),
('b', "binarytype", "Ls::Options::BinaryType", "HasArg"),
for i in [ "architecture", "binarytype", "component", "format",
"greaterorequal", "greaterthan", "regex", "suite",
"source-and-binary", "help" ]:
- if not Cnf.has_key("Ls::Options::%s" % (i)):
- Cnf["Ls::Options::%s" % (i)] = ""
+ if not cnf.has_key("Ls::Options::%s" % (i)):
+ cnf["Ls::Options::%s" % (i)] = ""
- packages = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Ls::Options")
+ packages = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Ls::Options")
if Options["Help"]:
usage()
if not packages:
utils.fubar("need at least one package name as an argument.")
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# If cron.daily is running; warn the user that our output might seem strange
- if os.path.exists(os.path.join(Cnf["Dir::Root"], "Archive_Maintenance_In_Progress")):
+ if os.path.exists(os.path.join(cnf["Dir::Root"], "Archive_Maintenance_In_Progress")):
utils.warn("Archive maintenance is in progress; database inconsistencies are possible.")
# Handle buildd maintenance helper options
if Options["Source-And-Binary"]:
new_packages = []
for package in packages:
- q = projectB.query("SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s '%s' %s" % (comparison_operator, package, con_suites))
- new_packages.extend([ i[0] for i in q.getresult() ])
+ q = session.execute("SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s :package %s" % (comparison_operator, con_suites),
+ {'package': package})
+ new_packages.extend([ i[0] for i in q.fetchall() ])
if package not in new_packages:
new_packages.append(package)
packages = new_packages
results = 0
for package in packages:
- q = projectB.query("""
+ q = session.execute("""
SELECT b.package, b.version, a.arch_string, su.suite_name, c.name, m.name
FROM binaries b, architecture a, suite su, bin_associations ba,
files f, location l, component c, maintainer m
- WHERE b.package %s '%s' AND a.id = b.architecture AND su.id = ba.suite
+ WHERE b.package %s :package AND a.id = b.architecture AND su.id = ba.suite
AND b.id = ba.bin AND b.file = f.id AND f.location = l.id
AND l.component = c.id AND b.maintainer = m.id %s %s %s
-""" % (comparison_operator, package, con_suites, con_architectures, con_bintype))
- ql = q.getresult()
+""" % (comparison_operator, con_suites, con_architectures, con_bintype), {'package': package})
+ ql = q.fetchall()
if check_source:
- q = projectB.query("""
+ q = session.execute("""
SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name
FROM source s, suite su, src_associations sa, files f, location l,
component c, maintainer m
- WHERE s.source %s '%s' AND su.id = sa.suite AND s.id = sa.source
+ WHERE s.source %s :package AND su.id = sa.suite AND s.id = sa.source
AND s.file = f.id AND f.location = l.id AND l.component = c.id
AND s.maintainer = m.id %s
-""" % (comparison_operator, package, con_suites))
- ql.extend(q.getresult())
+""" % (comparison_operator, con_suites), {'package': package})
+ ql.extend(q.fetchall())
d = {}
highver = {}
for i in ql:
################################################################################
-import pg
import sys
import apt_pkg
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
+from daklib import textutils
from daklib.regexes import re_comments
################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
maintainer_from_source_cache = {} #: caches the maintainer name <email> per source_id
packages = {} #: packages data to write out
-fixed_maintainer_cache = {} #: caches fixed ( L{daklib.utils.fix_maintainer} ) maintainer data
+fixed_maintainer_cache = {} #: caches fixed ( L{daklib.textutils.fix_maintainer} ) maintainer data
################################################################################
Fixup maintainer entry, cache the result.
@type maintainer: string
- @param maintainer: A maintainer entry as passed to L{daklib.utils.fix_maintainer}
+ @param maintainer: A maintainer entry as passed to L{daklib.textutils.fix_maintainer}
@rtype: tuple
@returns: fixed maintainer tuple
global fixed_maintainer_cache
if not fixed_maintainer_cache.has_key(maintainer):
- fixed_maintainer_cache[maintainer] = utils.fix_maintainer(maintainer)[0]
+ fixed_maintainer_cache[maintainer] = textutils.fix_maintainer(maintainer)[0]
return fixed_maintainer_cache[maintainer]
-def get_maintainer (maintainer):
+def get_maintainer(maintainer, session):
"""
Retrieves maintainer name from database, passes it through fix_maintainer and
passes on whatever that returns.
@type maintainer: int
@param maintainer: maintainer_id
"""
- return fix_maintainer(database.get_maintainer(maintainer))
+ q = session.execute("SELECT name FROM maintainer WHERE id = :id", {'id': maintainer}).fetchall()
+ return fix_maintainer(q[0][0])
-def get_maintainer_from_source (source_id):
+def get_maintainer_from_source(source_id, session):
"""
Returns maintainer name for given source_id.
global maintainer_from_source_cache
if not maintainer_from_source_cache.has_key(source_id):
- q = projectB.query("SELECT m.name FROM maintainer m, source s WHERE s.id = %s and s.maintainer = m.id" % (source_id))
- maintainer = q.getresult()[0][0]
+ q = session.execute("""SELECT m.name FROM maintainer m, source s
+ WHERE s.id = :sourceid AND s.maintainer = m.id""",
+ {'sourceid': source_id})
+ maintainer = q.fetchall()[0][0]
maintainer_from_source_cache[source_id] = fix_maintainer(maintainer)
return maintainer_from_source_cache[source_id]
################################################################################
def main():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Make-Maintainers::Options::Help")]
- if not Cnf.has_key("Make-Maintainers::Options::Help"):
- Cnf["Make-Maintainers::Options::Help"] = ""
+ if not cnf.has_key("Make-Maintainers::Options::Help"):
+ cnf["Make-Maintainers::Options::Help"] = ""
- extra_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Make-Maintainers::Options")
+ extra_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Make-Maintainers::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
- for suite in Cnf.SubTree("Suite").List():
+ for suite in cnf.SubTree("Suite").List():
suite = suite.lower()
- suite_priority = int(Cnf["Suite::%s::Priority" % (suite)])
+ suite_priority = int(cnf["Suite::%s::Priority" % (suite)])
# Source packages
- q = projectB.query("SELECT s.source, s.version, m.name FROM src_associations sa, source s, suite su, maintainer m WHERE su.suite_name = '%s' AND sa.suite = su.id AND sa.source = s.id AND m.id = s.maintainer" % (suite))
- sources = q.getresult()
- for source in sources:
+ q = session.execute("""SELECT s.source, s.version, m.name
+ FROM src_associations sa, source s, suite su, maintainer m
+ WHERE su.suite_name = :suite_name
+ AND sa.suite = su.id AND sa.source = s.id
+ AND m.id = s.maintainer""",
+ {'suite_name': suite})
+ for source in q.fetchall():
package = source[0]
version = source[1]
maintainer = fix_maintainer(source[2])
packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version }
# Binary packages
- q = projectB.query("SELECT b.package, b.source, b.maintainer, b.version FROM bin_associations ba, binaries b, suite s WHERE s.suite_name = '%s' AND ba.suite = s.id AND ba.bin = b.id" % (suite))
- binaries = q.getresult()
- for binary in binaries:
+ q = session.execute("""SELECT b.package, b.source, b.maintainer, b.version
+ FROM bin_associations ba, binaries b, suite s
+ WHERE s.suite_name = :suite_name
+ AND ba.suite = s.id AND ba.bin = b.id""",
+ {'suite_name': suite})
+ for binary in q.fetchall():
package = binary[0]
source_id = binary[1]
version = binary[3]
# Use the source maintainer first; falling back on the binary maintainer as a last resort only
if source_id:
- maintainer = get_maintainer_from_source(source_id)
+ maintainer = get_maintainer_from_source(source_id, session)
else:
- maintainer = get_maintainer(binary[2])
+ maintainer = get_maintainer(binary[2], session)
if packages.has_key(package):
if packages[package]["priority"] <= suite_priority:
if apt_pkg.VersionCompare(packages[package]["version"], version) < 0:
################################################################################
-import pg
+import os
import sys
import apt_pkg
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
-override = {} #: override data to write out
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib import utils
################################################################################
################################################################################
-def do_list(output_file, suite, component, otype):
+def do_list(output_file, suite, component, otype, session):
"""
Fetch override data for suite from the database and dump it.
@type output_file: fileobject
@param output_file: where to write the overrides to
- @type suite: string
- @param suite: The name of the suite
+ @type suite: Suite object
+ @param suite: A suite object describing the Suite
- @type component: string
+ @type component: Component object
@param component: The name of the component
- @type otype: string
- @param otype: type of override. deb/udeb/dsc
-
- """
- global override
-
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
- utils.fubar("Suite '%s' not recognised." % (suite))
+ @type otype: OverrideType object
+ @param otype: object of type of override. deb/udeb/dsc
- component_id = database.get_component_id(component)
- if component_id == -1:
- utils.fubar("Component '%s' not recognised." % (component))
+ @type session: SQLA Session
+ @param session: the database session in use
- otype_id = database.get_override_type_id(otype)
- if otype_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
-
- override.setdefault(suite, {})
- override[suite].setdefault(component, {})
- override[suite][component].setdefault(otype, {})
+ """
+ # Here's a nice example of why the object API isn't always the
+ # right answer. On my laptop, the object version of the code
+ # takes 1:45, the 'dumb' tuple-based one takes 0:16 - mhy
+
+ if otype.overridetype == "dsc":
+ #q = session.query(Override).filter_by(suite_id = suite.suite_id)
+ #q = q.filter_by(component_id = component.component_id)
+ #q = q.filter_by(overridetype_id = otype.overridetype_id)
+ #q = q.join(Section).order_by(Section.section, Override.package)
+ #for o in q.all():
+ # dat = (o.package, o.section.section, o.maintainer)
+ # output_file.write(utils.result_join(dat) + '\n')
+ q = session.execute("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite.suite_id, component.component_id, otype.overridetype_id))
+ for i in q.fetchall():
+ output_file.write(utils.result_join(i) + '\n')
- if otype == "dsc":
- q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, otype_id))
- for i in q.getresult():
- override[suite][component][otype][i[0]] = i
- output_file.write(utils.result_join(i)+'\n')
else:
- q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, otype_id))
- for i in q.getresult():
- i = i[:-1]; # Strip the priority level
- override[suite][component][otype][i[0]] = i
- output_file.write(utils.result_join(i)+'\n')
+ #q = session.query(Override).filter_by(suite_id = suite.suite_id)
+ #q = q.filter_by(component_id = component.component_id)
+ #q = q.filter_by(overridetype_id = otype.overridetype_id)
+ #q = q.join(Priority).join(Section).order_by(Section.section, Priority.level, Override.package)
+ #for o in q.all():
+ # dat = (o.package, o.priority.priority, o.section.section, o.maintainer)
+ # output_file.write(utils.result_join(dat) + '\n')
+ q = session.execute("SELECT o.package, p.priority, s.section, o.maintainer FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite.suite_id, component.component_id, otype.overridetype_id))
+ for i in q.fetchall():
+ output_file.write(utils.result_join(i) + '\n')
################################################################################
def main ():
- global Cnf, projectB, override
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Make-Overrides::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Make-Overrides::Options::%s" % (i)):
- Cnf["Make-Overrides::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Make-Overrides::Options")
+ if not cnf.has_key("Make-Overrides::Options::%s" % (i)):
+ cnf["Make-Overrides::Options::%s" % (i)] = ""
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Make-Overrides::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ d = DBConn()
+ session = d.session()
- for suite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
- if database.get_suite_untouchable(suite):
+ for suite_name in cnf.SubTree("Check-Overrides::OverrideSuites").List():
+ suite = get_suite(suite_name.lower(), session)
+ if not suite:
+ utils.fubar('Suite %s not found' % suite_name)
+ if suite.untouchable:
continue
- suite = suite.lower()
- sys.stderr.write("Processing %s...\n" % (suite))
- override_suite = Cnf["Suite::%s::OverrideCodeName" % (suite)]
- for component in Cnf.SubTree("Component").List():
- for otype in Cnf.ValueList("OverrideType"):
- if otype == "deb":
+ sys.stderr.write("Processing %s...\n" % (suite.suite_name))
+ override_suite = cnf["Suite::%s::OverrideCodeName" % (suite_name)]
+
+ for component_name in cnf.SubTree("Component").List():
+ component = get_component(component_name, session)
+ if not component:
+ utils.fubar('Component %s not found' % component_name)
+
+ for otype_name in cnf.ValueList("OverrideType"):
+ otype = get_override_type(otype_name, session)
+ if not otype:
+ utils.fubar('OverrideType %s not found' % otype_name)
+
+ if otype_name == "deb":
suffix = ""
- elif otype == "udeb":
+ elif otype_name == "udeb":
if component == "contrib":
continue # Ick2
suffix = ".debian-installer"
- elif otype == "dsc":
+ elif otype_name == "dsc":
suffix = ".src"
- filename = "%s/override.%s.%s%s" % (Cnf["Dir::Override"], override_suite, component, suffix)
+
+ filename = os.path.join(cnf["Dir::Override"], "override.%s.%s%s" % (override_suite, component.component_name, suffix))
output_file = utils.open_file(filename, 'w')
- do_list(output_file, suite, component, otype)
+ do_list(output_file, suite, component, otype, session)
output_file.close()
################################################################################
################################################################################
import os
-import pg
import sys
-from daklib import database
-from daklib import utils
-################################################################################
-
-projectB = None #: database connection, pgobject
+from daklib.dbconn import *
################################################################################
ORDER BY source, version, package, bin_version
"""
- for row in projectB.query(query_sources).getresult():
+ session = DBConn().session()
+
+ for row in session.execute(query_sources).fetchall():
(source, version, path) = row
print "Path: %s"%path
print "Source: %s"%source
print "Source-Version: %s"%version
print
- for row in projectB.query(query_binaries).getresult():
+ for row in session.execute(query_binaries).fetchall():
(source, version, arch, path, bin, binv) = row
print "Path: %s"%path
print "Source: %s"%source
################################################################################
def main():
- global projectB
-
- Cnf = utils.get_conf()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ DBConn()
build_mapping()
#########################################################################################
import copy
import os
-import pg
import sys
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib import daklog
from daklib import utils
################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
Logger = None #: Logger object
Options = None #: Parsed CommandLine arguments
#####################################################
def delete_packages(delete_versions, pkg, dominant_arch, suite,
- dominant_version, delete_table, delete_col, packages):
- suite_id = database.get_suite_id(suite)
+ dominant_version, delete_table, delete_col, packages, session):
+ suite_o = get_suite(suite.lower(), session)
+ suite_id = suite_o.suite_id
for version in delete_versions:
delete_unique_id = version[1]
if not packages.has_key(delete_unique_id):
delete_version = version[0]
delete_id = packages[delete_unique_id]["sourceid"]
delete_arch = packages[delete_unique_id]["arch"]
- if not database.get_suite_untouchable(suite) or Options["Force"]:
+ if Options["Force"] or not suite_o.untouchable:
if Options["No-Delete"]:
print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch)
else:
Logger.log(["dominated", pkg, delete_arch, delete_version, dominant_version, dominant_arch])
- projectB.query("DELETE FROM %s WHERE suite = %s AND %s = %s" % (delete_table, suite_id, delete_col, delete_id))
+ # TODO: Fix properly
+ session.execute("DELETE FROM %s WHERE suite = :suiteid AND %s = :delid" % (delete_table, delete_col), {'suiteid': suite_id, 'delid': delete_id})
del packages[delete_unique_id]
else:
if Options["No-Delete"]:
#####################################################
-def resolve_arch_all_vs_any(versions, packages):
+def resolve_arch_all_vs_any(versions, packages, session):
""" Per-suite&pkg: resolve arch-all, vs. arch-any, assumes only one arch-all """
arch_all_version = None
arch_any_versions = copy.copy(versions)
if apt_pkg.VersionCompare(highest_arch_any_version, arch_all_version) < 1:
# arch: all dominates
delete_packages(arch_any_versions, pkg, "all", suite,
- arch_all_version, delete_table, delete_col, packages)
+ arch_all_version, delete_table, delete_col, packages, session)
else:
# arch: any dominates
delete_packages(arch_all_versions, pkg, "any", suite,
highest_arch_any_version, delete_table, delete_col,
- packages)
+ packages, session)
#####################################################
################################################################################
-def cleanup(packages):
+def cleanup(packages, session):
# Build up the index used by the clean up functions
d = {}
for unique_id in packages.keys():
for arch in arches.keys():
if arch != "source":
versions.extend(d[suite][pkg][arch])
- resolve_arch_all_vs_any(versions, packages)
+ resolve_arch_all_vs_any(versions, packages, session)
################################################################################
def write_filelist(suite, component, arch, type, list, packages, dislocated_files):
+ cnf = Config()
+
# Work out the filename
if arch != "source":
if type == "udeb":
arch = "debian-installer_binary-%s" % (arch)
elif type == "deb":
arch = "binary-%s" % (arch)
- filename = os.path.join(Cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch))
+ filename = os.path.join(cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch))
output = utils.open_file(filename, "w")
# Generate the final list of files
files = {}
################################################################################
-def write_filelists(packages, dislocated_files):
+def write_filelists(packages, dislocated_files, session):
# Build up the index to iterate over
+ cnf = Config()
+
d = {}
for unique_id in packages.keys():
suite = packages[unique_id]["suite"]
d[suite][component][arch][packagetype].append(unique_id)
# Flesh out the index
if not Options["Suite"]:
- suites = Cnf.SubTree("Suite").List()
+ suites = cnf.SubTree("Suite").List()
else:
suites = utils.split_args(Options["Suite"])
for suite in [ i.lower() for i in suites ]:
d.setdefault(suite, {})
if not Options["Component"]:
- components = Cnf.ValueList("Suite::%s::Components" % (suite))
+ components = cnf.ValueList("Suite::%s::Components" % (suite))
else:
components = utils.split_args(Options["Component"])
- udeb_components = Cnf.ValueList("Suite::%s::UdebComponents" % (suite))
+ udeb_components = cnf.ValueList("Suite::%s::UdebComponents" % (suite))
for component in components:
d[suite].setdefault(component, {})
if component in udeb_components:
else:
binary_types = [ "deb" ]
if not Options["Architecture"]:
- architectures = database.get_suite_architectures(suite)
+ architectures = [ a.arch_string for a in get_suite_architectures(suite, session=session) ]
else:
architectures = utils.split_args(Options["Architecture"])
for arch in [ i.lower() for i in architectures ]:
d[suite][component][arch].setdefault(packagetype, [])
# Then walk it
for suite in d.keys():
- if Cnf.has_key("Suite::%s::Components" % (suite)):
+ if cnf.has_key("Suite::%s::Components" % (suite)):
for component in d[suite].keys():
for arch in d[suite][component].keys():
if arch == "all":
filelist = d[suite][component][arch][packagetype]
# If it's a binary, we need to add in the arch: all debs too
if arch != "source":
- archall_suite = Cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
+ archall_suite = cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
if archall_suite:
filelist.extend(d[archall_suite][component]["all"][packagetype])
elif d[suite][component].has_key("all") and \
if Options["Suite"]:
suites = utils.split_args(Options["Suite"])
for suite in suites:
- archall_suite = Cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
+ archall_suite = cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
if archall_suite and archall_suite not in suites:
utils.warn("Adding %s as %s maps Arch: all from it." % (archall_suite, suite))
suites.append(archall_suite)
(con_suites, con_architectures, con_components, check_source) = \
utils.parse_args(Options)
+ session = DBConn().session()
+
dislocated_files = {}
+ # TODO: Fix this properly
+ query_args = {'con_suites': con_suites, 'con_architectures': con_architectures, 'con_components': con_components}
+
query = """
SELECT b.id, b.package, a.arch_string, b.version, l.path, f.filename, c.name,
f.id, su.suite_name, b.type
component c, suite su
WHERE b.id = ba.bin AND b.file = f.id AND b.architecture = a.id
AND f.location = l.id AND l.component = c.id AND ba.suite = su.id
- %s %s %s""" % (con_suites, con_architectures, con_components)
+ %(con_suites)s %(con_architectures)s %(con_components)s""" % query_args
if check_source:
query += """
UNION
su.suite_name, 'dsc'
FROM source s, src_associations sa, files f, location l, component c, suite su
WHERE s.id = sa.source AND s.file = f.id AND f.location = l.id
- AND l.component = c.id AND sa.suite = su.id %s %s""" % (con_suites, con_components)
- q = projectB.query(query)
- ql = q.getresult()
+ AND l.component = c.id AND sa.suite = su.id %(con_suites)s %(con_components)s""" % query_args
+
# Build up the main index of packages
packages = {}
unique_id = 0
- for i in ql:
+
+ q = session.execute(query)
+ for i in q.fetchall():
(sourceid, pkg, arch, version, path, filename, component, file_id, suite, filetype) = i
+
# 'id' comes from either 'binaries' or 'source', so it's not unique
unique_id += 1
packages[unique_id] = Dict(sourceid=sourceid, pkg=pkg, arch=arch, version=version,
path=path, filename=filename,
component=component, file_id=file_id,
suite=suite, filetype = filetype)
- cleanup(packages)
- write_filelists(packages, dislocated_files)
+ cleanup(packages, session)
+ session.commit()
+ write_filelists(packages, dislocated_files, session)
################################################################################
def main():
- global Cnf, projectB, Options, Logger
+ global Options, Logger
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a', "architecture", "Make-Suite-File-List::Options::Architecture", "HasArg"),
('c', "component", "Make-Suite-File-List::Options::Component", "HasArg"),
('h', "help", "Make-Suite-File-List::Options::Help"),
('f', "force", "Make-Suite-File-List::Options::Force"),
('s', "suite", "Make-Suite-File-List::Options::Suite", "HasArg")]
for i in ["architecture", "component", "help", "no-delete", "suite", "force" ]:
- if not Cnf.has_key("Make-Suite-File-List::Options::%s" % (i)):
- Cnf["Make-Suite-File-List::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Make-Suite-File-List::Options")
+ if not cnf.has_key("Make-Suite-File-List::Options::%s" % (i)):
+ cnf["Make-Suite-File-List::Options::%s" % (i)] = ""
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Make-Suite-File-List::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
- Logger = logging.Logger(Cnf, "make-suite-file-list")
+ DBConn()
+
+ Logger = daklog.Logger(cnf.Cnf, "make-suite-file-list")
do_da_do_da()
Logger.close()
import apt_pkg, os, sys, pwd, time, commands
from daklib import queue
-from daklib import logging
+from daklib import daklog
from daklib import utils
from daklib import database
from daklib.regexes import re_taint_free
if Options["No-Action"]:
Options["Sudo"] = ""
if not Options["Sudo"] and not Options["No-Action"]:
- Logger = Upload.Logger = logging.Logger(Cnf, "new-security-install")
+ Logger = Upload.Logger = daklog.Logger(Cnf, "new-security-install")
return arguments
## That Alisha Rules The World
################################################################################
-import pg, sys
+import os
+import sys
import apt_pkg
-from daklib import logging
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None
-projectB = None
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
+from daklib import utils
################################################################################
sys.exit(exit_code)
def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Override::Options::Help"),
('d',"done","Override::Options::Done", "HasArg"),
('s',"suite","Override::Options::Suite", "HasArg"),
]
for i in ["help", "no-action"]:
- if not Cnf.has_key("Override::Options::%s" % (i)):
- Cnf["Override::Options::%s" % (i)] = ""
- if not Cnf.has_key("Override::Options::Suite"):
- Cnf["Override::Options::Suite"] = "unstable"
+ if not cnf.has_key("Override::Options::%s" % (i)):
+ cnf["Override::Options::%s" % (i)] = ""
+ if not cnf.has_key("Override::Options::Suite"):
+ cnf["Override::Options::Suite"] = "unstable"
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Override::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Override::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
if not arguments:
utils.fubar("package name is a required argument.")
if arguments and len(arguments) == 1:
# Determine if the argument is a priority or a section...
arg = arguments.pop()
- q = projectB.query("""
- SELECT ( SELECT COUNT(*) FROM section WHERE section=%s ) AS secs,
- ( SELECT COUNT(*) FROM priority WHERE priority=%s ) AS prios
- """ % ( pg._quote(arg,"str"), pg._quote(arg,"str")))
- r = q.getresult()
+ q = session.execute("""
+ SELECT ( SELECT COUNT(*) FROM section WHERE section = :arg ) AS secs,
+ ( SELECT COUNT(*) FROM priority WHERE priority = :arg ) AS prios
+ """, {'arg': arg})
+ r = q.fetchall()
if r[0][0] == 1:
- arguments = (arg,".")
+ arguments = (arg, ".")
elif r[0][1] == 1:
- arguments = (".",arg)
+ arguments = (".", arg)
else:
utils.fubar("%s is not a valid section or priority" % (arg))
eqdsc = '!='
if packagetype == 'source':
eqdsc = '='
- q = projectB.query("""
+ q = session.execute("""
SELECT priority.priority AS prio, section.section AS sect, override_type.type AS type
FROM override, priority, section, suite, override_type
WHERE override.priority = priority.id
AND override.type = override_type.id
AND override_type.type %s 'dsc'
AND override.section = section.id
- AND override.package = %s
+ AND override.package = :package
AND override.suite = suite.id
- AND suite.suite_name = %s
- """ % (eqdsc, pg._quote(package,"str"), pg._quote(suite,"str")))
+ AND suite.suite_name = :suite
+ """ % (eqdsc), {'package': package, 'suite': suite})
- if q.ntuples() == 0:
+ if q.rowcount == 0:
continue
- if q.ntuples() > 1:
+ if q.rowcount > 1:
utils.fubar("%s is ambiguous. Matches %d packages" % (package,q.ntuples()))
- r = q.getresult()
+ r = q.fetchone()
if packagetype == 'binary':
- oldsection = r[0][1]
- oldpriority = r[0][0]
+ oldsection = r[1]
+ oldpriority = r[0]
else:
- oldsourcesection = r[0][1]
+ oldsourcesection = r[1]
oldpriority = 'source'
if not oldpriority and not oldsourcesection:
utils.fubar("Unable to find package %s" % (package))
+
if oldsection and oldsourcesection and oldsection != oldsourcesection:
# When setting overrides, both source & binary will become the same section
utils.warn("Source is in section '%s' instead of '%s'" % (oldsourcesection, oldsection))
+
if not oldsection:
oldsection = oldsourcesection
if not arguments:
print "%s is in section '%s' at priority '%s'" % (
- package,oldsection,oldpriority)
+ package, oldsection, oldpriority)
sys.exit(0)
# At this point, we have a new section and priority... check they're valid...
if newpriority == ".":
newpriority = oldpriority
- q = projectB.query("SELECT id FROM section WHERE section=%s" % (
- pg._quote(newsection,"str")))
-
- if q.ntuples() == 0:
+ s = get_section(newsection, session)
+ if s is None:
utils.fubar("Supplied section %s is invalid" % (newsection))
- newsecid = q.getresult()[0][0]
+ newsecid = s.section_id
- q = projectB.query("SELECT id FROM priority WHERE priority=%s" % (
- pg._quote(newpriority,"str")))
-
- if q.ntuples() == 0:
+ p = get_priority(newpriority, session)
+ if p is None:
utils.fubar("Supplied priority %s is invalid" % (newpriority))
- newprioid = q.getresult()[0][0]
+ newprioid = p.priority_id
if newpriority == oldpriority and newsection == oldsection:
print "I: Doing nothing"
if newpriority != oldpriority:
print "I: Will change priority from %s to %s" % (oldpriority,newpriority)
+
if newsection != oldsection:
print "I: Will change section from %s to %s" % (oldsection,newsection)
game_over()
- Logger = logging.Logger(Cnf, "override")
+ Logger = daklog.Logger(cnf.Cnf, "override")
- projectB.query("BEGIN WORK")
+ dsc_otype_id = get_override_type('dsc').overridetype_id
+
+ # We're already in a transaction
# We're in "do it" mode, we have something to do... do it
if newpriority != oldpriority:
- q = projectB.query("""
+ session.execute("""
UPDATE override
- SET priority=%d
- WHERE package=%s
- AND override.type != %d
- AND suite = (SELECT id FROM suite WHERE suite_name=%s)""" % (
- newprioid,
- pg._quote(package,"str"), database.get_override_type_id("dsc"),
- pg._quote(suite,"str") ))
- Logger.log(["changed priority",package,oldpriority,newpriority])
+ SET priority = :newprioid
+ WHERE package = :package
+ AND override.type != :otypedsc
+ AND suite = (SELECT id FROM suite WHERE suite_name = :suite)""",
+ {'newprioid': newprioid, 'package': package,
+ 'otypedsc': dsc_otype_id, 'suite': suite})
+
+ Logger.log(["changed priority", package, oldpriority, newpriority])
if newsection != oldsection:
- q = projectB.query("""
+ q = session.execute("""
UPDATE override
- SET section=%d
- WHERE package=%s
- AND suite = (SELECT id FROM suite WHERE suite_name=%s)""" % (
- newsecid,
- pg._quote(package,"str"),
- pg._quote(suite,"str") ))
- Logger.log(["changed section",package,oldsection,newsection])
- projectB.query("COMMIT WORK")
+ SET section = :newsecid
+ WHERE package = :package
+ AND suite = (SELECT id FROM suite WHERE suite_name = :suite)""",
+ {'newsecid': newsecid, 'package': package,
+ 'suite': suite})
+
+ Logger.log(["changed section", package, oldsection, newsection])
+
+ session.commit()
if Options.has_key("Done"):
Subst = {}
- Subst["__OVERRIDE_ADDRESS__"] = Cnf["Override::MyEmailAddress"]
- Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
+ Subst["__OVERRIDE_ADDRESS__"] = cnf["Override::MyEmailAddress"]
+ Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
bcc = []
- if Cnf.Find("Dinstall::Bcc") != "":
- bcc.append(Cnf["Dinstall::Bcc"])
- if Cnf.Find("Override::Bcc") != "":
- bcc.append(Cnf["Override::Bcc"])
+ if cnf.Find("Dinstall::Bcc") != "":
+ bcc.append(cnf["Dinstall::Bcc"])
+ if cnf.Find("Override::Bcc") != "":
+ bcc.append(cnf["Override::Bcc"])
if bcc:
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
summary += "Changed section from %s to %s\n" % (oldsection,newsection)
Subst["__SUMMARY__"] = summary
+ template = os.path.join(cnf["Dir::Templates"], "override.bug-close")
for bug in utils.split_args(Options["Done"]):
Subst["__BUG_NUMBER__"] = bug
- mail_message = utils.TemplateSubst(
- Subst,Cnf["Dir::Templates"]+"/override.bug-close")
+ mail_message = utils.TemplateSubst(Subst, template)
utils.send_mail(mail_message)
- Logger.log(["closed bug",bug])
+ Logger.log(["closed bug", bug])
Logger.close()
- print "Done"
-
#################################################################################
if __name__ == '__main__':
import fcntl
import os
import sys
-import time
+from datetime import datetime
import re
import apt_pkg, commands
-from daklib import database
-from daklib import logging
-from daklib import queue
+
+from daklib import daklog
+from daklib.queue import *
from daklib import utils
+from daklib.dbconn import *
+from daklib.binary import copy_temporary_contents
from daklib.dak_exceptions import *
from daklib.regexes import re_default_answer, re_issource, re_fdnic
+from daklib.urgencylog import UrgencyLog
+from daklib.summarystats import SummaryStats
+from daklib.config import Config
###############################################################################
-Cnf = None
Options = None
Logger = None
-Urgency_Logger = None
-projectB = None
-Upload = None
-pkg = None
-
-reject_message = ""
-changes = None
-dsc = None
-dsc_files = None
-files = None
-Subst = None
-
-install_count = 0
-install_bytes = 0.0
-
-installing_to_stable = 0
-
-###############################################################################
-
-# FIXME: this should go away to some Debian specific file
-# FIXME: should die if file already exists
-
-class Urgency_Log:
- "Urgency Logger object"
- def __init__ (self, Cnf):
- "Initialize a new Urgency Logger object"
- self.Cnf = Cnf
- self.timestamp = time.strftime("%Y%m%d%H%M%S")
- # Create the log directory if it doesn't exist
- self.log_dir = Cnf["Dir::UrgencyLog"]
- if not os.path.exists(self.log_dir) or not os.access(self.log_dir, os.W_OK):
- utils.warn("UrgencyLog directory %s does not exist or is not writeable, using /srv/ftp.debian.org/tmp/ instead" % (self.log_dir))
- self.log_dir = '/srv/ftp.debian.org/tmp/'
- # Open the logfile
- self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp)
- self.log_file = utils.open_file(self.log_filename, 'w')
- self.writes = 0
-
- def log (self, source, version, urgency):
- "Log an event"
- self.log_file.write(" ".join([source, version, urgency])+'\n')
- self.log_file.flush()
- self.writes += 1
-
- def close (self):
- "Close a Logger object"
- self.log_file.flush()
- self.log_file.close()
- if self.writes:
- new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp)
- utils.move(self.log_filename, new_filename)
- else:
- os.unlink(self.log_filename)
-
-
-###############################################################################
-
-
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-# Recheck anything that relies on the database; since that's not
-# frozen between accept and our run time.
-
-def check():
- propogate={}
- nopropogate={}
- for checkfile in files.keys():
- # The .orig.tar.gz can disappear out from under us is it's a
- # duplicate of one in the archive.
- if not files.has_key(checkfile):
- continue
- # Check that the source still exists
- if files[checkfile]["type"] == "deb":
- source_version = files[checkfile]["source version"]
- source_package = files[checkfile]["source package"]
- if not changes["architecture"].has_key("source") \
- and not Upload.source_exists(source_package, source_version, changes["distribution"].keys()):
- reject("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
-
- # Version and file overwrite checks
- if not installing_to_stable:
- if files[checkfile]["type"] == "deb":
- reject(Upload.check_binary_against_db(checkfile), "")
- elif files[checkfile]["type"] == "dsc":
- reject(Upload.check_source_against_db(checkfile), "")
- (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(checkfile)
- reject(reject_msg, "")
-
- # propogate in the case it is in the override tables:
- if changes.has_key("propdistribution"):
- for suite in changes["propdistribution"].keys():
- if Upload.in_override_p(files[checkfile]["package"], files[checkfile]["component"], suite, files[checkfile].get("dbtype",""), checkfile):
- propogate[suite] = 1
- else:
- nopropogate[suite] = 1
-
- for suite in propogate.keys():
- if suite in nopropogate:
- continue
- changes["distribution"][suite] = 1
-
- for checkfile in files.keys():
- # Check the package is still in the override tables
- for suite in changes["distribution"].keys():
- if not Upload.in_override_p(files[checkfile]["package"], files[checkfile]["component"], suite, files[checkfile].get("dbtype",""), checkfile):
- reject("%s is NEW for %s." % (checkfile, suite))
###############################################################################
def init():
- global Cnf, Options, Upload, projectB, changes, dsc, dsc_files, files, pkg, Subst
+ global Options
- Cnf = utils.get_conf()
+ # Initialize config and connection to db
+ cnf = Config()
+ DBConn()
Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
('h',"help","Dinstall::Options::Help"),
for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
"version", "directory"]:
- if not Cnf.has_key("Dinstall::Options::%s" % (i)):
- Cnf["Dinstall::Options::%s" % (i)] = ""
+ if not cnf.has_key("Dinstall::Options::%s" % (i)):
+ cnf["Dinstall::Options::%s" % (i)] = ""
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Dinstall::Options")
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Dinstall::Options")
if Options["Help"]:
usage()
# If we have a directory flag, use it to find our files
- if Cnf["Dinstall::Options::Directory"] != "":
+ if cnf["Dinstall::Options::Directory"] != "":
# Note that we clobber the list of files we were given in this case
# so warn if the user has done both
if len(changes_files) > 0:
utils.warn("Directory provided so ignoring files given on command line")
- changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
-
- Upload = queue.Upload(Cnf)
- projectB = Upload.projectB
-
- changes = Upload.pkg.changes
- dsc = Upload.pkg.dsc
- dsc_files = Upload.pkg.dsc_files
- files = Upload.pkg.files
- pkg = Upload.pkg
- Subst = Upload.Subst
+ changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
return changes_files
###############################################################################
-def action (queue=""):
- (summary, short_summary) = Upload.build_summaries()
+def action (u, stable_queue=None, log_urgency=True, session=None):
+ (summary, short_summary) = u.build_summaries()
+ pi = u.package_info()
(prompt, answer) = ("", "XXX")
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
- if reject_message.find("Rejected") != -1:
- print "REJECT\n" + reject_message,
+ if len(u.rejects) > 0:
+ print "REJECT\n" + pi
prompt = "[R]eject, Skip, Quit ?"
if Options["Automatic"]:
answer = 'R'
else:
- print "INSTALL to " + ", ".join(changes["distribution"].keys())
- print reject_message + summary,
+ print "INSTALL to " + ", ".join(u.pkg.changes["distribution"].keys())
+ print pi + summary,
prompt = "[I]nstall, Skip, Quit ?"
if Options["Automatic"]:
answer = 'I'
answer = answer[:1].upper()
if answer == 'R':
- do_reject ()
+ u.do_unaccept()
+ Logger.log(["unaccepted", u.pkg.changes_file])
elif answer == 'I':
- if not installing_to_stable:
- install()
+ if stable_queue:
+ stable_install(u, summary, short_summary, stable_queue, log_urgency)
else:
- stable_install(summary, short_summary, queue)
+ install(u, session, log_urgency)
elif answer == 'Q':
sys.exit(0)
+
###############################################################################
+def add_poolfile(filename, datadict, location_id, session):
+ poolfile = PoolFile()
+ poolfile.filename = filename
+ poolfile.filesize = datadict["size"]
+ poolfile.md5sum = datadict["md5sum"]
+ poolfile.sha1sum = datadict["sha1sum"]
+ poolfile.sha256sum = datadict["sha256sum"]
+ poolfile.location_id = location_id
+
+ session.add(poolfile)
+ # Flush to get a file id (NB: This is not a commit)
+ session.flush()
+
+ return poolfile
+
+def add_dsc_to_db(u, filename, session):
+ entry = u.pkg.files[filename]
+ source = DBSource()
+
+ source.source = u.pkg.dsc["source"]
+ source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
+ source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
+ source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
+ source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
+ source.install_date = datetime.now().date()
+
+ dsc_component = entry["component"]
+ dsc_location_id = entry["location id"]
+
+ source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
+
+ # Set up a new poolfile if necessary
+ if not entry.has_key("files id") or not entry["files id"]:
+ filename = entry["pool name"] + filename
+ poolfile = add_poolfile(filename, entry, dsc_location_id, session)
+ entry["files id"] = poolfile.file_id
+
+ source.poolfile_id = entry["files id"]
+ session.add(source)
+ session.flush()
+
+ for suite_name in u.pkg.changes["distribution"].keys():
+ sa = SrcAssociation()
+ sa.source_id = source.source_id
+ sa.suite_id = get_suite(suite_name).suite_id
+ session.add(sa)
+
+ session.flush()
+
+ # Add the source files to the DB (files and dsc_files)
+ dscfile = DSCFile()
+ dscfile.source_id = source.source_id
+ dscfile.poolfile_id = entry["files id"]
+ session.add(dscfile)
+
+ for dsc_file, dentry in u.pkg.dsc_files.items():
+ df = DSCFile()
+ df.source_id = source.source_id
+
+ # If the .orig.tar.gz is already in the pool, it's
+ # files id is stored in dsc_files by check_dsc().
+ files_id = dentry.get("files id", None)
+
+ # Find the entry in the files hash
+ # TODO: Bail out here properly
+ dfentry = None
+ for f, e in u.pkg.files.items():
+ if f == dsc_file:
+ dfentry = e
+ break
+
+ if files_id is None:
+ filename = dfentry["pool name"] + dsc_file
+
+ (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
+ # FIXME: needs to check for -1/-2 and or handle exception
+ if found and obj is not None:
+ files_id = obj.file_id
+
+ # If still not found, add it
+ if files_id is None:
+ # HACK: Force sha1sum etc into dentry
+ dentry["sha1sum"] = dfentry["sha1sum"]
+ dentry["sha256sum"] = dfentry["sha256sum"]
+ poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
+ files_id = poolfile.file_id
+
+ df.poolfile_id = files_id
+ session.add(df)
+
+ session.flush()
+
+ # Add the src_uploaders to the DB
+ uploader_ids = [source.maintainer_id]
+ if u.pkg.dsc.has_key("uploaders"):
+ for up in u.pkg.dsc["uploaders"].split(","):
+ up = up.strip()
+ uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
+
+ added_ids = {}
+ for up in uploader_ids:
+ if added_ids.has_key(up):
+ utils.warn("Already saw uploader %s for source %s" % (up, source.source))
+ continue
-# Our reject is not really a reject, but an unaccept, but since a) the
-# code for that is non-trivial (reopen bugs, unannounce etc.), b) this
-# should be exteremly rare, for now we'll go with whining at our admin
-# folks...
-
-def do_reject ():
- Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
- Subst["__REJECT_MESSAGE__"] = reject_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
- reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-accepted.unaccept")
-
- # Write the rejection email out as the <foo>.reason file
- reason_filename = os.path.basename(pkg.changes_file[:-8]) + ".reason"
- reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename
- # If we fail here someone is probably trying to exploit the race
- # so let's just raise an exception ...
- if os.path.exists(reject_filename):
- os.unlink(reject_filename)
- fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
- os.write(fd, reject_mail_message)
- os.close(fd)
-
- utils.send_mail(reject_mail_message)
- Logger.log(["unaccepted", pkg.changes_file])
+ added_ids[u]=1
-###############################################################################
+ su = SrcUploader()
+ su.maintainer_id = up
+ su.source_id = source.source_id
+ session.add(su)
-def install ():
- global install_count, install_bytes
+ session.flush()
- print "Installing."
+ return dsc_component, dsc_location_id
+
+def add_deb_to_db(u, filename, session):
+ """
+ Contrary to what you might expect, this routine deals with both
+ debs and udebs. That info is in 'dbtype', whilst 'type' is
+ 'deb' for both of them
+ """
+ cnf = Config()
+ entry = u.pkg.files[filename]
+
+ bin = DBBinary()
+ bin.package = entry["package"]
+ bin.version = entry["version"]
+ bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
+ bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
+ bin.arch_id = get_architecture(entry["architecture"], session).arch_id
+ bin.binarytype = entry["dbtype"]
+
+ # Find poolfile id
+ filename = entry["pool name"] + filename
+ fullpath = os.path.join(cnf["Dir::Pool"], filename)
+ if not entry.get("location id", None):
+ entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
+
+ if not entry.get("files id", None):
+ poolfile = add_poolfile(filename, entry, entry["location id"], session)
+ entry["files id"] = poolfile.file_id
+
+ bin.poolfile_id = entry["files id"]
- Logger.log(["installing changes",pkg.changes_file])
+ # Find source id
+ bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
+ if len(bin_sources) != 1:
+ raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
+ (bin.package, bin.version, bin.architecture.arch_string,
+ filename, bin.binarytype, u.pkg.changes["fingerprint"])
+
+ bin.source_id = bin_sources[0].source_id
+
+ # Add and flush object so it has an ID
+ session.add(bin)
+ session.flush()
+
+ # Add BinAssociations
+ for suite_name in u.pkg.changes["distribution"].keys():
+ ba = BinAssociation()
+ ba.binary_id = bin.binary_id
+ ba.suite_id = get_suite(suite_name).suite_id
+ session.add(ba)
+
+ session.flush()
+
+ # Deal with contents
+ contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
+ if not contents:
+ print "REJECT\nCould not determine contents of package %s" % bin.package
+ session.rollback()
+ raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
+
+
+def install(u, session, log_urgency=True):
+ cnf = Config()
+ summarystats = SummaryStats()
+
+ print "Installing."
- # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
- projectB.query("BEGIN WORK")
+ Logger.log(["installing changes", u.pkg.changes_file])
# Ensure that we have all the hashes we need below.
- rejmsg = utils.ensure_hashes(changes, dsc, files, dsc_files)
- if len(rejmsg) > 0:
+ u.ensure_hashes()
+ if len(u.rejects) > 0:
# There were errors. Print them and SKIP the changes.
- for msg in rejmsg:
+ for msg in u.rejects:
utils.warn(msg)
return
- # Add the .dsc file to the DB
- for newfile in files.keys():
- if files[newfile]["type"] == "dsc":
- package = dsc["source"]
- version = dsc["version"] # NB: not files[file]["version"], that has no epoch
- maintainer = dsc["maintainer"]
- maintainer = maintainer.replace("'", "\\'")
- maintainer_id = database.get_or_set_maintainer_id(maintainer)
- changedby = changes["changed-by"]
- changedby = changedby.replace("'", "\\'")
- changedby_id = database.get_or_set_maintainer_id(changedby)
- fingerprint_id = database.get_or_set_fingerprint_id(dsc["fingerprint"])
- install_date = time.strftime("%Y-%m-%d")
- filename = files[newfile]["pool name"] + newfile
- dsc_component = files[newfile]["component"]
- dsc_location_id = files[newfile]["location id"]
- if dsc.has_key("dm-upload-allowed") and dsc["dm-upload-allowed"] == "yes":
- dm_upload_allowed = "true"
- else:
- dm_upload_allowed = "false"
- if not files[newfile].has_key("files id") or not files[newfile]["files id"]:
- files[newfile]["files id"] = database.set_files_id (filename, files[newfile]["size"], files[newfile]["md5sum"], files[newfile]["sha1sum"], files[newfile]["sha256sum"], dsc_location_id)
- projectB.query("INSERT INTO source (source, version, maintainer, changedby, file, install_date, sig_fpr, dm_upload_allowed) VALUES ('%s', '%s', %d, %d, %d, '%s', %s, %s)"
- % (package, version, maintainer_id, changedby_id, files[newfile]["files id"], install_date, fingerprint_id, dm_upload_allowed))
-
- for suite in changes["distribution"].keys():
- suite_id = database.get_suite_id(suite)
- projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
-
- # Add the source files to the DB (files and dsc_files)
- projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[newfile]["files id"]))
- for dsc_file in dsc_files.keys():
- filename = files[newfile]["pool name"] + dsc_file
- # If the .orig.tar.gz is already in the pool, it's
- # files id is stored in dsc_files by check_dsc().
- files_id = dsc_files[dsc_file].get("files id", None)
- if files_id == None:
- files_id = database.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id)
- # FIXME: needs to check for -1/-2 and or handle exception
- if files_id == None:
- files_id = database.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], files[dsc_file]["sha1sum"], files[dsc_file]["sha256sum"], dsc_location_id)
- projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id))
-
- # Add the src_uploaders to the DB
- uploader_ids = [maintainer_id]
- if dsc.has_key("uploaders"):
- for u in dsc["uploaders"].split(","):
- u = u.replace("'", "\\'")
- u = u.strip()
- uploader_ids.append(
- database.get_or_set_maintainer_id(u))
- added_ids = {}
- for u in uploader_ids:
- if added_ids.has_key(u):
- utils.warn("Already saw uploader %s for source %s" % (u, package))
- continue
- added_ids[u]=1
- projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u))
-
-
- # Add the .deb files to the DB
- for newfile in files.keys():
- if files[newfile]["type"] == "deb":
- package = files[newfile]["package"]
- version = files[newfile]["version"]
- maintainer = files[newfile]["maintainer"]
- maintainer = maintainer.replace("'", "\\'")
- maintainer_id = database.get_or_set_maintainer_id(maintainer)
- fingerprint_id = database.get_or_set_fingerprint_id(changes["fingerprint"])
- architecture = files[newfile]["architecture"]
- architecture_id = database.get_architecture_id (architecture)
- filetype = files[newfile]["dbtype"]
- source = files[newfile]["source package"]
- source_version = files[newfile]["source version"]
- filename = files[newfile]["pool name"] + newfile
- if not files[newfile].has_key("location id") or not files[newfile]["location id"]:
- files[newfile]["location id"] = database.get_location_id(Cnf["Dir::Pool"],files[newfile]["component"],utils.where_am_i())
- if not files[newfile].has_key("files id") or not files[newfile]["files id"]:
- files[newfile]["files id"] = database.set_files_id (filename, files[newfile]["size"], files[newfile]["md5sum"], files[newfile]["sha1sum"], files[newfile]["sha256sum"], files[newfile]["location id"])
- source_id = database.get_source_id (source, source_version)
- if source_id:
- projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, %d, '%s', %d)"
- % (package, version, maintainer_id, source_id, architecture_id, files[newfile]["files id"], filetype, fingerprint_id))
- else:
- raise NoSourceFieldError, "Unable to find a source id for %s (%s), %s, file %s, type %s, signed by %s" % (package, version, architecture, newfile, filetype, changes["fingerprint"])
- for suite in changes["distribution"].keys():
- suite_id = database.get_suite_id(suite)
- projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id))
-
- if not database.copy_temporary_contents(package, version, architecture, newfile, reject):
- print "REJECT\n" + reject_message,
- projectB.query("ROLLBACK")
- raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (package, newfile )
+ # Add the .dsc file to the DB first
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "dsc":
+ dsc_component, dsc_location_id = add_dsc_to_db(u, newfile, session)
-
- orig_tar_id = Upload.pkg.orig_tar_id
- orig_tar_location = Upload.pkg.orig_tar_location
+ # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "deb":
+ add_deb_to_db(u, newfile, session)
# If this is a sourceful diff only upload that is moving
# cross-component we need to copy the .orig.tar.gz into the new
# component too for the same reasons as above.
#
- if changes["architecture"].has_key("source") and orig_tar_id and \
- orig_tar_location != dsc_location_id:
- q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum, f.sha1sum, f.sha256sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id))
- ql = q.getresult()[0]
- old_filename = ql[0] + ql[1]
- file_size = ql[2]
- file_md5sum = ql[3]
- file_sha1sum = ql[4]
- file_sha256sum = ql[5]
- new_filename = utils.poolify(changes["source"], dsc_component) + os.path.basename(old_filename)
- new_files_id = database.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id)
- if new_files_id == None:
- utils.copy(old_filename, Cnf["Dir::Pool"] + new_filename)
- new_files_id = database.set_files_id(new_filename, file_size, file_md5sum, file_sha1sum, file_sha256sum, dsc_location_id)
- projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, database.get_source_id(changes["source"], changes["version"]), orig_tar_id))
+ if u.pkg.changes["architecture"].has_key("source") and u.pkg.orig_tar_id and \
+ u.pkg.orig_tar_location != dsc_location_id:
+
+ oldf = get_poolfile_by_id(u.pkg.orig_tar_id, session)
+ old_filename = os.path.join(oldf.location.path, oldf.filename)
+ old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
+ 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
+
+ new_filename = os.path.join(utils.poolify(u.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
+
+ # TODO: Care about size/md5sum collisions etc
+ (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
+
+ if newf is None:
+ utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
+ newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
+
+ # TODO: Check that there's only 1 here
+ source = get_sources_from_name(u.pkg.changes["source"], u.pkg.changes["version"])[0]
+ dscf = get_dscfiles(source_id = source.source_id, poolfile_id=u.pkg.orig_tar_id, session=session)[0]
+ dscf.poolfile_id = newf.file_id
+ session.add(dscf)
+ session.flush()
# Install the files into the pool
- for newfile in files.keys():
- destination = Cnf["Dir::Pool"] + files[newfile]["pool name"] + newfile
+ for newfile, entry in u.pkg.files.items():
+ destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
utils.move(newfile, destination)
- Logger.log(["installed", newfile, files[newfile]["type"], files[newfile]["size"], files[newfile]["architecture"]])
- install_bytes += float(files[newfile]["size"])
+ Logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
+ summarystats.accept_bytes += float(entry["size"])
# Copy the .changes file across for suite which need it.
copy_changes = {}
copy_dot_dak = {}
- for suite in changes["distribution"].keys():
- if Cnf.has_key("Suite::%s::CopyChanges" % (suite)):
- copy_changes[Cnf["Suite::%s::CopyChanges" % (suite)]] = ""
+ for suite_name in u.pkg.changes["distribution"].keys():
+ if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
+ copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
# and the .dak file...
- if Cnf.has_key("Suite::%s::CopyDotDak" % (suite)):
- copy_dot_dak[Cnf["Suite::%s::CopyDotDak" % (suite)]] = ""
+ if cnf.has_key("Suite::%s::CopyDotDak" % (suite_name)):
+ copy_dot_dak[cnf["Suite::%s::CopyDotDak" % (suite_name)]] = ""
+
for dest in copy_changes.keys():
- utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest)
+ utils.copy(u.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
+
for dest in copy_dot_dak.keys():
- utils.copy(Upload.pkg.changes_file[:-8]+".dak", dest)
- projectB.query("COMMIT WORK")
+ utils.copy(u.pkg.changes_file[:-8]+".dak", dest)
+
+ # We're done - commit the database changes
+ session.commit()
# Move the .changes into the 'done' directory
- utils.move (pkg.changes_file,
- os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file)))
+ utils.move(u.pkg.changes_file,
+ os.path.join(cnf["Dir::Queue::Done"], os.path.basename(u.pkg.changes_file)))
# Remove the .dak file
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ os.unlink(u.pkg.changes_file[:-8] + ".dak")
+
+ if u.pkg.changes["architecture"].has_key("source") and log_urgency:
+ UrgencyLog().log(u.pkg.dsc["source"], u.pkg.dsc["version"], u.pkg.changes["urgency"])
- if changes["architecture"].has_key("source") and Urgency_Logger:
- Urgency_Logger.log(dsc["source"], dsc["version"], changes["urgency"])
+ # Our SQL session will automatically start a new transaction after
+ # the last commit
# Undo the work done in queue.py(accept) to help auto-building
# from accepted.
- projectB.query("BEGIN WORK")
- for suite in changes["distribution"].keys():
- if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
+ now_date = datetime.now()
+
+ for suite_name in u.pkg.changes["distribution"].keys():
+ if suite_name not in cnf.ValueList("Dinstall::QueueBuildSuites"):
continue
- now_date = time.strftime("%Y-%m-%d %H:%M")
- suite_id = database.get_suite_id(suite)
- dest_dir = Cnf["Dir::QueueBuild"]
- if Cnf.FindB("Dinstall::SecurityQueueBuild"):
- dest_dir = os.path.join(dest_dir, suite)
- for newfile in files.keys():
+
+ suite = get_suite(suite_name, session)
+ dest_dir = cnf["Dir::QueueBuild"]
+
+ if cnf.FindB("Dinstall::SecurityQueueBuild"):
+ dest_dir = os.path.join(dest_dir, suite_name)
+
+ for newfile, entry in u.pkg.files.items():
dest = os.path.join(dest_dir, newfile)
+
+ qb = get_queue_build(dest, suite.suite_id, session)
+
# Remove it from the list of packages for later processing by apt-ftparchive
- projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, dest, suite_id))
- if not Cnf.FindB("Dinstall::SecurityQueueBuild"):
+ if qb:
+ qb.last_used = now_date
+ qb.in_queue = False
+ session.add(qb)
+
+ if not cnf.FindB("Dinstall::SecurityQueueBuild"):
# Update the symlink to point to the new location in the pool
- pool_location = utils.poolify (changes["source"], files[newfile]["component"])
- src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(newfile))
+ pool_location = utils.poolify(u.pkg.changes["source"], entry["component"])
+ src = os.path.join(cnf["Dir::Pool"], pool_location, os.path.basename(newfile))
if os.path.islink(dest):
os.unlink(dest)
os.symlink(src, dest)
+
# Update last_used on any non-upload .orig.tar.gz symlink
- if orig_tar_id:
+ if u.pkg.orig_tar_id:
# Determine the .orig.tar.gz file name
- for dsc_file in dsc_files.keys():
+ for dsc_file in u.pkg.dsc_files.keys():
if dsc_file.endswith(".orig.tar.gz"):
- orig_tar_gz = os.path.join(dest_dir, dsc_file)
+ u.pkg.orig_tar_gz = os.path.join(dest_dir, dsc_file)
+
# Remove it from the list of packages for later processing by apt-ftparchive
- projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, orig_tar_gz, suite_id))
- projectB.query("COMMIT WORK")
+ qb = get_queue_build(u.pkg.orig_tar_gz, suite.suite_id, session)
+ if qb:
+ qb.in_queue = False
+ qb.last_used = now_date
+ session.add(qb)
+
+ session.commit()
# Finally...
- install_count += 1
+ summarystats.accept_count += 1
################################################################################
-def stable_install (summary, short_summary, fromsuite="proposed-updates"):
- global install_count
+def stable_install(u, session, summary, short_summary, fromsuite_name="proposed-updates"):
+ summarystats = SummaryStats()
- fromsuite = fromsuite.lower()
- tosuite = "Stable"
- if fromsuite == "oldstable-proposed-updates":
- tosuite = "OldStable"
+ fromsuite_name = fromsuite_name.lower()
+ tosuite_name = "Stable"
+ if fromsuite_name == "oldstable-proposed-updates":
+ tosuite_name = "OldStable"
- print "Installing from %s to %s." % (fromsuite, tosuite)
+ print "Installing from %s to %s." % (fromsuite_name, tosuite_name)
- # Begin a transaction; if we bomb out anywhere between here and
- # the COMMIT WORK below, the DB won't be changed.
- projectB.query("BEGIN WORK")
+ fromsuite = get_suite(fromsuite_name)
+ tosuite = get_suite(tosuite_name)
# Add the source to stable (and remove it from proposed-updates)
- for newfile in files.keys():
- if files[newfile]["type"] == "dsc":
- package = dsc["source"]
- version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
- q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
- ql = q.getresult()
- if not ql:
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "dsc":
+ package = u.pkg.dsc["source"]
+ # NB: not files[file]["version"], that has no epoch
+ version = u.pkg.dsc["version"]
+
+ source = get_sources_from_name(package, version, session)
+ if len(source) < 1:
utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version))
- source_id = ql[0][0]
- suite_id = database.get_suite_id(fromsuite)
- projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id))
- suite_id = database.get_suite_id(tosuite.lower())
- projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id))
+ source = source[0]
+
+ # Remove from old suite
+ old = session.query(SrcAssociation).filter_by(source_id = source.source_id)
+ old = old.filter_by(suite_id = fromsuite.suite_id)
+ old.delete()
+
+ # Add to new suite
+ new = SrcAssociation()
+ new.source_id = source.source_id
+ new.suite_id = tosuite.suite_id
+ session.add(new)
# Add the binaries to stable (and remove it/them from proposed-updates)
- for newfile in files.keys():
- if files[newfile]["type"] == "deb":
- package = files[newfile]["package"]
- version = files[newfile]["version"]
- architecture = files[newfile]["architecture"]
- q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
- ql = q.getresult()
- if not ql:
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "deb":
+ package = entry["package"]
+ version = entry["version"]
+ architecture = entry["architecture"]
+
+ binary = get_binaries_from_name(package, version, [architecture, 'all'])
+
+ if len(binary) < 1:
utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture))
+ binary = binary[0]
+
+ # Remove from old suite
+ old = session.query(BinAssociation).filter_by(binary_id = binary.binary_id)
+ old = old.filter_by(suite_id = fromsuite.suite_id)
+ old.delete()
- binary_id = ql[0][0]
- suite_id = database.get_suite_id(fromsuite)
- projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id))
- suite_id = database.get_suite_id(tosuite.lower())
- projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id))
+ # Add to new suite
+ new = BinAssociation()
+ new.binary_id = binary.binary_id
+ new.suite_id = tosuite.suite_id
+ session.add(new)
- projectB.query("COMMIT WORK")
+ session.commit()
- utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/process-accepted/' + os.path.basename(pkg.changes_file))
+ utils.move(u.pkg.changes_file,
+ os.path.join(cnf["Dir::Morgue"], 'process-accepted', os.path.basename(u.pkg.changes_file)))
## Update the Stable ChangeLog file
- new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::%s::ChangeLogBase" % (tosuite)] + ".ChangeLog"
- changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::%s::ChangeLogBase" % (tosuite)] + "ChangeLog"
+ # TODO: URGH - Use a proper tmp file
+ new_changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + ".ChangeLog"
+ changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + "ChangeLog"
if os.path.exists(new_changelog_filename):
- os.unlink (new_changelog_filename)
+ os.unlink(new_changelog_filename)
new_changelog = utils.open_file(new_changelog_filename, 'w')
- for newfile in files.keys():
- if files[newfile]["type"] == "deb":
- new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.lower(), files[newfile]["component"], files[newfile]["architecture"], newfile))
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "deb":
+ new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.suite_name,
+ entry["component"],
+ entry["architecture"],
+ newfile))
elif re_issource.match(newfile):
- new_changelog.write("%s/%s/source/%s\n" % (tosuite.lower(), files[newfile]["component"], newfile))
+ new_changelog.write("%s/%s/source/%s\n" % (tosuite.suite_name,
+ entry["component"],
+ newfile))
else:
new_changelog.write("%s\n" % (newfile))
- chop_changes = re_fdnic.sub("\n", changes["changes"])
+
+ chop_changes = re_fdnic.sub("\n", u.pkg.changes["changes"])
new_changelog.write(chop_changes + '\n\n')
+
if os.access(changelog_filename, os.R_OK) != 0:
changelog = utils.open_file(changelog_filename)
new_changelog.write(changelog.read())
+
new_changelog.close()
+
if os.access(changelog_filename, os.R_OK) != 0:
os.unlink(changelog_filename)
utils.move(new_changelog_filename, changelog_filename)
- install_count += 1
+ summarystats.accept_count += 1
+
+ if not Options["No-Mail"] and u.pkg.changes["architecture"].has_key("source"):
+ u.Subst["__SUITE__"] = " into %s" % (tosuite)
+ u.Subst["__SUMMARY__"] = summary
+ u.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
- if not Options["No-Mail"] and changes["architecture"].has_key("source"):
- Subst["__SUITE__"] = " into %s" % (tosuite)
- Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-accepted.install")
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+
+ template = os.path.join(cnf["Dir::Templates"], 'process-accepted.install')
+
+ mail_message = utils.TemplateSubst(u.Subst, template)
utils.send_mail(mail_message)
- Upload.announce(short_summary, 1)
+ u.announce(short_summary, True)
# Finally remove the .dak file
- dot_dak_file = os.path.join(Cnf["Suite::%s::CopyDotDak" % (fromsuite)], os.path.basename(Upload.pkg.changes_file[:-8]+".dak"))
+ dot_dak_file = os.path.join(cnf["Suite::%s::CopyDotDak" % (fromsuite.suite_name)],
+ os.path.basename(u.pkg.changes_file[:-8]+".dak"))
os.unlink(dot_dak_file)
################################################################################
-def process_it (changes_file, queue=""):
- global reject_message
+def process_it(changes_file, stable_queue, log_urgency, session):
+ cnf = Config()
+ u = Upload()
- reject_message = ""
+ overwrite_checks = True
# Absolutize the filename to avoid the requirement of being in the
# same directory as the .changes file.
- pkg.changes_file = os.path.abspath(changes_file)
+ cfile = os.path.abspath(changes_file)
# And since handling of installs to stable munges with the CWD
# save and restore it.
- pkg.directory = os.getcwd()
+ u.prevdir = os.getcwd()
- if installing_to_stable:
- old = Upload.pkg.changes_file
- Upload.pkg.changes_file = os.path.basename(old)
- os.chdir(Cnf["Suite::%s::CopyDotDak" % (queue)])
+ if stable_queue:
+ old = cfile
+ cfile = os.path.basename(old)
+ os.chdir(cnf["Suite::%s::CopyDotDak" % (stable_queue)])
+ # overwrite_checks should not be performed if installing to stable
+ overwrite_checks = False
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
+ u.pkg.load_dot_dak(cfile)
+ u.update_subst()
- if installing_to_stable:
- Upload.pkg.changes_file = old
+ if stable_queue:
+ u.pkg.changes_file = old
- check()
- action(queue)
+ u.accepted_checks(overwrite_checks, session)
+ action(u, stable_queue, log_urgency, session)
# Restore CWD
- os.chdir(pkg.directory)
+ os.chdir(u.prevdir)
###############################################################################
def main():
- global projectB, Logger, Urgency_Logger, installing_to_stable
+ global Logger
+ cnf = Config()
+ summarystats = SummaryStats()
changes_files = init()
+ log_urgency = False
+ stable_queue = None
# -n/--dry-run invalidates some other options which would involve things happening
if Options["No-Action"]:
# Check that we aren't going to clash with the daily cron job
- if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]:
+ if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (cnf["Dir::Root"])) and not Options["No-Lock"]:
utils.fubar("Archive maintenance in progress. Try again later.")
# If running from within proposed-updates; assume an install to stable
queue = ""
if os.getenv('PWD').find('oldstable-proposed-updates') != -1:
- queue = "Oldstable-Proposed-Updates"
- installing_to_stable = 1
+ stable_queue = "Oldstable-Proposed-Updates"
elif os.getenv('PWD').find('proposed-updates') != -1:
- queue = "Proposed-Updates"
- installing_to_stable = 1
+ stable_queue = "Proposed-Updates"
# Obtain lock if not in no-action mode and initialize the log
if not Options["No-Action"]:
- lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+ lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError, e:
utils.fubar("Couldn't obtain lock; assuming another 'dak process-accepted' is already running.")
else:
raise
- Logger = Upload.Logger = logging.Logger(Cnf, "process-accepted")
- if not installing_to_stable and Cnf.get("Dir::UrgencyLog"):
- Urgency_Logger = Urgency_Log(Cnf)
-
- # Initialize the substitution template mapping global
- bcc = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Subst["__BCC__"] = bcc
+ Logger = daklog.Logger(cnf, "process-accepted")
+ if not stable_queue and cnf.get("Dir::UrgencyLog"):
+ # Initialise UrgencyLog()
+ log_urgency = True
+ UrgencyLog()
# Sort the .changes files so that we process sourceful ones first
changes_files.sort(utils.changes_compare)
+
# Process the changes files
for changes_file in changes_files:
print "\n" + changes_file
- process_it (changes_file, queue)
+ session = DBConn().session()
+ process_it(changes_file, stable_queue, log_urgency, session)
+ session.close()
- if install_count:
+ if summarystats.accept_count:
sets = "set"
- if install_count > 1:
+ if summarystats.accept_count > 1:
sets = "sets"
- sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))))
- Logger.log(["total",install_count,install_bytes])
+ sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets,
+ utils.size_type(int(summarystats.accept_bytes))))
+ Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
if not Options["No-Action"]:
Logger.close()
- if Urgency_Logger:
- Urgency_Logger.close()
+ if log_urgency:
+ UrgencyLog().close()
###############################################################################
import pwd
import apt_pkg, apt_inst
import examine_package
+
from daklib import database
-from daklib import logging
+from daklib import daklog
from daklib import queue
from daklib import utils
from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
+from daklib.summarystats import SummaryStats
# Globals
Cnf = None #: Configuration, apt_pkg.Configuration
if not Options["No-Action"]:
try:
- Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
+ Logger = Upload.Logger = daklog.Logger(Cnf, "process-new")
except CantOpenError, e:
Options["Trainee"] = "True"
################################################################################
def end():
- accept_count = Upload.accept_count
- accept_bytes = Upload.accept_bytes
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
if accept_count:
sets = "set"
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
@license: GNU General Public License version 2 or later
"""
import apt_inst
import apt_pkg
from debian_bundle import deb822
-from daklib.dbconn import DBConn
+
+from daklib.dbconn import *
from daklib.binary import Binary
-from daklib import logging
-from daklib import queue
+from daklib import daklog
+from daklib.queue import *
from daklib import utils
+from daklib.textutils import fix_maintainer
from daklib.dak_exceptions import *
-from daklib.regexes import re_valid_version, re_valid_pkg_name, re_changelog_versions, \
- re_strip_revision, re_strip_srcver, re_spacestrip, \
- re_isanum, re_no_epoch, re_no_revision, re_taint_free, \
- re_isadeb, re_extract_src_version, re_issource, re_default_answer
+from daklib.regexes import re_default_answer
+from daklib.summarystats import SummaryStats
+from daklib.holding import Holding
+from daklib.config import Config
from types import *
################################################################################
# Globals
-Cnf = None
Options = None
Logger = None
-Upload = None
-
-reprocess = 0
-in_holding = {}
-
-# Aliases to the real vars in the Upload class; hysterical raisins.
-reject_message = ""
-changes = {}
-dsc = {}
-dsc_files = {}
-files = {}
-pkg = {}
###############################################################################
def init():
- global Cnf, Options, Upload, changes, dsc, dsc_files, files, pkg
+ global Options
apt_pkg.init()
-
- Cnf = apt_pkg.newConfiguration()
- apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file())
+ cnf = Config()
Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
('h',"help","Dinstall::Options::Help"),
for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
"override-distribution", "version", "directory"]:
- Cnf["Dinstall::Options::%s" % (i)] = ""
+ cnf["Dinstall::Options::%s" % (i)] = ""
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Dinstall::Options")
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Dinstall::Options")
if Options["Help"]:
usage()
# If we have a directory flag, use it to find our files
- if Cnf["Dinstall::Options::Directory"] != "":
+ if cnf["Dinstall::Options::Directory"] != "":
# Note that we clobber the list of files we were given in this case
# so warn if the user has done both
if len(changes_files) > 0:
utils.warn("Directory provided so ignoring files given on command line")
- changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
-
- Upload = queue.Upload(Cnf)
-
- changes = Upload.pkg.changes
- dsc = Upload.pkg.dsc
- dsc_files = Upload.pkg.dsc_files
- files = Upload.pkg.files
- pkg = Upload.pkg
+ changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
return changes_files
################################################################################
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-################################################################################
-
-def copy_to_holding(filename):
- global in_holding
-
- base_filename = os.path.basename(filename)
-
- dest = Cnf["Dir::Queue::Holding"] + '/' + base_filename
- try:
- fd = os.open(dest, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0640)
- os.close(fd)
- except OSError, e:
- # Shouldn't happen, but will if, for example, someone lists a
- # file twice in the .changes.
- if errno.errorcode[e.errno] == 'EEXIST':
- reject("%s: already exists in holding area; can not overwrite." % (base_filename))
- return
- raise
-
- try:
- shutil.copy(filename, dest)
- except IOError, e:
- # In either case (ENOENT or EACCES) we want to remove the
- # O_CREAT | O_EXCLed ghost file, so add the file to the list
- # of 'in holding' even if it's not the real file.
- if errno.errorcode[e.errno] == 'ENOENT':
- reject("%s: can not copy to holding area: file not found." % (base_filename))
- os.unlink(dest)
- return
- elif errno.errorcode[e.errno] == 'EACCES':
- reject("%s: can not copy to holding area: read permission denied." % (base_filename))
- os.unlink(dest)
- return
- raise
-
- in_holding[base_filename] = ""
-
-################################################################################
-
-def clean_holding():
- global in_holding
-
- cwd = os.getcwd()
- os.chdir(Cnf["Dir::Queue::Holding"])
- for f in in_holding.keys():
- if os.path.exists(f):
- if f.find('/') != -1:
- utils.fubar("WTF? clean_holding() got a file ('%s') with / in it!" % (f))
- else:
- os.unlink(f)
- in_holding = {}
- os.chdir(cwd)
-
-################################################################################
-
-def check_changes():
- filename = pkg.changes_file
-
- # Parse the .changes field into a dictionary
- try:
- changes.update(utils.parse_changes(filename))
- except CantOpenError:
- reject("%s: can't read file." % (filename))
- return 0
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (filename, line))
- return 0
- except ChangesUnicodeError:
- reject("%s: changes file not proper utf-8" % (filename))
- return 0
-
- # Parse the Files field from the .changes into another dictionary
- try:
- files.update(utils.build_file_list(changes))
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (filename, line))
- except UnknownFormatError, format:
- reject("%s: unknown format '%s'." % (filename, format))
- return 0
-
- # Check for mandatory fields
- for i in ("source", "binary", "architecture", "version", "distribution",
- "maintainer", "files", "changes", "description"):
- if not changes.has_key(i):
- reject("%s: Missing mandatory field `%s'." % (filename, i))
- return 0 # Avoid <undef> errors during later tests
-
- # Strip a source version in brackets from the source field
- if re_strip_srcver.search(changes["source"]):
- changes["source"] = re_strip_srcver.sub('', changes["source"])
-
- # Ensure the source field is a valid package name.
- if not re_valid_pkg_name.match(changes["source"]):
- reject("%s: invalid source name '%s'." % (filename, changes["source"]))
-
- # Split multi-value fields into a lower-level dictionary
- for i in ("architecture", "distribution", "binary", "closes"):
- o = changes.get(i, "")
- if o != "":
- del changes[i]
- changes[i] = {}
- for j in o.split():
- changes[i][j] = 1
-
- # Fix the Maintainer: field to be RFC822/2047 compatible
- try:
- (changes["maintainer822"], changes["maintainer2047"],
- changes["maintainername"], changes["maintaineremail"]) = \
- utils.fix_maintainer (changes["maintainer"])
- except ParseMaintError, msg:
- reject("%s: Maintainer field ('%s') failed to parse: %s" \
- % (filename, changes["maintainer"], msg))
-
- # ...likewise for the Changed-By: field if it exists.
- try:
- (changes["changedby822"], changes["changedby2047"],
- changes["changedbyname"], changes["changedbyemail"]) = \
- utils.fix_maintainer (changes.get("changed-by", ""))
- except ParseMaintError, msg:
- (changes["changedby822"], changes["changedby2047"],
- changes["changedbyname"], changes["changedbyemail"]) = \
- ("", "", "", "")
- reject("%s: Changed-By field ('%s') failed to parse: %s" \
- % (filename, changes["changed-by"], msg))
-
- # Ensure all the values in Closes: are numbers
- if changes.has_key("closes"):
- for i in changes["closes"].keys():
- if re_isanum.match (i) == None:
- reject("%s: `%s' from Closes field isn't a number." % (filename, i))
-
-
- # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
- changes["chopversion"] = re_no_epoch.sub('', changes["version"])
- changes["chopversion2"] = re_no_revision.sub('', changes["chopversion"])
-
- # Check there isn't already a changes file of the same name in one
- # of the queue directories.
- base_filename = os.path.basename(filename)
- for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
- if os.path.exists(Cnf["Dir::Queue::%s" % (d) ]+'/'+base_filename):
- reject("%s: a file with this name already exists in the %s directory." % (base_filename, d))
-
- # Check the .changes is non-empty
- if not files:
- reject("%s: nothing to do (Files field is empty)." % (base_filename))
- return 0
-
- return 1
-
-################################################################################
-
-def check_distributions():
- "Check and map the Distribution field of a .changes file."
-
- # Handle suite mappings
- for m in Cnf.ValueList("SuiteMappings"):
- args = m.split()
- mtype = args[0]
- if mtype == "map" or mtype == "silent-map":
- (source, dest) = args[1:3]
- if changes["distribution"].has_key(source):
- del changes["distribution"][source]
- changes["distribution"][dest] = 1
- if mtype != "silent-map":
- reject("Mapping %s to %s." % (source, dest),"")
- if changes.has_key("distribution-version"):
- if changes["distribution-version"].has_key(source):
- changes["distribution-version"][source]=dest
- elif mtype == "map-unreleased":
- (source, dest) = args[1:3]
- if changes["distribution"].has_key(source):
- for arch in changes["architecture"].keys():
- if arch not in DBConn().get_suite_architectures(source):
- reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),"")
- del changes["distribution"][source]
- changes["distribution"][dest] = 1
- break
- elif mtype == "ignore":
- suite = args[1]
- if changes["distribution"].has_key(suite):
- del changes["distribution"][suite]
- reject("Ignoring %s as a target suite." % (suite), "Warning: ")
- elif mtype == "reject":
- suite = args[1]
- if changes["distribution"].has_key(suite):
- reject("Uploads to %s are not accepted." % (suite))
- elif mtype == "propup-version":
- # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
- #
- # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
- if changes["distribution"].has_key(args[1]):
- changes.setdefault("distribution-version", {})
- for suite in args[2:]: changes["distribution-version"][suite]=suite
-
- # Ensure there is (still) a target distribution
- if changes["distribution"].keys() == []:
- reject("no valid distribution.")
-
- # Ensure target distributions exist
- for suite in changes["distribution"].keys():
- if not Cnf.has_key("Suite::%s" % (suite)):
- reject("Unknown distribution `%s'." % (suite))
-
-################################################################################
-
-def check_files():
- global reprocess
-
- archive = utils.where_am_i()
- file_keys = files.keys()
-
- # if reprocess is 2 we've already done this and we're checking
- # things again for the new .orig.tar.gz.
- # [Yes, I'm fully aware of how disgusting this is]
- if not Options["No-Action"] and reprocess < 2:
- cwd = os.getcwd()
- os.chdir(pkg.directory)
- for f in file_keys:
- copy_to_holding(f)
- os.chdir(cwd)
-
- # Check there isn't already a .changes or .dak file of the same name in
- # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
- # [NB: this check must be done post-suite mapping]
- base_filename = os.path.basename(pkg.changes_file)
- dot_dak_filename = base_filename[:-8]+".dak"
- for suite in changes["distribution"].keys():
- copychanges = "Suite::%s::CopyChanges" % (suite)
- if Cnf.has_key(copychanges) and \
- os.path.exists(Cnf[copychanges]+"/"+base_filename):
- reject("%s: a file with this name already exists in %s" \
- % (base_filename, Cnf[copychanges]))
-
- copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
- if Cnf.has_key(copy_dot_dak) and \
- os.path.exists(Cnf[copy_dot_dak]+"/"+dot_dak_filename):
- reject("%s: a file with this name already exists in %s" \
- % (dot_dak_filename, Cnf[copy_dot_dak]))
-
- reprocess = 0
- has_binaries = 0
- has_source = 0
-
- cursor = DBConn().cursor()
- # Check for packages that have moved from one component to another
- # STU: this should probably be changed to not join on architecture, suite tables but instead to used their cached name->id mappings from DBConn
- DBConn().prepare("moved_pkg_q", """
- PREPARE moved_pkg_q(text,text,text) AS
- SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
- component c, architecture a, files f
- WHERE b.package = $1 AND s.suite_name = $2
- AND (a.arch_string = $3 OR a.arch_string = 'all')
- AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
- AND f.location = l.id
- AND l.component = c.id
- AND b.file = f.id""")
-
- for f in file_keys:
- # Ensure the file does not already exist in one of the accepted directories
- for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
- if not Cnf.has_key("Dir::Queue::%s" % (d)): continue
- if os.path.exists(Cnf["Dir::Queue::%s" % (d) ] + '/' + f):
- reject("%s file already exists in the %s directory." % (f, d))
- if not re_taint_free.match(f):
- reject("!!WARNING!! tainted filename: '%s'." % (f))
- # Check the file is readable
- if os.access(f, os.R_OK) == 0:
- # When running in -n, copy_to_holding() won't have
- # generated the reject_message, so we need to.
- if Options["No-Action"]:
- if os.path.exists(f):
- reject("Can't read `%s'. [permission denied]" % (f))
- else:
- reject("Can't read `%s'. [file not found]" % (f))
- files[f]["type"] = "unreadable"
- continue
- # If it's byhand skip remaining checks
- if files[f]["section"] == "byhand" or files[f]["section"][:4] == "raw-":
- files[f]["byhand"] = 1
- files[f]["type"] = "byhand"
- # Checks for a binary package...
- elif re_isadeb.match(f):
- has_binaries = 1
- files[f]["type"] = "deb"
-
- # Extract package control information
- deb_file = utils.open_file(f)
- try:
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
- except:
- reject("%s: debExtractControl() raised %s." % (f, sys.exc_type))
- deb_file.close()
- # Can't continue, none of the checks on control would work.
- continue
-
- # Check for mandantory "Description:"
- deb_file.seek ( 0 )
- try:
- apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
- except:
- reject("%s: Missing Description in binary package" % (f))
- continue
-
- deb_file.close()
-
- # Check for mandatory fields
- for field in [ "Package", "Architecture", "Version" ]:
- if control.Find(field) == None:
- reject("%s: No %s field in control." % (f, field))
- # Can't continue
- continue
-
- # Ensure the package name matches the one give in the .changes
- if not changes["binary"].has_key(control.Find("Package", "")):
- reject("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
-
- # Validate the package field
- package = control.Find("Package")
- if not re_valid_pkg_name.match(package):
- reject("%s: invalid package name '%s'." % (f, package))
-
- # Validate the version field
- version = control.Find("Version")
- if not re_valid_version.match(version):
- reject("%s: invalid version number '%s'." % (f, version))
-
- # Ensure the architecture of the .deb is one we know about.
- default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable")
- architecture = control.Find("Architecture")
- upload_suite = changes["distribution"].keys()[0]
- if architecture not in DBConn().get_suite_architectures(default_suite) and architecture not in DBConn().get_suite_architectures(upload_suite):
- reject("Unknown architecture '%s'." % (architecture))
-
- # Ensure the architecture of the .deb is one of the ones
- # listed in the .changes.
- if not changes["architecture"].has_key(architecture):
- reject("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
-
- # Sanity-check the Depends field
- depends = control.Find("Depends")
- if depends == '':
- reject("%s: Depends field is empty." % (f))
-
- # Sanity-check the Provides field
- provides = control.Find("Provides")
- if provides:
- provide = re_spacestrip.sub('', provides)
- if provide == '':
- reject("%s: Provides field is empty." % (f))
- prov_list = provide.split(",")
- for prov in prov_list:
- if not re_valid_pkg_name.match(prov):
- reject("%s: Invalid Provides field content %s." % (f, prov))
-
-
- # Check the section & priority match those given in the .changes (non-fatal)
- if control.Find("Section") and files[f]["section"] != "" and files[f]["section"] != control.Find("Section"):
- reject("%s control file lists section as `%s', but changes file has `%s'." % (f, control.Find("Section", ""), files[f]["section"]), "Warning: ")
- if control.Find("Priority") and files[f]["priority"] != "" and files[f]["priority"] != control.Find("Priority"):
- reject("%s control file lists priority as `%s', but changes file has `%s'." % (f, control.Find("Priority", ""), files[f]["priority"]),"Warning: ")
-
- files[f]["package"] = package
- files[f]["architecture"] = architecture
- files[f]["version"] = version
- files[f]["maintainer"] = control.Find("Maintainer", "")
- if f.endswith(".udeb"):
- files[f]["dbtype"] = "udeb"
- elif f.endswith(".deb"):
- files[f]["dbtype"] = "deb"
- else:
- reject("%s is neither a .deb or a .udeb." % (f))
- files[f]["source"] = control.Find("Source", files[f]["package"])
- # Get the source version
- source = files[f]["source"]
- source_version = ""
- if source.find("(") != -1:
- m = re_extract_src_version.match(source)
- source = m.group(1)
- source_version = m.group(2)
- if not source_version:
- source_version = files[f]["version"]
- files[f]["source package"] = source
- files[f]["source version"] = source_version
-
- # Ensure the filename matches the contents of the .deb
- m = re_isadeb.match(f)
- # package name
- file_package = m.group(1)
- if files[f]["package"] != file_package:
- reject("%s: package part of filename (%s) does not match package name in the %s (%s)." % (f, file_package, files[f]["dbtype"], files[f]["package"]))
- epochless_version = re_no_epoch.sub('', control.Find("Version"))
- # version
- file_version = m.group(2)
- if epochless_version != file_version:
- reject("%s: version part of filename (%s) does not match package version in the %s (%s)." % (f, file_version, files[f]["dbtype"], epochless_version))
- # architecture
- file_architecture = m.group(3)
- if files[f]["architecture"] != file_architecture:
- reject("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % (f, file_architecture, files[f]["dbtype"], files[f]["architecture"]))
-
- # Check for existent source
- source_version = files[f]["source version"]
- source_package = files[f]["source package"]
- if changes["architecture"].has_key("source"):
- if source_version != changes["version"]:
- reject("source version (%s) for %s doesn't match changes version %s." % (source_version, f, changes["version"]))
- else:
- # Check in the SQL database
- if not Upload.source_exists(source_package, source_version, changes["distribution"].keys()):
- # Check in one of the other directories
- source_epochless_version = re_no_epoch.sub('', source_version)
- dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if os.path.exists(Cnf["Dir::Queue::Byhand"] + '/' + dsc_filename):
- files[f]["byhand"] = 1
- elif os.path.exists(Cnf["Dir::Queue::New"] + '/' + dsc_filename):
- files[f]["new"] = 1
- else:
- dsc_file_exists = 0
- for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
- if Cnf.has_key("Dir::Queue::%s" % (myq)):
- if os.path.exists(Cnf["Dir::Queue::"+myq] + '/' + dsc_filename):
- dsc_file_exists = 1
- break
- if not dsc_file_exists:
- reject("no source found for %s %s (%s)." % (source_package, source_version, f))
- # Check the version and for file overwrites
- reject(Upload.check_binary_against_db(f),"")
-
- Binary(f, reject).scan_package()
-
- # Checks for a source package...
- else:
- m = re_issource.match(f)
- if m:
- has_source = 1
- files[f]["package"] = m.group(1)
- files[f]["version"] = m.group(2)
- files[f]["type"] = m.group(3)
-
- # Ensure the source package name matches the Source filed in the .changes
- if changes["source"] != files[f]["package"]:
- reject("%s: changes file doesn't say %s for Source" % (f, files[f]["package"]))
-
- # Ensure the source version matches the version in the .changes file
- if files[f]["type"] == "orig.tar.gz":
- changes_version = changes["chopversion2"]
- else:
- changes_version = changes["chopversion"]
- if changes_version != files[f]["version"]:
- reject("%s: should be %s according to changes file." % (f, changes_version))
-
- # Ensure the .changes lists source in the Architecture field
- if not changes["architecture"].has_key("source"):
- reject("%s: changes file doesn't list `source' in Architecture field." % (f))
-
- # Check the signature of a .dsc file
- if files[f]["type"] == "dsc":
- dsc["fingerprint"] = utils.check_signature(f, reject)
-
- files[f]["architecture"] = "source"
-
- # Not a binary or source package? Assume byhand...
- else:
- files[f]["byhand"] = 1
- files[f]["type"] = "byhand"
-
- # Per-suite file checks
- files[f]["oldfiles"] = {}
- for suite in changes["distribution"].keys():
- # Skip byhand
- if files[f].has_key("byhand"):
- continue
-
- # Handle component mappings
- for m in Cnf.ValueList("ComponentMappings"):
- (source, dest) = m.split()
- if files[f]["component"] == source:
- files[f]["original component"] = source
- files[f]["component"] = dest
-
- # Ensure the component is valid for the target suite
- if Cnf.has_key("Suite:%s::Components" % (suite)) and \
- files[f]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)):
- reject("unknown component `%s' for suite `%s'." % (files[f]["component"], suite))
- continue
-
- # Validate the component
- component = files[f]["component"]
- component_id = DBConn().get_component_id(component)
- if component_id == -1:
- reject("file '%s' has unknown component '%s'." % (f, component))
- continue
-
- # See if the package is NEW
- if not Upload.in_override_p(files[f]["package"], files[f]["component"], suite, files[f].get("dbtype",""), f):
- files[f]["new"] = 1
-
- # Validate the priority
- if files[f]["priority"].find('/') != -1:
- reject("file '%s' has invalid priority '%s' [contains '/']." % (f, files[f]["priority"]))
-
- # Determine the location
- location = Cnf["Dir::Pool"]
- location_id = DBConn().get_location_id(location, component, archive)
- if location_id == -1:
- reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
- files[f]["location id"] = location_id
-
- # Check the md5sum & size against existing files (if any)
- files[f]["pool name"] = utils.poolify (changes["source"], files[f]["component"])
- files_id = DBConn().get_files_id(files[f]["pool name"] + f, files[f]["size"], files[f]["md5sum"], files[f]["location id"])
- if files_id == -1:
- reject("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
- elif files_id == -2:
- reject("md5sum and/or size mismatch on existing copy of %s." % (f))
- files[f]["files id"] = files_id
-
- # Check for packages that have moved from one component to another
- files[f]['suite'] = suite
- cursor.execute("""EXECUTE moved_pkg_q( %(package)s, %(suite)s, %(architecture)s )""", ( files[f] ) )
- ql = cursor.fetchone()
- if ql:
- files[f]["othercomponents"] = ql[0][0]
-
- # If the .changes file says it has source, it must have source.
- if changes["architecture"].has_key("source"):
- if not has_source:
- reject("no source found and Architecture line in changes mention source.")
-
- if not has_binaries and Cnf.FindB("Dinstall::Reject::NoSourceOnly"):
- reject("source only uploads are not supported.")
-
-###############################################################################
-
-def check_dsc():
- global reprocess
-
- # Ensure there is source to check
- if not changes["architecture"].has_key("source"):
- return 1
-
- # Find the .dsc
- dsc_filename = None
- for f in files.keys():
- if files[f]["type"] == "dsc":
- if dsc_filename:
- reject("can not process a .changes file with multiple .dsc's.")
- return 0
- else:
- dsc_filename = f
-
- # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
- if not dsc_filename:
- reject("source uploads must contain a dsc file")
- return 0
-
- # Parse the .dsc file
- try:
- dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
- except CantOpenError:
- # if not -n copy_to_holding() will have done this for us...
- if Options["No-Action"]:
- reject("%s: can't read file." % (dsc_filename))
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (dsc_filename, line))
- except InvalidDscError, line:
- reject("%s: syntax error on line %s." % (dsc_filename, line))
- except ChangesUnicodeError:
- reject("%s: dsc file not proper utf-8." % (dsc_filename))
-
- # Build up the file list of files mentioned by the .dsc
- try:
- dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1))
- except NoFilesFieldError:
- reject("%s: no Files: field." % (dsc_filename))
- return 0
- except UnknownFormatError, format:
- reject("%s: unknown format '%s'." % (dsc_filename, format))
- return 0
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (dsc_filename, line))
- return 0
-
- # Enforce mandatory fields
- for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
- if not dsc.has_key(i):
- reject("%s: missing mandatory field `%s'." % (dsc_filename, i))
- return 0
-
- # Validate the source and version fields
- if not re_valid_pkg_name.match(dsc["source"]):
- reject("%s: invalid source name '%s'." % (dsc_filename, dsc["source"]))
- if not re_valid_version.match(dsc["version"]):
- reject("%s: invalid version number '%s'." % (dsc_filename, dsc["version"]))
-
- # Bumping the version number of the .dsc breaks extraction by stable's
- # dpkg-source. So let's not do that...
- if dsc["format"] != "1.0":
- reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename))
-
- # Validate the Maintainer field
- try:
- utils.fix_maintainer (dsc["maintainer"])
- except ParseMaintError, msg:
- reject("%s: Maintainer field ('%s') failed to parse: %s" \
- % (dsc_filename, dsc["maintainer"], msg))
-
- # Validate the build-depends field(s)
- for field_name in [ "build-depends", "build-depends-indep" ]:
- field = dsc.get(field_name)
- if field:
- # Check for broken dpkg-dev lossage...
- if field.startswith("ARRAY"):
- reject("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % (dsc_filename, field_name.title()))
-
- # Have apt try to parse them...
- try:
- apt_pkg.ParseSrcDepends(field)
- except:
- reject("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
- pass
-
- # Ensure the version number in the .dsc matches the version number in the .changes
- epochless_dsc_version = re_no_epoch.sub('', dsc["version"])
- changes_version = files[dsc_filename]["version"]
- if epochless_dsc_version != files[dsc_filename]["version"]:
- reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
-
- # Ensure there is a .tar.gz in the .dsc file
- has_tar = 0
- for f in dsc_files.keys():
- m = re_issource.match(f)
- if not m:
- reject("%s: %s in Files field not recognised as source." % (dsc_filename, f))
- continue
- ftype = m.group(3)
- if ftype == "orig.tar.gz" or ftype == "tar.gz":
- has_tar = 1
- if not has_tar:
- reject("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
-
- # Ensure source is newer than existing source in target suites
- reject(Upload.check_source_against_db(dsc_filename),"")
-
- (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(dsc_filename)
- reject(reject_msg, "")
- if is_in_incoming:
- if not Options["No-Action"]:
- copy_to_holding(is_in_incoming)
- orig_tar_gz = os.path.basename(is_in_incoming)
- files[orig_tar_gz] = {}
- files[orig_tar_gz]["size"] = os.stat(orig_tar_gz)[stat.ST_SIZE]
- files[orig_tar_gz]["md5sum"] = dsc_files[orig_tar_gz]["md5sum"]
- files[orig_tar_gz]["sha1sum"] = dsc_files[orig_tar_gz]["sha1sum"]
- files[orig_tar_gz]["sha256sum"] = dsc_files[orig_tar_gz]["sha256sum"]
- files[orig_tar_gz]["section"] = files[dsc_filename]["section"]
- files[orig_tar_gz]["priority"] = files[dsc_filename]["priority"]
- files[orig_tar_gz]["component"] = files[dsc_filename]["component"]
- files[orig_tar_gz]["type"] = "orig.tar.gz"
- reprocess = 2
-
- return 1
-
-################################################################################
-
-def get_changelog_versions(source_dir):
- """Extracts a the source package and (optionally) grabs the
- version history out of debian/changelog for the BTS."""
-
- # Find the .dsc (again)
- dsc_filename = None
- for f in files.keys():
- if files[f]["type"] == "dsc":
- dsc_filename = f
-
- # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
- if not dsc_filename:
- return
-
- # Create a symlink mirror of the source files in our temporary directory
- for f in files.keys():
- m = re_issource.match(f)
- if m:
- src = os.path.join(source_dir, f)
- # If a file is missing for whatever reason, give up.
- if not os.path.exists(src):
- return
- ftype = m.group(3)
- if ftype == "orig.tar.gz" and pkg.orig_tar_gz:
- continue
- dest = os.path.join(os.getcwd(), f)
- os.symlink(src, dest)
-
- # If the orig.tar.gz is not a part of the upload, create a symlink to the
- # existing copy.
- if pkg.orig_tar_gz:
- dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz))
- os.symlink(pkg.orig_tar_gz, dest)
-
- # Extract the source
- cmd = "dpkg-source -sn -x %s" % (dsc_filename)
- (result, output) = commands.getstatusoutput(cmd)
- if (result != 0):
- reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
- reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
- return
-
- if not Cnf.Find("Dir::Queue::BTSVersionTrack"):
- return
-
- # Get the upstream version
- upstr_version = re_no_epoch.sub('', dsc["version"])
- if re_strip_revision.search(upstr_version):
- upstr_version = re_strip_revision.sub('', upstr_version)
-
- # Ensure the changelog file exists
- changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version)
- if not os.path.exists(changelog_filename):
- reject("%s: debian/changelog not found in extracted source." % (dsc_filename))
- return
-
- # Parse the changelog
- dsc["bts changelog"] = ""
- changelog_file = utils.open_file(changelog_filename)
- for line in changelog_file.readlines():
- m = re_changelog_versions.match(line)
- if m:
- dsc["bts changelog"] += line
- changelog_file.close()
-
- # Check we found at least one revision in the changelog
- if not dsc["bts changelog"]:
- reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
-
-########################################
-
-def check_source():
- # Bail out if:
- # a) there's no source
- # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
- # or c) the orig.tar.gz is MIA
- if not changes["architecture"].has_key("source") or reprocess == 2 \
- or pkg.orig_tar_gz == -1:
- return
-
- tmpdir = utils.temp_dirname()
-
- # Move into the temporary directory
- cwd = os.getcwd()
- os.chdir(tmpdir)
-
- # Get the changelog version history
- get_changelog_versions(cwd)
-
- # Move back and cleanup the temporary tree
- os.chdir(cwd)
- try:
- shutil.rmtree(tmpdir)
- except OSError, e:
- if errno.errorcode[e.errno] != 'EACCES':
- utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"]))
-
- reject("%s: source tree could not be cleanly removed." % (dsc["source"]))
- # We probably have u-r or u-w directories so chmod everything
- # and try again.
- cmd = "chmod -R u+rwx %s" % (tmpdir)
- result = os.system(cmd)
- if result != 0:
- utils.fubar("'%s' failed with result %s." % (cmd, result))
- shutil.rmtree(tmpdir)
- except:
- utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"]))
-
-################################################################################
-
-# FIXME: should be a debian specific check called from a hook
-
-def check_urgency ():
- if changes["architecture"].has_key("source"):
- if not changes.has_key("urgency"):
- changes["urgency"] = Cnf["Urgency::Default"]
- # Urgency may be followed by space & comment (policy 5.6.17)
- changes["urgency"] = changes["urgency"].split(" ")[0].lower();
- if changes["urgency"] not in Cnf.ValueList("Urgency::Valid"):
- reject("%s is not a valid urgency; it will be treated as %s by testing." % (changes["urgency"], Cnf["Urgency::Default"]), "Warning: ")
- changes["urgency"] = Cnf["Urgency::Default"]
-
-################################################################################
-
-def check_hashes ():
- utils.check_hash(".changes", files, "md5", apt_pkg.md5sum)
- utils.check_size(".changes", files)
- utils.check_hash(".dsc", dsc_files, "md5", apt_pkg.md5sum)
- utils.check_size(".dsc", dsc_files)
-
- # This is stupid API, but it'll have to do for now until
- # we actually have proper abstraction
- for m in utils.ensure_hashes(changes, dsc, files, dsc_files):
- reject(m)
-
-################################################################################
-
-# Sanity check the time stamps of files inside debs.
-# [Files in the near future cause ugly warnings and extreme time
-# travel can cause errors on extraction]
-
-def check_timestamps():
- class Tar:
- def __init__(self, future_cutoff, past_cutoff):
- self.reset()
- self.future_cutoff = future_cutoff
- self.past_cutoff = past_cutoff
-
- def reset(self):
- self.future_files = {}
- self.ancient_files = {}
-
- def callback(self, Kind,Name,Link,Mode,UID,GID,Size,MTime,Major,Minor):
- if MTime > self.future_cutoff:
- self.future_files[Name] = MTime
- if MTime < self.past_cutoff:
- self.ancient_files[Name] = MTime
- ####
-
- future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
- past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
- tar = Tar(future_cutoff, past_cutoff)
- for filename in files.keys():
- if files[filename]["type"] == "deb":
- tar.reset()
- try:
- deb_file = utils.open_file(filename)
- apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz")
- deb_file.seek(0)
- try:
- apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz")
- except SystemError, e:
- # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
- if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
- raise
- deb_file.seek(0)
- apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
- deb_file.close()
- #
- future_files = tar.future_files.keys()
- if future_files:
- num_future_files = len(future_files)
- future_file = future_files[0]
- future_date = tar.future_files[future_file]
- reject("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
- % (filename, num_future_files, future_file,
- time.ctime(future_date)))
- #
- ancient_files = tar.ancient_files.keys()
- if ancient_files:
- num_ancient_files = len(ancient_files)
- ancient_file = ancient_files[0]
- ancient_date = tar.ancient_files[ancient_file]
- reject("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
- % (filename, num_ancient_files, ancient_file,
- time.ctime(ancient_date)))
- except:
- reject("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
-
-################################################################################
-
-def lookup_uid_from_fingerprint(fpr):
- """
- Return the uid,name,isdm for a given gpg fingerprint
-
- @type fpr: string
- @param fpr: a 40 byte GPG fingerprint
-
- @return: (uid, name, isdm)
- """
- cursor = DBConn().cursor()
- cursor.execute( "SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
- qs = cursor.fetchone()
- if qs:
- return qs
- else:
- return (None, None, False)
-
-def check_signed_by_key():
- """Ensure the .changes is signed by an authorized uploader."""
-
- (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(changes["fingerprint"])
- if uid_name == None:
- uid_name = ""
-
- # match claimed name with actual name:
- if uid is None:
- # This is fundamentally broken but need us to refactor how we get
- # the UIDs/Fingerprints in order for us to fix it properly
- uid, uid_email = changes["fingerprint"], uid
- may_nmu, may_sponsor = 1, 1
- # XXX by default new dds don't have a fingerprint/uid in the db atm,
- # and can't get one in there if we don't allow nmu/sponsorship
- elif is_dm is False:
- # If is_dm is False, we allow full upload rights
- uid_email = "%s@debian.org" % (uid)
- may_nmu, may_sponsor = 1, 1
- else:
- # Assume limited upload rights unless we've discovered otherwise
- uid_email = uid
- may_nmu, may_sponsor = 0, 0
-
-
- if uid_email in [changes["maintaineremail"], changes["changedbyemail"]]:
- sponsored = 0
- elif uid_name in [changes["maintainername"], changes["changedbyname"]]:
- sponsored = 0
- if uid_name == "": sponsored = 1
- else:
- sponsored = 1
- if ("source" in changes["architecture"] and
- uid_email and utils.is_email_alias(uid_email)):
- sponsor_addresses = utils.gpg_get_key_addresses(changes["fingerprint"])
- if (changes["maintaineremail"] not in sponsor_addresses and
- changes["changedbyemail"] not in sponsor_addresses):
- changes["sponsoremail"] = uid_email
-
- if sponsored and not may_sponsor:
- reject("%s is not authorised to sponsor uploads" % (uid))
-
- cursor = DBConn().cursor()
- if not sponsored and not may_nmu:
- source_ids = []
- cursor.execute( "SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = %(source)s AND s.dm_upload_allowed = 'yes'", changes )
-
- highest_sid, highest_version = None, None
-
- should_reject = True
- while True:
- si = cursor.fetchone()
- if not si:
- break
-
- if highest_version == None or apt_pkg.VersionCompare(si[1], highest_version) == 1:
- highest_sid = si[0]
- highest_version = si[1]
-
- if highest_sid == None:
- reject("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % changes["source"])
- else:
-
- cursor.execute("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT su.maintainer FROM src_uploaders su JOIN source s ON (s.id = su.source) WHERE su.source = %s)" % (highest_sid))
-
- while True:
- m = cursor.fetchone()
- if not m:
- break
-
- (rfc822, rfc2047, name, email) = utils.fix_maintainer(m[0])
- if email == uid_email or name == uid_name:
- should_reject=False
- break
-
- if should_reject == True:
- reject("%s is not in Maintainer or Uploaders of source package %s" % (uid, changes["source"]))
-
- for b in changes["binary"].keys():
- for suite in changes["distribution"].keys():
- suite_id = DBConn().get_suite_id(suite)
-
- cursor.execute("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = %(package)s AND ba.suite = %(suite)s" , {'package':b, 'suite':suite_id} )
- while True:
- s = cursor.fetchone()
- if not s:
- break
-
- if s[0] != changes["source"]:
- reject("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
-
- for f in files.keys():
- if files[f].has_key("byhand"):
- reject("%s may not upload BYHAND file %s" % (uid, f))
- if files[f].has_key("new"):
- reject("%s may not upload NEW file %s" % (uid, f))
-
-
-################################################################################
-################################################################################
-
-# If any file of an upload has a recent mtime then chances are good
-# the file is still being uploaded.
-
-def upload_too_new():
- too_new = 0
- # Move back to the original directory to get accurate time stamps
- cwd = os.getcwd()
- os.chdir(pkg.directory)
- file_list = pkg.files.keys()
- file_list.extend(pkg.dsc_files.keys())
- file_list.append(pkg.changes_file)
- for f in file_list:
- try:
- last_modified = time.time()-os.path.getmtime(f)
- if last_modified < int(Cnf["Dinstall::SkipTime"]):
- too_new = 1
- break
- except:
- pass
- os.chdir(cwd)
- return too_new
-
-################################################################################
-
def action ():
# changes["distribution"] may not exist in corner cases
# (e.g. unreadable changes files)
- if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
- changes["distribution"] = {}
+ if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType):
+ u.pkg.changes["distribution"] = {}
- (summary, short_summary) = Upload.build_summaries()
+ (summary, short_summary) = u.build_summaries()
# q-unapproved hax0ring
queue_info = {
"Autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand },
"Byhand" : { "is": is_byhand, "process": do_byhand },
"OldStableUpdate" : { "is": is_oldstableupdate,
- "process": do_oldstableupdate },
+ "process": do_oldstableupdate },
"StableUpdate" : { "is": is_stableupdate, "process": do_stableupdate },
"Unembargo" : { "is": is_unembargo, "process": queue_unembargo },
"Embargo" : { "is": is_embargo, "process": queue_embargo },
}
+
queues = [ "New", "Autobyhand", "Byhand" ]
- if Cnf.FindB("Dinstall::SecurityQueueHandling"):
+ if cnf.FindB("Dinstall::SecurityQueueHandling"):
queues += [ "Unembargo", "Embargo" ]
else:
queues += [ "OldStableUpdate", "StableUpdate" ]
queuekey = ''
- if reject_message.find("Rejected") != -1:
- if upload_too_new():
- print "SKIP (too new)\n" + reject_message,
+ pi = u.package_info()
+
+ if len(u.rejects) > 0:
+ if u.upload_too_new():
+ print "SKIP (too new)\n" + pi,
prompt = "[S]kip, Quit ?"
else:
- print "REJECT\n" + reject_message,
+ print "REJECT\n" + pi
prompt = "[R]eject, Skip, Quit ?"
if Options["Automatic"]:
answer = 'R'
else:
qu = None
for q in queues:
- if queue_info[q]["is"]():
+ if queue_info[q]["is"](u):
qu = q
break
if qu:
- print "%s for %s\n%s%s" % (
- qu.upper(), ", ".join(changes["distribution"].keys()),
- reject_message, summary),
+ print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary)
queuekey = qu[0].upper()
if queuekey in "RQSA":
queuekey = "D"
if Options["Automatic"]:
answer = queuekey
else:
- print "ACCEPT\n" + reject_message + summary,
+ print "ACCEPT\n" + pi + summary,
prompt = "[A]ccept, Skip, Quit ?"
if Options["Automatic"]:
answer = 'A'
answer = answer[:1].upper()
if answer == 'R':
- os.chdir (pkg.directory)
- Upload.do_reject(0, reject_message)
+ os.chdir(u.pkg.directory)
+ u.do_reject(0, pi)
elif answer == 'A':
- accept(summary, short_summary)
- remove_from_unchecked()
+ u.accept(summary, short_summary)
+ u.check_override()
+ u.remove()
elif answer == queuekey:
- queue_info[qu]["process"](summary, short_summary)
- remove_from_unchecked()
+ queue_info[qu]["process"](u, summary, short_summary)
+ u.remove()
elif answer == 'Q':
sys.exit(0)
-def remove_from_unchecked():
- os.chdir (pkg.directory)
- for f in files.keys():
- os.unlink(f)
- os.unlink(pkg.changes_file)
-
################################################################################
-def accept (summary, short_summary):
- Upload.accept(summary, short_summary)
- Upload.check_override()
+def package_to_suite(u, suite):
+ if not u.pkg.changes["distribution"].has_key(suite):
+ return False
-################################################################################
+ ret = True
-def move_to_dir (dest, perms=0660, changesperms=0664):
- utils.move (pkg.changes_file, dest, perms=changesperms)
- file_keys = files.keys()
- for f in file_keys:
- utils.move (f, dest, perms=perms)
+ if not u.pkg.changes["architecture"].has_key("source"):
+ s = DBConn().session()
+ q = s.query(SrcAssociation.sa_id)
+ q = q.join(Suite).filter_by(suite_name=suite)
+ q = q.join(DBSource).filter_by(source=u.pkg.changes['source'])
+ q = q.filter_by(version=u.pkg.changes['version']).limit(1)
-################################################################################
+ if q.count() < 1:
+ ret = False
-def is_unembargo ():
- cursor = DBConn().cursor()
- cursor.execute( "SELECT package FROM disembargo WHERE package = %(source)s AND version = %(version)s", changes )
- if cursor.fetchone():
- return 1
+ s.close()
- oldcwd = os.getcwd()
- os.chdir(Cnf["Dir::Queue::Disembargo"])
- disdir = os.getcwd()
- os.chdir(oldcwd)
-
- if pkg.directory == disdir:
- if changes["architecture"].has_key("source"):
- if Options["No-Action"]: return 1
+ return ret
- cursor.execute( "INSERT INTO disembargo (package, version) VALUES ('%(package)s', '%(version)s')",
- changes )
- cursor.execute( "COMMIT" )
- return 1
+def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, announce=None):
+ cnf = Config()
+ dir = cnf["Dir::Queue::%s" % queue]
- return 0
+ print "Moving to %s holding area" % queue.upper()
+ Logger.log(["Moving to %s" % queue, u.pkg.changes_file])
-def queue_unembargo (summary, short_summary):
- print "Moving to UNEMBARGOED holding area."
- Logger.log(["Moving to unembargoed", pkg.changes_file])
-
- Upload.dump_vars(Cnf["Dir::Queue::Unembargoed"])
- move_to_dir(Cnf["Dir::Queue::Unembargoed"])
- Upload.queue_build("unembargoed", Cnf["Dir::Queue::Unembargoed"])
+ u.pkg.write_dot_dak(dir)
+ u.move_to_dir(dir, perms=perms)
+ if build:
+ get_queue(queue.lower()).autobuild_upload(u.pkg, dir)
# Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
-
- # Send accept mail, announce to lists, close bugs and check for
- # override disparities
- if not Cnf["Dinstall::Options::No-Mail"]:
- Upload.Subst["__SUITE__"] = ""
- mail_message = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
+ u.check_override()
+
+ # Send accept mail, announce to lists and close bugs
+ if announce and not cnf["Dinstall::Options::No-Mail"]:
+ template = os.path.join(cnf["Dir::Templates"], announce)
+ u.update_subst()
+ u.Subst["__SUITE__"] = ""
+ mail_message = utils.TemplateSubst(u.Subst, template)
utils.send_mail(mail_message)
- Upload.announce(short_summary, 1)
+ u.announce(short_summary, True)
################################################################################
-def is_embargo ():
- # if embargoed queues are enabled always embargo
- return 1
+def is_unembargo(u):
+ session = DBConn().session()
+ cnf = Config()
-def queue_embargo (summary, short_summary):
- print "Moving to EMBARGOED holding area."
- Logger.log(["Moving to embargoed", pkg.changes_file])
+ q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes)
+ if q.rowcount > 0:
+ session.close()
+ return True
- Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
- move_to_dir(Cnf["Dir::Queue::Embargoed"])
- Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
+ oldcwd = os.getcwd()
+ os.chdir(cnf["Dir::Queue::Disembargo"])
+ disdir = os.getcwd()
+ os.chdir(oldcwd)
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
-
- # Send accept mail, announce to lists, close bugs and check for
- # override disparities
- if not Cnf["Dinstall::Options::No-Mail"]:
- Upload.Subst["__SUITE__"] = ""
- mail_message = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
- utils.send_mail(mail_message)
- Upload.announce(short_summary, 1)
+ ret = False
-################################################################################
+ if u.pkg.directory == disdir:
+ if u.pkg.changes["architecture"].has_key("source"):
+ if not Options["No-Action"]:
+ session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes)
+ session.commit()
-def is_stableupdate ():
- if not changes["distribution"].has_key("proposed-updates"):
- return 0
+ ret = True
- if not changes["architecture"].has_key("source"):
- pusuite = DBConn().get_suite_id("proposed-updates")
- cursor = DBConn().cursor()
- cursor.execute( """SELECT 1 FROM source s
- JOIN src_associations sa ON (s.id = sa.source)
- WHERE s.source = %(source)s
- AND s.version = %(version)s
- AND sa.suite = %(suite)s""",
- {'source' : changes['source'],
- 'version' : changes['version'],
- 'suite' : pusuite})
+ session.close()
- if cursor.fetchone():
- # source is already in proposed-updates so no need to hold
- return 0
+ return ret
- return 1
+def queue_unembargo(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "Unembargoed",
+ perms=0660, build=True, announce='process-unchecked.accepted')
-def do_stableupdate (summary, short_summary):
- print "Moving to PROPOSED-UPDATES holding area."
- Logger.log(["Moving to proposed-updates", pkg.changes_file])
+################################################################################
- Upload.dump_vars(Cnf["Dir::Queue::ProposedUpdates"])
- move_to_dir(Cnf["Dir::Queue::ProposedUpdates"], perms=0664)
+def is_embargo(u):
+ # if embargoed queues are enabled always embargo
+ return True
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
+def queue_embargo(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "Unembargoed",
+ perms=0660, build=True, announce='process-unchecked.accepted')
################################################################################
-def is_oldstableupdate ():
- if not changes["distribution"].has_key("oldstable-proposed-updates"):
- return 0
-
- if not changes["architecture"].has_key("source"):
- pusuite = DBConn().get_suite_id("oldstable-proposed-updates")
- cursor = DBConn().cursor()
- cursor.execute( """SELECT 1 FROM source s
- JOIN src_associations sa ON (s.id = sa.source)
- WHERE s.source = %(source)s
- AND s.version = %(version)s
- AND sa.suite = %(suite)s""",
- {'source' : changes['source'],
- 'version' : changes['version'],
- 'suite' : pusuite})
- if cursor.fetchone():
- return 0
-
- return 1
-
-def do_oldstableupdate (summary, short_summary):
- print "Moving to OLDSTABLE-PROPOSED-UPDATES holding area."
- Logger.log(["Moving to oldstable-proposed-updates", pkg.changes_file])
-
- Upload.dump_vars(Cnf["Dir::Queue::OldProposedUpdates"])
- move_to_dir(Cnf["Dir::Queue::OldProposedUpdates"], perms=0664)
+def is_stableupdate(u):
+ return package_to_suite(u, 'proposed-updates')
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
+def do_stableupdate(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "ProposedUpdates",
+ perms=0664, build=False, announce=None)
################################################################################
-def is_autobyhand ():
+def is_oldstableupdate(u):
+ return package_to_suite(u, 'oldstable-proposed-updates')
+
+def do_oldstableupdate(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "OldProposedUpdates",
+ perms=0664, build=False, announce=None)
+
+################################################################################
+
+def is_autobyhand(u):
+ cnf = Config()
+
all_auto = 1
any_auto = 0
- for f in files.keys():
- if files[f].has_key("byhand"):
+ for f in u.pkg.files.keys():
+ if u.pkg.files[f].has_key("byhand"):
any_auto = 1
# filename is of form "PKG_VER_ARCH.EXT" where PKG, VER and ARCH
continue
(pckg, ver, archext) = f.split("_", 2)
- if archext.count(".") < 1 or changes["version"] != ver:
+ if archext.count(".") < 1 or u.pkg.changes["version"] != ver:
all_auto = 0
continue
- ABH = Cnf.SubTree("AutomaticByHandPackages")
+ ABH = cnf.SubTree("AutomaticByHandPackages")
if not ABH.has_key(pckg) or \
- ABH["%s::Source" % (pckg)] != changes["source"]:
- print "not match %s %s" % (pckg, changes["source"])
+ ABH["%s::Source" % (pckg)] != u.pkg.changes["source"]:
+ print "not match %s %s" % (pckg, u.pkg.changes["source"])
all_auto = 0
continue
(arch, ext) = archext.split(".", 1)
- if arch not in changes["architecture"]:
+ if arch not in u.pkg.changes["architecture"]:
all_auto = 0
continue
- files[f]["byhand-arch"] = arch
- files[f]["byhand-script"] = ABH["%s::Script" % (pckg)]
+ u.pkg.files[f]["byhand-arch"] = arch
+ u.pkg.files[f]["byhand-script"] = ABH["%s::Script" % (pckg)]
return any_auto and all_auto
-def do_autobyhand (summary, short_summary):
+def do_autobyhand(u, summary, short_summary):
print "Attempting AUTOBYHAND."
- byhandleft = 0
- for f in files.keys():
+ byhandleft = True
+ for f, entry in u.pkg.files.items():
byhandfile = f
- if not files[f].has_key("byhand"):
+
+ if not entry.has_key("byhand"):
continue
- if not files[f].has_key("byhand-script"):
- byhandleft = 1
+
+ if not entry.has_key("byhand-script"):
+ byhandleft = True
continue
os.system("ls -l %s" % byhandfile)
+
result = os.system("%s %s %s %s %s" % (
- files[f]["byhand-script"], byhandfile,
- changes["version"], files[f]["byhand-arch"],
- os.path.abspath(pkg.changes_file)))
+ entry["byhand-script"],
+ byhandfile,
+ u.pkg.changes["version"],
+ entry["byhand-arch"],
+ os.path.abspath(u.pkg.changes_file)))
+
if result == 0:
os.unlink(byhandfile)
- del files[f]
+ del entry
else:
print "Error processing %s, left as byhand." % (f)
- byhandleft = 1
+ byhandleft = True
if byhandleft:
- do_byhand(summary, short_summary)
+ do_byhand(u, summary, short_summary)
else:
- accept(summary, short_summary)
+ u.accept(summary, short_summary)
+ u.check_override()
+ # XXX: We seem to be missing a u.remove() here
+ # This might explain why we get byhand leftovers in unchecked - mhy
################################################################################
-def is_byhand ():
- for f in files.keys():
- if files[f].has_key("byhand"):
- return 1
- return 0
-
-def do_byhand (summary, short_summary):
- print "Moving to BYHAND holding area."
- Logger.log(["Moving to byhand", pkg.changes_file])
+def is_byhand(u):
+ for f in u.pkg.files.keys():
+ if u.pkg.files[f].has_key("byhand"):
+ return True
+ return False
- Upload.dump_vars(Cnf["Dir::Queue::Byhand"])
- move_to_dir(Cnf["Dir::Queue::Byhand"])
-
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
+def do_byhand(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "Byhand",
+ perms=0660, build=False, announce=None)
################################################################################
-def is_new ():
- for f in files.keys():
- if files[f].has_key("new"):
- return 1
- return 0
+def is_new(u):
+ for f in u.pkg.files.keys():
+ if u.pkg.files[f].has_key("new"):
+ return True
+ return False
-def acknowledge_new (summary, short_summary):
- Subst = Upload.Subst
+def acknowledge_new(u, summary, short_summary):
+ cnf = Config()
print "Moving to NEW holding area."
- Logger.log(["Moving to new", pkg.changes_file])
+ Logger.log(["Moving to new", u.pkg.changes_file])
- Upload.dump_vars(Cnf["Dir::Queue::New"])
- move_to_dir(Cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
+ u.pkg.write_dot_dak(cnf["Dir::Queue::New"])
+ u.move_to_dir(cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
if not Options["No-Mail"]:
print "Sending new ack."
- Subst["__SUMMARY__"] = summary
- new_ack_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.new")
+ template = os.path.join(cnf["Dir::Templates"], 'process-unchecked.new')
+ u.Subst["__SUMMARY__"] = summary
+ new_ack_message = utils.TemplateSubst(u.Subst, template)
utils.send_mail(new_ack_message)
################################################################################
# we force the .orig.tar.gz into the .changes structure and reprocess
# the .changes file.
-def process_it (changes_file):
- global reprocess, reject_message
+def process_it(changes_file):
+ global Logger
+
+ cnf = Config()
+
+ holding = Holding()
+
+ u = Upload()
+ u.pkg.changes_file = changes_file
+ u.pkg.directory = os.getcwd()
+ u.logger = Logger
+ origchanges = os.path.join(u.pkg.directory, u.pkg.changes_file)
- # Reset some globals
- reprocess = 1
- Upload.init_vars()
# Some defaults in case we can't fully process the .changes file
- changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]
- changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]
- reject_message = ""
+ u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
+ u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
- # Absolutize the filename to avoid the requirement of being in the
- # same directory as the .changes file.
- pkg.changes_file = os.path.abspath(changes_file)
+ # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
+ bcc = "X-DAK: dak process-unchecked\nX-Katie: $Revision: 1.65 $"
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ else:
+ u.Subst["__BCC__"] = bcc
# Remember where we are so we can come back after cd-ing into the
- # holding directory.
- pkg.directory = os.getcwd()
+ # holding directory. TODO: Fix this stupid hack
+ u.prevdir = os.getcwd()
+
+ # TODO: Figure out something better for this (or whether it's even
+ # necessary - it seems to have been for use when we were
+ # still doing the is_unchecked check; reprocess = 2)
+ u.reprocess = 1
try:
# If this is the Real Thing(tm), copy things into a private
# holding directory first to avoid replacable file races.
if not Options["No-Action"]:
- os.chdir(Cnf["Dir::Queue::Holding"])
- copy_to_holding(pkg.changes_file)
+ os.chdir(cnf["Dir::Queue::Holding"])
+
+ # Absolutize the filename to avoid the requirement of being in the
+ # same directory as the .changes file.
+ holding.copy_to_holding(origchanges)
+
# Relativize the filename so we use the copy in holding
# rather than the original...
- pkg.changes_file = os.path.basename(pkg.changes_file)
- changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject)
- if changes["fingerprint"]:
- valid_changes_p = check_changes()
+ changespath = os.path.basename(u.pkg.changes_file)
+
+ (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changespath)
+
+ if u.pkg.changes["fingerprint"]:
+ valid_changes_p = u.load_changes(changespath)
else:
- valid_changes_p = 0
+ valid_changes_p = False
+ u.rejects.extend(rejects)
+
if valid_changes_p:
- while reprocess:
- check_distributions()
- check_files()
- valid_dsc_p = check_dsc()
+ while u.reprocess:
+ u.check_distributions()
+ u.check_files(not Options["No-Action"])
+ valid_dsc_p = u.check_dsc(not Options["No-Action"])
if valid_dsc_p:
- check_source()
- check_hashes()
- check_urgency()
- check_timestamps()
- check_signed_by_key()
- Upload.update_subst(reject_message)
- action()
+ u.check_source()
+ u.check_hashes()
+ u.check_urgency()
+ u.check_timestamps()
+ u.check_signed_by_key()
+
+ action(u)
+
except SystemExit:
raise
+
except:
print "ERROR"
traceback.print_exc(file=sys.stderr)
- pass
# Restore previous WD
- os.chdir(pkg.directory)
+ os.chdir(u.prevdir)
###############################################################################
def main():
- global Cnf, Options, Logger
+ global Options, Logger
+ cnf = Config()
changes_files = init()
# -n/--dry-run invalidates some other options which would involve things happening
if Options["No-Action"]:
Options["Automatic"] = ""
+ # Initialize our Holding singleton
+ holding = Holding()
+
# Ensure all the arguments we were given are .changes files
for f in changes_files:
if not f.endswith(".changes"):
changes_files.remove(f)
if changes_files == []:
- if Cnf["Dinstall::Options::Directory"] == "":
+ if cnf["Dinstall::Options::Directory"] == "":
utils.fubar("Need at least one .changes file as an argument.")
else:
sys.exit(0)
# Check that we aren't going to clash with the daily cron job
-
- if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]:
+ if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
utils.fubar("Archive maintenance in progress. Try again later.")
# Obtain lock if not in no-action mode and initialize the log
-
if not Options["No-Action"]:
- lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+ lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError, e:
utils.fubar("Couldn't obtain lock; assuming another 'dak process-unchecked' is already running.")
else:
raise
- Logger = Upload.Logger = logging.Logger(Cnf, "process-unchecked")
-
- # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
- bcc = "X-DAK: dak process-unchecked\nX-Katie: $Revision: 1.65 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Upload.Subst["__BCC__"] = bcc
-
+ Logger = daklog.Logger(cnf, "process-unchecked")
# Sort the .changes files so that we process sourceful ones first
changes_files.sort(utils.changes_compare)
process_it (changes_file)
finally:
if not Options["No-Action"]:
- clean_holding()
+ holding.clean()
+
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
- accept_count = Upload.accept_count
- accept_bytes = Upload.accept_bytes
if accept_count:
sets = "set"
if accept_count > 1:
################################################################################
-import copy, glob, os, stat, sys, time
+from copy import copy
+import glob, os, stat, sys, time
import apt_pkg
import cgi
-from daklib import queue
-from daklib import database
+
from daklib import utils
+from daklib.dbconn import DBConn, has_new_comment
+from daklib.textutils import fix_maintainer
from daklib.dak_exceptions import *
Cnf = None
-Upload = None
direction = []
row_number = 0
-projectB = None
################################################################################
# Read in all the .changes files
for filename in changes_files:
try:
- Upload.pkg.changes_file = filename
- Upload.init_vars()
- Upload.update_vars()
- cache[filename] = copy.copy(Upload.pkg.changes)
+ c = Changes()
+ c.load_dot_dak(filename)
+ cache[filename] = copy(c.changes)
cache[filename]["filename"] = filename
except:
break
else:
if mtime < oldest:
oldest = mtime
- have_note += (database.has_new_comment(d["source"], d["version"]))
+ have_note += has_new_comment(d["source"], d["version"])
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
- utils.fix_maintainer (j["maintainer"])
+ fix_maintainer (j["maintainer"])
except ParseMaintError, msg:
print "Problems while parsing maintainer address\n"
maintainer["maintainername"] = "Unknown"
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
- utils.fix_maintainer (j["changed-by"])
+ fix_maintainer (j["changed-by"])
except ParseMaintError, msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
################################################################################
def main():
- global Cnf, Upload
+ global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Queue-Report::Options::Help"),
if Options["Help"]:
usage()
- Upload = queue.Upload(Cnf)
- projectB = Upload.projectB
-
if Cnf.has_key("Queue-Report::Options::New"):
header()
+ # Initialize db so we can get the NEW comments
+ dbconn = DBConn()
+
directories = [ ]
if Cnf.has_key("Queue-Report::Options::Directories"):
import commands
import os
-import pg
import re
import sys
import apt_pkg
import apt_inst
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import *
from daklib.regexes import re_strip_source_version, re_build_dep_arch
################################################################################
-Cnf = None
Options = None
-projectB = None
################################################################################
################################################################################
def reverse_depends_check(removals, suites, arches=None):
+ cnf = Config()
+
print "Checking reverse dependencies..."
- components = Cnf.ValueList("Suite::%s::Components" % suites[0])
+ components = cnf.ValueList("Suite::%s::Components" % suites[0])
dep_problem = 0
p2c = {}
all_broken = {}
sources = {}
virtual_packages = {}
for component in components:
- filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suites[0], component, architecture)
+ filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suites[0], component, architecture)
# apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
# Check source dependencies (Build-Depends and Build-Depends-Indep)
all_broken.clear()
for component in components:
- filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suites[0], component)
+ filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suites[0], component)
# apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
result, output = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
################################################################################
def main ():
- global Cnf, Options, projectB
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Rm::Options::Help"),
('a',"architecture","Rm::Options::Architecture", "HasArg"),
for i in [ "architecture", "binary-only", "carbon-copy", "component",
"done", "help", "no-action", "partial", "rdep-check", "reason",
"source-only" ]:
- if not Cnf.has_key("Rm::Options::%s" % (i)):
- Cnf["Rm::Options::%s" % (i)] = ""
- if not Cnf.has_key("Rm::Options::Suite"):
- Cnf["Rm::Options::Suite"] = "unstable"
+ if not cnf.has_key("Rm::Options::%s" % (i)):
+ cnf["Rm::Options::%s" % (i)] = ""
+ if not cnf.has_key("Rm::Options::Suite"):
+ cnf["Rm::Options::Suite"] = "unstable"
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Rm::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Rm::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# Sanity check options
if not arguments:
carbon_copy = []
for copy_to in utils.split_args(Options.get("Carbon-Copy")):
if copy_to.isdigit():
- carbon_copy.append(copy_to + "@" + Cnf["Dinstall::BugServer"])
+ carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
elif copy_to == 'package':
for package in arguments:
- carbon_copy.append(package + "@" + Cnf["Dinstall::PackagesServer"])
- if Cnf.has_key("Dinstall::TrackingServer"):
- carbon_copy.append(package + "@" + Cnf["Dinstall::TrackingServer"])
+ carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"])
+ if cnf.has_key("Dinstall::TrackingServer"):
+ carbon_copy.append(package + "@" + cnf["Dinstall::TrackingServer"])
elif '@' in copy_to:
carbon_copy.append(copy_to)
else:
suites_list = utils.join_with_commas_and(suites)
if not Options["No-Action"]:
for suite in suites:
- suite_id = database.get_suite_id(suite)
- if suite_id != -1:
- suite_ids_list.append(suite_id)
+ s = get_suite(suite, session=session)
+ if s is not None:
+ suite_ids_list.append(s.suite_id)
if suite == "stable":
print "**WARNING** About to remove from the stable suite!"
print "This should only be done just prior to a (point) release and not at"
# latter is a nasty mess, but very nice from a UI perspective so
# we try to support it.
+ # XXX: TODO: This all needs converting to use placeholders or the object
+ # API. It's an SQL injection dream at the moment
+
if Options["Binary-Only"]:
# Binary-only
- q = projectB.query("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
- for i in q.getresult():
+ q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
+ for i in q.fetchall():
to_remove.append(i)
else:
# Source-only
source_packages = {}
- q = projectB.query("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
- for i in q.getresult():
+ q = session.execute("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
+ for i in q.fetchall():
source_packages[i[2]] = i[:2]
to_remove.append(i[2:])
if not Options["Source-Only"]:
# Source + Binary
binary_packages = {}
# First get a list of binary package names we suspect are linked to the source
- q = projectB.query("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
- for i in q.getresult():
+ q = session.execute("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
+ for i in q.fetchall():
binary_packages[i[0]] = ""
# Then parse each .dsc that we found earlier to see what binary packages it thinks it produces
for i in source_packages.keys():
# source package and if so add it to the list of packages
# to be removed.
for package in binary_packages.keys():
- q = projectB.query("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package))
- for i in q.getresult():
+ q = session.execute("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package))
+ for i in q.fetchall():
filename = "/".join(i[:2])
control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(filename)))
source = control.Find("Source", control.Find("Package"))
date = commands.getoutput('date -R')
# Log first; if it all falls apart I want a record that we at least tried.
- logfile = utils.open_file(Cnf["Rm::LogFile"], 'a')
+ logfile = utils.open_file(cnf["Rm::LogFile"], 'a')
logfile.write("=========================================================================\n")
logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
# Do the actual deletion
print "Deleting...",
sys.stdout.flush()
- projectB.query("BEGIN WORK")
+
for i in to_remove:
package = i[0]
architecture = i[2]
package_id = i[3]
for suite_id in suite_ids_list:
if architecture == "source":
- projectB.query("DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id))
+ session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
#print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)
else:
- projectB.query("DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id))
+ session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
#print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)
# Delete from the override file
if not Options["Partial"]:
type_id = dsc_type_id
else:
type_id = deb_type_id
- projectB.query("DELETE FROM override WHERE package = '%s' AND type = %s AND suite = %s %s" % (package, type_id, suite_id, over_con_components))
- projectB.query("COMMIT WORK")
+ # TODO: Again, fix this properly to remove the remaining non-bind argument
+ session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
+ session.commit()
print "done."
# Send the bug closing messages
if Options["Done"]:
Subst = {}
- Subst["__RM_ADDRESS__"] = Cnf["Rm::MyEmailAddress"]
- Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
+ Subst["__RM_ADDRESS__"] = cnf["Rm::MyEmailAddress"]
+ Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
bcc = []
- if Cnf.Find("Dinstall::Bcc") != "":
- bcc.append(Cnf["Dinstall::Bcc"])
- if Cnf.Find("Rm::Bcc") != "":
- bcc.append(Cnf["Rm::Bcc"])
+ if cnf.Find("Dinstall::Bcc") != "":
+ bcc.append(cnf["Dinstall::Bcc"])
+ if cnf.Find("Rm::Bcc") != "":
+ bcc.append(cnf["Rm::Bcc"])
if bcc:
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
Subst["__SUITE_LIST__"] = suites_list
Subst["__SUMMARY__"] = summary
- Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
- Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
+ Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst["__WHOAMI__"] = whoami
whereami = utils.where_am_i()
- Archive = Cnf.SubTree("Archive::%s" % (whereami))
+ Archive = cnf.SubTree("Archive::%s" % (whereami))
Subst["__MASTER_ARCHIVE__"] = Archive["OriginServer"]
Subst["__PRIMARY_MIRROR__"] = Archive["PrimaryMirror"]
for bug in utils.split_args(Options["Done"]):
Subst["__BUG_NUMBER__"] = bug
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/rm.bug-close")
+ mail_message = utils.TemplateSubst(Subst,cnf["Dir::Templates"]+"/rm.bug-close")
utils.send_mail(mail_message)
logfile.write("=========================================================================\n")
import apt_pkg
import tempfile
from debian_bundle import deb822
-from daklib import database
-from daklib import queue
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_html_escaping, html_escaping
uploader = re.sub(r'^\s*(\S.*)\s+<.*>',r'\1',uploader)
if Cnf.has_key("Show-Deferred::LinkPath"):
isnew = 0
- suites = database.get_suites(achanges['source'],src=1)
+ suites = get_suites_source_in(achanges['source'])
if 'unstable' not in suites and 'experimental' not in suites:
isnew = 1
+
for b in achanges['binary'].split():
- suites = database.get_suites(b)
+ suites = get_suites_binary_in(b)
if 'unstable' not in suites and 'experimental' not in suites:
isnew = 1
+
if not isnew:
# we don't link .changes because we don't want other people to
# upload it with the existing signature.
sys.exit(exit_code)
def init():
- global Cnf, Options, Upload, projectB
+ global Cnf, Options
Cnf = utils.get_conf()
Arguments = [('h',"help","Show-Deferred::Options::Help"),
("p","link-path","Show-Deferred::LinkPath","HasArg"),
Options = Cnf.SubTree("Show-Deferred::Options")
if Options["help"]:
usage()
- Upload = queue.Upload(Cnf)
- projectB = Upload.projectB
+
+ # Initialise database connection
+ DBConn()
+
return args
def main():
################################################################################
-import copy, os, sys, time
+from copy import copy
+import os, sys, time
import apt_pkg
import examine_package
-from daklib import database
-from daklib import queue
+
+from daklib.queue import determine_new, check_valid
from daklib import utils
# Globals
Cnf = None
Options = None
-Upload = None
-projectB = None
sources = set()
def do_pkg(changes_file):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- files = Upload.pkg.files
- changes = Upload.pkg.changes
-
- changes["suite"] = copy.copy(changes["distribution"])
- distribution = changes["distribution"].keys()[0]
+ c = Changes()
+ c.load_dot_dak(changes_file)
+ files = c.files
+ changes = c.changes
+
+ c.changes["suite"] = copy(c.changes["distribution"])
+ distribution = c.changes["distribution"].keys()[0]
# Find out what's new
- new = queue.determine_new(changes, files, projectB, 0)
+ new = determine_new(c.changes, c.files, 0)
stdout_fd = sys.stdout
- htmlname = changes["source"] + "_" + changes["version"] + ".html"
+ htmlname = c.changes["source"] + "_" + c.changes["version"] + ".html"
sources.add(htmlname)
# do not generate html output if that source/version already has one.
if not os.path.exists(os.path.join(Cnf["Show-New::HTMLPath"],htmlname)):
filestoexamine = []
for pkg in new.keys():
for fn in new[pkg]["files"]:
- if ( files[fn].has_key("new") and not
- files[fn]["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2"] ):
+ if ( c.files[fn].has_key("new") and not
+ c.files[fn]["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2"] ):
filestoexamine.append(fn)
- html_header(changes["source"], filestoexamine)
+ html_header(c.changes["source"], filestoexamine)
- queue.check_valid(new)
- examine_package.display_changes( distribution, Upload.pkg.changes_file)
+ check_valid(new)
+ examine_package.display_changes( distribution, changes_file)
for fn in filter(lambda fn: fn.endswith(".dsc"), filestoexamine):
examine_package.check_dsc(distribution, fn)
################################################################################
def init():
- global Cnf, Options, Upload, projectB
+ global Cnf, Options
Cnf = utils.get_conf()
if Options["help"]:
usage()
- Upload = queue.Upload(Cnf)
-
- projectB = Upload.projectB
-
return changes_files
################################################################################
-import pg, sys
+import sys
import apt_pkg
+
from daklib import utils
-from daklib import database
+from daklib.dbconn import DBConn, get_suite_architectures, Suite, Architecture, \
+ BinAssociation
################################################################################
Cnf = None
-projectB = None
################################################################################
################################################################################
def per_arch_space_use():
- q = projectB.query("""
-SELECT a.arch_string as Architecture, sum(f.size)
+ session = DBConn().session()
+ q = session.execute("""
+SELECT a.arch_string as Architecture, sum(f.size) AS sum
FROM files f, binaries b, architecture a
WHERE a.id=b.architecture AND f.id=b.file
- GROUP BY a.arch_string""")
- print q
- q = projectB.query("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'")
- print q
+ GROUP BY a.arch_string ORDER BY sum""").fetchall()
+ for j in q:
+ print "%-15.15s %s" % (j[0], j[1])
+ print
+ q = session.execute("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'").fetchall()
+ print "%-15.15s %s" % ("Source", q[0][0])
################################################################################
suites = {}
suite_ids = {}
d = {}
+ session = DBConn().session()
# Build up suite mapping
- q = projectB.query("SELECT id, suite_name FROM suite")
- suite_ql = q.getresult()
- for i in suite_ql:
- (sid, name) = i
- suites[sid] = name
- suite_ids[name] = sid
+ for i in session.query(Suite).all():
+ suites[i.suite_id] = i.suite_name
+ suite_ids[i.suite_name] = i.suite_id
# Build up architecture mapping
- q = projectB.query("SELECT id, arch_string FROM architecture")
- for i in q.getresult():
- (aid, name) = i
- arches[aid] = name
- arch_ids[name] = aid
+ for i in session.query(Architecture).all():
+ arches[i.arch_id] = i.arch_string
+ arch_ids[i.arch_string] = i.arch_id
# Pre-create the dictionary
for suite_id in suites.keys():
d[suite_id] = {}
for arch_id in arches.keys():
d[suite_id][arch_id] = 0
# Get the raw data for binaries
- q = projectB.query("""
-SELECT ba.suite, b.architecture
- FROM binaries b, bin_associations ba
- WHERE b.id = ba.bin""")
# Simultate 'GROUP by suite, architecture' with a dictionary
- for i in q.getresult():
- (suite_id, arch_id) = i
+ # XXX: Why don't we just get the DB to do this?
+ for i in session.query(BinAssociation):
+ suite_id = i.suite_id
+ arch_id = i.binary.arch_id
d[suite_id][arch_id] = d[suite_id][arch_id] + 1
# Get the raw data for source
arch_id = arch_ids["source"]
- q = projectB.query("""
-SELECT suite, count(suite) FROM src_associations GROUP BY suite;""")
- for i in q.getresult():
+ for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').all():
(suite_id, count) = i
d[suite_id][arch_id] = d[suite_id][arch_id] + count
## Print the results
for suite in suite_list:
suite_id = suite_ids[suite]
suite_arches[suite_id] = {}
- for arch in database.get_suite_architectures(suite_id):
- suite_arches[suite_id][arch] = ""
+ for arch in get_suite_architectures(suite):
+ suite_arches[suite_id][arch.arch_string] = ""
suite_id_list.append(suite_id)
output_list = [ output_format(i) for i in suite_list ]
longest_suite = longest(output_list)
################################################################################
def main ():
- global Cnf, projectB
+ global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Stats::Options::Help")]
usage(1)
mode = args[0].lower()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
if mode == "arch-space":
per_arch_space_use()
elif mode == "pkg-nums":
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# Test utils.fix_maintainer()
+# Test textutils.fix_maintainer()
# Copyright (C) 2004, 2006 James Troup <james@nocrew.org>
# This program is free software; you can redistribute it and/or modify
sys.path.append(os.path.abspath('../../'))
-import utils
+import textutils
################################################################################
################################################################################
def check_valid(s, xa, xb, xc, xd):
- (a, b, c, d) = utils.fix_maintainer(s)
+ (a, b, c, d) = textutils.fix_maintainer(s)
if a != xa:
fail("rfc822_maint: %s (returned) != %s (expected [From: '%s']" % (a, xa, s))
if b != xb:
def check_invalid(s):
try:
- utils.fix_maintainer(s)
+ textutils.fix_maintainer(s)
fail("%s was parsed successfully but is expected to be invalid." % (s))
except utils.ParseMaintError, unused:
pass
################################################################################
import os
-import pg
import sys
import time
import errno
import tempfile
import pwd
import apt_pkg
-from daklib import database
+
+from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import TransitionsError
from daklib.regexes import re_broken_package
# Globals
Cnf = None #: Configuration, apt_pkg.Configuration
Options = None #: Parsed CommandLine arguments
-projectB = None #: database connection, pgobject
################################################################################
@attention: This function may run B{within sudo}
"""
- global Cnf, Options, projectB
+ global Cnf, Options
apt_pkg.init()
print "Non-dak user: %s" % username
Options["sudo"] = "y"
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ # Initialise DB connection
+ DBConn()
################################################################################
to_dump = 0
to_remove = []
info = {}
+
+ session = DBConn().session()
+
# Now look through all defined transitions
for trans in transitions:
t = transitions[trans]
source = t["source"]
expected = t["new"]
- # Will be None if nothing is in testing.
- current = database.get_suite_version(source, "testing")
+ # Will be an empty list if nothing is in testing.
+ sources = get_source_in_suite(source, "testing", session)
info[trans] = get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
print info[trans]
- if current == None:
+ if len(sources) < 1:
# No package in testing
print "Transition source %s not in testing, transition still ongoing." % (source)
else:
+ current = sources[0].version
compare = apt_pkg.VersionCompare(current, expected)
if compare < 0:
# This is still valid, the current version in database is older than
source = t["source"]
expected = t["new"]
- # Will be None if nothing is in testing.
- current = database.get_suite_version(source, "testing")
+ # Will be empty list if nothing is in testing.
+ sources = get_suite_version(source, "testing")
print get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
- if current == None:
+ if len(sources) < 1:
# No package in testing
print "Transition source %s not in testing, transition still ongoing." % (source)
else:
+ current = sources[0].version
compare = apt_pkg.VersionCompare(current, expected)
print "Apt compare says: %s" % (compare)
if compare < 0:
import apt_pkg
import time
import errno
-from daklib import database
+
from daklib import utils
from daklib.dak_exceptions import DBUpdateError
################################################################################
Cnf = None
-projectB = None
-required_database_schema = 13
+required_database_schema = 14
################################################################################
################################################################################
def get_db_rev(self):
- global projectB
-
# We keep database revision info the config table
# Try and access it
################################################################################
def init (self):
- global Cnf, projectB
+ global Cnf
Cnf = utils.get_conf()
arguments = [('h', "help", "Update-DB::Options::Help")]
import os
import sys
import shutil
-import tempfile
import tarfile
import commands
import traceback
import atexit
+
from debian_bundle import deb822
-from dbconn import DBConn
+
+from dbconn import *
from config import Config
-import logging
import utils
+################################################################################
+
+__all__ = []
+
+################################################################################
+
class Binary(object):
def __init__(self, filename, reject=None):
"""
self.tmpdir = None
self.chunks = None
self.wrapped_reject = reject
+ # Store rejects for later use
+ self.rejects = []
def reject(self, message):
"""
otherwise send it to stderr.
"""
print >> sys.stderr, message
+ self.rejects.append(message)
if self.wrapped_reject:
self.wrapped_reject(message)
return not rejected
- def scan_package(self, bootstrap_id=0, relaxed=False):
+ def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
"""
Unpack the .deb, do sanity checking, and gather info from it.
data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
if bootstrap_id:
- result = DBConn().insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
else:
pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
pkg = pkgs.next()
- result = DBConn().insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ result = insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
except:
traceback.print_exc()
os.chdir(cwd)
+__all__.append('Binary')
+
+def copy_temporary_contents(package, version, archname, deb, reject, session=None):
+ """
+ copy the previously stored contents from the temp table to the permanant one
+
+ during process-unchecked, the deb should have been scanned and the
+ contents stored in pending_content_associations
+ """
+
+ cnf = Config()
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ arch = get_architecture(archname, session=session)
+
+ # first see if contents exist:
+ in_pcaq = """SELECT 1 FROM pending_content_associations
+ WHERE package=:package
+ AND version=:version
+ AND architecture=:archid LIMIT 1"""
+
+ vals = {'package': package,
+ 'version': version,
+ 'archid': arch.arch_id}
+
+ exists = None
+ check = session.execute(in_pcaq, vals)
+
+ if check.rowcount > 0:
+ # This should NOT happen. We should have added contents
+ # during process-unchecked. if it did, log an error, and send
+ # an email.
+ subst = {
+ "__PACKAGE__": package,
+ "__VERSION__": version,
+ "__ARCH__": arch,
+ "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
+ "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
+
+ message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
+ utils.send_mail(message)
+
+ exists = Binary(deb, reject).scan_package()
+
+ if exists:
+ sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
+ SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations
+ WHERE package=:package AND version=:version AND architecture=:archid"""
+ session.execute(sql, vals)
+
+ sql = """DELETE from pending_content_associations
+ WHERE package=:package AND version=:version AND architecture=:archid"""
+ session.execute(sql, vals)
+ session.commit()
+
+ if privatetrans:
+ session.close()
+
+ return exists
+__all__.append('copy_temporary_contents')
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Changes class for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+import os
+import stat
+from cPickle import Unpickler, Pickler
+from errno import EPERM
+
+from apt_inst import debExtractControl
+from apt_pkg import ParseSection
+
+from utils import open_file, fubar, poolify
+
+###############################################################################
+
+__all__ = []
+
+###############################################################################
+
+CHANGESFIELDS_MANDATORY = [ "distribution", "source", "architecture",
+ "version", "maintainer", "urgency", "fingerprint", "changedby822",
+ "changedby2047", "changedbyname", "maintainer822", "maintainer2047",
+ "maintainername", "maintaineremail", "closes", "changes" ]
+
+__all__.append('CHANGESFIELDS_MANDATORY')
+
+CHANGESFIELDS_OPTIONAL = [ "changed-by", "filecontents", "format",
+ "process-new note", "adv id", "distribution-version", "sponsoremail" ]
+
+__all__.append('CHANGESFIELDS_OPTIONAL')
+
+CHANGESFIELDS_FILES = [ "package", "version", "architecture", "type", "size",
+ "md5sum", "sha1sum", "sha256sum", "component", "location id",
+ "source package", "source version", "maintainer", "dbtype", "files id",
+ "new", "section", "priority", "othercomponents", "pool name",
+ "original component" ]
+
+__all__.append('CHANGESFIELDS_FILES')
+
+CHANGESFIELDS_DSC = [ "source", "version", "maintainer", "fingerprint",
+ "uploaders", "bts changelog", "dm-upload-allowed" ]
+
+__all__.append('CHANGESFIELDS_DSC')
+
+CHANGESFIELDS_DSCFILES_MANDATORY = [ "size", "md5sum" ]
+
+__all__.append('CHANGESFIELDS_DSCFILES_MANDATORY')
+
+CHANGESFIELDS_DSCFILES_OPTIONAL = [ "files id" ]
+
+__all__.append('CHANGESFIELDS_DSCFILES_OPTIONAL')
+
+###############################################################################
+
+class Changes(object):
+ """ Convenience wrapper to carry around all the package information """
+
+ def __init__(self, **kwds):
+ self.reset()
+
+ def reset(self):
+ self.changes_file = ""
+
+ self.changes = {}
+ self.dsc = {}
+ self.files = {}
+ self.dsc_files = {}
+
+ self.orig_tar_id = None
+ self.orig_tar_location = ""
+ self.orig_tar_gz = None
+
+ def file_summary(self):
+ # changes["distribution"] may not exist in corner cases
+ # (e.g. unreadable changes files)
+ if not self.changes.has_key("distribution") or not \
+ isinstance(self.changes["distribution"], dict):
+ self.changes["distribution"] = {}
+
+ byhand = False
+ new = False
+ summary = ""
+ override_summary = ""
+
+ for name, entry in sorted(self.files.items()):
+ if entry.has_key("byhand"):
+ byhand = True
+ summary += name + " byhand\n"
+
+ elif entry.has_key("new"):
+ new = True
+ summary += "(new) %s %s %s\n" % (name, entry["priority"], entry["section"])
+
+ if entry.has_key("othercomponents"):
+ summary += "WARNING: Already present in %s distribution.\n" % (entry["othercomponents"])
+
+ if entry["type"] == "deb":
+ deb_fh = open_file(name)
+ summary += ParseSection(debExtractControl(deb_fh))["Description"] + '\n'
+ deb_fh.close()
+
+ else:
+ entry["pool name"] = poolify(self.changes.get("source", ""), entry["component"])
+ destination = entry["pool name"] + name
+ summary += name + "\n to " + destination + "\n"
+
+ if not entry.has_key("type"):
+ entry["type"] = "unknown"
+
+ if entry["type"] in ["deb", "udeb", "dsc"]:
+ # (queue/unchecked), there we have override entries already, use them
+ # (process-new), there we dont have override entries, use the newly generated ones.
+ override_prio = entry.get("override priority", entry["priority"])
+ override_sect = entry.get("override section", entry["section"])
+ override_summary += "%s - %s %s\n" % (name, override_prio, override_sect)
+
+ return (byhand, new, summary, override_summary)
+
+ def check_override(self):
+ """
+ Checks override entries for validity.
+
+ Returns an empty string if there are no problems
+ or the text of a warning if there are
+ """
+
+ summary = ""
+
+ # Abandon the check if it's a non-sourceful upload
+ if not self.changes["architecture"].has_key("source"):
+ return summary
+
+ for name, entry in sorted(self.files.items()):
+ if not entry.has_key("new") and entry["type"] == "deb":
+ if entry["section"] != "-":
+ if entry["section"].lower() != entry["override section"].lower():
+ summary += "%s: package says section is %s, override says %s.\n" % (name,
+ entry["section"],
+ entry["override section"])
+
+ if entry["priority"] != "-":
+ if entry["priority"] != entry["override priority"]:
+ summary += "%s: package says priority is %s, override says %s.\n" % (name,
+ entry["priority"],
+ entry["override priority"])
+
+ return summary
+
+
+ def load_dot_dak(self, changesfile):
+ """
+ Update ourself by reading a previously created cPickle .dak dumpfile.
+ """
+
+ self.changes_file = changesfile
+ dump_filename = self.changes_file[:-8]+".dak"
+ dump_file = open_file(dump_filename)
+
+ p = Unpickler(dump_file)
+
+ self.changes.update(p.load())
+ self.dsc.update(p.load())
+ self.files.update(p.load())
+ self.dsc_files.update(p.load())
+
+ self.orig_tar_id = p.load()
+ self.orig_tar_location = p.load()
+
+ dump_file.close()
+
+ def sanitised_files(self):
+ ret = {}
+ for name, entry in self.files.items():
+ ret[name] = {}
+ for i in CHANGESFIELDS_FILES:
+ if entry.has_key(i):
+ ret[name][i] = entry[i]
+
+ return ret
+
+ def sanitised_changes(self):
+ ret = {}
+ # Mandatory changes fields
+ for i in CHANGESFIELDS_MANDATORY:
+ ret[i] = self.changes[i]
+
+ # Optional changes fields
+ for i in CHANGESFIELDS_OPTIONAL:
+ if self.changes.has_key(i):
+ ret[i] = self.changes[i]
+
+ return ret
+
+ def sanitised_dsc(self):
+ ret = {}
+ for i in CHANGESFIELDS_DSC:
+ if self.dsc.has_key(i):
+ ret[i] = self.dsc[i]
+
+ return ret
+
+ def sanitised_dsc_files(self):
+ ret = {}
+ for name, entry in self.dsc_files.items():
+ ret[name] = {}
+ # Mandatory dsc_files fields
+ for i in CHANGESFIELDS_DSCFILES_MANDATORY:
+ ret[name][i] = entry[i]
+
+ # Optional dsc_files fields
+ for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
+ if entry.has_key(i):
+ ret[name][i] = entry[i]
+
+ return ret
+
+ def write_dot_dak(self, dest_dir):
+ """
+ Dump ourself into a cPickle file.
+
+ @type dest_dir: string
+ @param dest_dir: Path where the dumpfile should be stored
+
+ @note: This could just dump the dictionaries as is, but I'd like to avoid this so
+ there's some idea of what process-accepted & process-new use from
+ process-unchecked. (JT)
+
+ """
+
+ dump_filename = os.path.join(dest_dir, self.changes_file[:-8] + ".dak")
+ dump_file = open_file(dump_filename, 'w')
+
+ try:
+ os.chmod(dump_filename, 0664)
+ except OSError, e:
+ # chmod may fail when the dumpfile is not owned by the user
+ # invoking dak (like e.g. when NEW is processed by a member
+ # of ftpteam)
+ if e.errno == EPERM:
+ perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE])
+ # security precaution, should never happen unless a weird
+ # umask is set anywhere
+ if perms & stat.S_IWOTH:
+ fubar("%s is world writable and chmod failed." % \
+ (dump_filename,))
+ # ignore the failed chmod otherwise as the file should
+ # already have the right privileges and is just, at worst,
+ # unreadable for world
+ else:
+ raise
+
+ p = Pickler(dump_file, 1)
+
+ p.dump(self.sanitised_changes())
+ p.dump(self.sanitised_dsc())
+ p.dump(self.sanitised_files())
+ p.dump(self.sanitised_dsc_files())
+ p.dump(self.orig_tar_id)
+ p.dump(self.orig_tar_location)
+
+ dump_file.close()
+
+ def unknown_files_fields(self, name):
+ return sorted(list( set(self.files[name].keys()) -
+ set(CHANGESFIELDS_FILES)))
+
+ def unknown_changes_fields(self):
+ return sorted(list( set(self.changes.keys()) -
+ set(CHANGESFIELDS_MANDATORY + CHANGESFIELDS_OPTIONAL)))
+
+ def unknown_dsc_fields(self):
+ return sorted(list( set(self.dsc.keys()) -
+ set(CHANGESFIELDS_DSC)))
+
+ def unknown_dsc_files_fields(self, name):
+ return sorted(list( set(self.dsc_files[name].keys()) -
+ set(CHANGESFIELDS_DSCFILES_MANDATORY + CHANGESFIELDS_DSCFILES_OPTIONAL)))
+
+ def str_files(self):
+ r = []
+ for name, entry in self.files.items():
+ r.append(" %s:" % (name))
+ for i in CHANGESFIELDS_FILES:
+ if entry.has_key(i):
+ r.append(" %s: %s" % (i.capitalize(), entry[i]))
+ xfields = self.unknown_files_fields(name)
+ if len(xfields) > 0:
+ r.append("files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
+
+ return r
+
+ def str_changes(self):
+ r = []
+ for i in CHANGESFIELDS_MANDATORY:
+ val = self.changes[i]
+ if isinstance(val, list):
+ val = " ".join(val)
+ elif isinstance(val, dict):
+ val = " ".join(val.keys())
+ r.append(' %s: %s' % (i.capitalize(), val))
+
+ for i in CHANGESFIELDS_OPTIONAL:
+ if self.changes.has_key(i):
+ r.append(' %s: %s' % (i.capitalize(), self.changes[i]))
+
+ xfields = self.unknown_changes_fields()
+ if len(xfields) > 0:
+ r.append("Warning: changes still has the following unrecognised fields: %s" % ", ".join(xfields))
+
+ return r
+
+ def str_dsc(self):
+ r = []
+ for i in CHANGESFIELDS_DSC:
+ if self.dsc.has_key(i):
+ r.append(' %s: %s' % (i.capitalize(), self.dsc[i]))
+
+ xfields = self.unknown_dsc_fields()
+ if len(xfields) > 0:
+ r.append("Warning: dsc still has the following unrecognised fields: %s" % ", ".join(xfields))
+
+ return r
+
+ def str_dsc_files(self):
+ r = []
+ for name, entry in self.dsc_files.items():
+ r.append(" %s:" % (name))
+ for i in CHANGESFIELDS_DSCFILES_MANDATORY:
+ r.append(" %s: %s" % (i.capitalize(), entry[i]))
+ for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
+ if entry.has_key(i):
+ r.append(" %s: %s" % (i.capitalize(), entry[i]))
+ xfields = self.unknown_dsc_files_fields(name)
+ if len(xfields) > 0:
+ r.append("dsc_files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
+
+ return r
+
+ def __str__(self):
+ r = []
+
+ r.append(" Changes:")
+ r += self.str_changes()
+
+ r.append("")
+
+ r.append(" Dsc:")
+ r += self.str_dsc()
+
+ r.append("")
+
+ r.append(" Files:")
+ r += self.str_files()
+
+ r.append("")
+
+ r.append(" Dsc Files:")
+ r += self.str_dsc_files()
+
+ return "\n".join(r)
+
+__all__.append('Changes')
self.get = self.Cnf.get
self.SubTree = self.Cnf.SubTree
self.ValueList = self.Cnf.ValueList
+ self.Find = self.Cnf.Find
+ self.FindB = self.Cnf.FindB
def _startup(self, *args, **kwargs):
self._readconf()
def __getitem__(self, name):
return self.Cnf[name]
+ def __setitem__(self, name, value):
+ self.Cnf[name] = value
--- /dev/null
+#!/usr/bin/env python
+
+"""
+Logging functions
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001, 2002, 2006 James Troup <james@nocrew.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import os
+import pwd
+import time
+import sys
+import utils
+
+################################################################################
+
+class Logger:
+ "Logger object"
+ Cnf = None
+ logfile = None
+ program = None
+
+ def __init__ (self, Cnf, program, debug=0):
+ "Initialize a new Logger object"
+ self.Cnf = Cnf
+ self.program = program
+ # Create the log directory if it doesn't exist
+ logdir = Cnf["Dir::Log"]
+ if not os.path.exists(logdir):
+ umask = os.umask(00000)
+ os.makedirs(logdir, 02775)
+ os.umask(umask)
+ # Open the logfile
+ logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m"))
+ logfile = None
+ if debug:
+ logfile = sys.stderr
+ else:
+ umask = os.umask(00002)
+ logfile = utils.open_file(logfilename, 'a')
+ os.umask(umask)
+ self.logfile = logfile
+ # Log the start of the program
+ user = pwd.getpwuid(os.getuid())[0]
+ self.log(["program start", user])
+
+ def log (self, details):
+ "Log an event"
+ # Prepend the timestamp and program name
+ details.insert(0, self.program)
+ timestamp = time.strftime("%Y%m%d%H%M%S")
+ details.insert(0, timestamp)
+ # Force the contents of the list to be string.join-able
+ details = [ str(i) for i in details ]
+ # Write out the log in TSV
+ self.logfile.write("|".join(details)+'\n')
+ # Flush the output to enable tail-ing
+ self.logfile.flush()
+
+ def close (self):
+ "Close a Logger object"
+ self.log(["program end"])
+ self.logfile.flush()
+ self.logfile.close()
import types
import utils
import pg
-from binary import Binary
################################################################################
Cnf = config
projectB = sql
-
-def do_query(query):
- """
- Executes a database query. Writes statistics / timing to stderr.
-
- @type query: string
- @param query: database query string, passed unmodified
-
- @return: db result
-
- @warning: The query is passed B{unmodified}, so be careful what you use this for.
- """
- sys.stderr.write("query: \"%s\" ... " % (query))
- before = time.time()
- r = projectB.query(query)
- time_diff = time.time()-before
- sys.stderr.write("took %.3f seconds.\n" % (time_diff))
- if type(r) is int:
- sys.stderr.write("int result: %s\n" % (r))
- elif type(r) is types.NoneType:
- sys.stderr.write("result: None\n")
- else:
- sys.stderr.write("pgresult: %s\n" % (r.getresult()))
- return r
-
################################################################################
def get_suite_id (suite):
import psycopg2
import traceback
-from singleton import Singleton
+from sqlalchemy import create_engine, Table, MetaData, select
+from sqlalchemy.orm import sessionmaker, mapper, relation
+
+# Don't remove this, we re-export the exceptions to scripts which import us
+from sqlalchemy.exc import *
+
+# Only import Config until Queue stuff is changed to store its config
+# in the database
from config import Config
+from singleton import Singleton
+from textutils import fix_maintainer
+
+################################################################################
+
+__all__ = ['IntegrityError', 'SQLAlchemyError']
+
+################################################################################
+
+class Architecture(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.arch_string== val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.arch_string != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Architecture %s>' % self.arch_string
+
+__all__.append('Architecture')
+
+def get_architecture(architecture, session=None):
+ """
+ Returns database id for given C{architecture}.
+
+ @type architecture: string
+ @param architecture: The name of the architecture
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Architecture
+ @return: Architecture object for the given arch (None if not present)
+
+ """
+ privatetrans = False
+
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Architecture).filter_by(arch_string=architecture)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_architecture')
+
+def get_architecture_suites(architecture, session=None):
+ """
+ Returns list of Suite objects for given C{architecture} name
+
+ @type source: str
+ @param source: Architecture name to search for
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of Suite objects for the given name (may be empty)
+ """
+ privatetrans = False
+
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Suite)
+ q = q.join(SuiteArchitecture)
+ q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_architecture_suites')
+
+################################################################################
+
+class Archive(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Archive %s>' % self.name
+
+__all__.append('Archive')
+
+def get_archive(archive, session=None):
+ """
+ returns database id for given c{archive}.
+
+ @type archive: string
+ @param archive: the name of the arhive
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Archive
+ @return: Archive object for the given name (None if not present)
+
+ """
+ archive = archive.lower()
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Archive).filter_by(archive_name=archive)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_archive')
+
+################################################################################
+
+class BinAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
+
+__all__.append('BinAssociation')
+
+################################################################################
+
+class DBBinary(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
+
+__all__.append('DBBinary')
+
+def get_suites_binary_in(package, session=None):
+ """
+ Returns list of Suite objects which given C{package} name is in
+
+ @type source: str
+ @param source: DBBinary package name to search for
+
+ @rtype: list
+ @return: list of Suite objects for the given package
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ ret = session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
+
+ session.close()
+
+ return ret
+
+__all__.append('get_suites_binary_in')
+
+def get_binary_from_id(id, session=None):
+ """
+ Returns DBBinary object for given C{id}
+
+ @type id: int
+ @param id: Id of the required binary
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: DBBinary
+ @return: DBBinary object for the given binary (None if not present)
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(DBBinary).filter_by(binary_id=id)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_binary_from_id')
+
+def get_binaries_from_name(package, version=None, architecture=None, session=None):
+ """
+ Returns list of DBBinary objects for given C{package} name
+
+ @type package: str
+ @param package: DBBinary package name to search for
+
+ @type version: str or None
+ @param version: Version to search for (or None)
+
+ @type package: str, list or None
+ @param package: Architectures to limit to (or None if no limit)
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of DBBinary objects for the given name (may be empty)
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(DBBinary).filter_by(package=package)
+
+ if version is not None:
+ q = q.filter_by(version=version)
+
+ if architecture is not None:
+ if not isinstance(architecture, list):
+ architecture = [architecture]
+ q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_binaries_from_name')
+
+def get_binaries_from_source_id(source_id, session=None):
+ """
+ Returns list of DBBinary objects for given C{source_id}
+
+ @type source_id: int
+ @param source_id: source_id to search for
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of DBBinary objects for the given name (may be empty)
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ ret = session.query(DBBinary).filter_by(source_id=source_id).all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+
+__all__.append('get_binaries_from_source_id')
+
+
+def get_binary_from_name_suite(package, suitename, session=None):
+ ### For dak examine-package
+ ### XXX: Doesn't use object API yet
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
+ FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
+ WHERE b.package=:package
+ AND b.file = fi.id
+ AND fi.location = l.id
+ AND l.component = c.id
+ AND ba.bin=b.id
+ AND ba.suite = su.id
+ AND su.suite_name=:suitename
+ ORDER BY b.version DESC"""
+
+ ret = session.execute(sql, {'package': package, 'suitename': suitename})
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_binary_from_name_suite')
+
+def get_binary_components(package, suitename, arch, session=None):
+ # Check for packages that have moved from one component to another
+ query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
+ WHERE b.package=:package AND s.suite_name=:suitename
+ AND (a.arch_string = :arch OR a.arch_string = 'all')
+ AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
+ AND f.location = l.id
+ AND l.component = c.id
+ AND b.file = f.id"""
+
+ vals = {'package': package, 'suitename': suitename, 'arch': arch}
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ ret = session.execute(query, vals)
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_binary_components')
+
+################################################################################
+
+class Component(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.component_name == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.component_name != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Component %s>' % self.component_name
+
+
+__all__.append('Component')
+
+def get_component(component, session=None):
+ """
+ Returns database id for given C{component}.
+
+ @type component: string
+ @param component: The name of the override type
+
+ @rtype: int
+ @return: the database id for the given component
+
+ """
+ component = component.lower()
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Component).filter_by(component_name=component)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_component')
+
+################################################################################
+
+class DBConfig(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<DBConfig %s>' % self.name
+
+__all__.append('DBConfig')
+
+################################################################################
+
+class ContentFilename(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<ContentFilename %s>' % self.filename
+
+__all__.append('ContentFilename')
+
+def get_or_set_contents_file_id(filename, session=None):
+ """
+ Returns database id for given filename.
+
+ If no matching file is found, a row is inserted.
+
+ @type filename: string
+ @param filename: The filename
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: int
+ @return: the database id for the given component
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(ContentFilename).filter_by(filename=filename)
+ if q.count() < 1:
+ cf = ContentFilename()
+ cf.filename = filename
+ session.add(cf)
+ if privatetrans:
+ session.commit()
+ else:
+ session.flush()
+ ret = cf.cafilename_id
+ else:
+ ret = q.one().cafilename_id
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_or_set_contents_file_id')
+
+def get_contents(suite, overridetype, section=None, session=None):
+ """
+ Returns contents for a suite / overridetype combination, limiting
+ to a section if not None.
+
+ @type suite: Suite
+ @param suite: Suite object
+
+ @type overridetype: OverrideType
+ @param overridetype: OverrideType object
+
+ @type section: Section
+ @param section: Optional section object to limit results to
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: ResultsProxy
+ @return: ResultsProxy object set up to return tuples of (filename, section,
+ package, arch_id)
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ # find me all of the contents for a given suite
+ contents_q = """SELECT (p.path||'/'||n.file) AS fn,
+ s.section,
+ b.package,
+ b.architecture
+ FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
+ JOIN content_file_names n ON (c.filename=n.id)
+ JOIN binaries b ON (b.id=c.binary_pkg)
+ JOIN override o ON (o.package=b.package)
+ JOIN section s ON (s.id=o.section)
+ WHERE o.suite = :suiteid AND o.type = :overridetypeid
+ AND b.type=:overridetypename"""
+
+ vals = {'suiteid': suite.suite_id,
+ 'overridetypeid': overridetype.overridetype_id,
+ 'overridetypename': overridetype.overridetype}
+
+ if section is not None:
+ contents_q += " AND s.id = :sectionid"
+ vals['sectionid'] = section.section_id
+
+ contents_q += " ORDER BY fn"
+
+ ret = session.execute(contents_q, vals)
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_contents')
+
+################################################################################
+
+class ContentFilepath(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<ContentFilepath %s>' % self.filepath
+
+__all__.append('ContentFilepath')
+
+def get_or_set_contents_path_id(filepath, session=None):
+ """
+ Returns database id for given path.
+
+ If no matching file is found, a row is inserted.
+
+ @type filename: string
+ @param filename: The filepath
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: int
+ @return: the database id for the given path
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(ContentFilepath).filter_by(filepath=filepath)
+ if q.count() < 1:
+ cf = ContentFilepath()
+ cf.filepath = filepath
+ session.add(cf)
+ if privatetrans:
+ session.commit()
+ else:
+ session.flush()
+ ret = cf.cafilepath_id
+ else:
+ ret = q.one().cafilepath_id
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_or_set_contents_path_id')
+
+################################################################################
+
+class ContentAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<ContentAssociation %s>' % self.ca_id
+
+__all__.append('ContentAssociation')
+
+def insert_content_paths(binary_id, fullpaths, session=None):
+ """
+ Make sure given path is associated with given binary id
+
+ @type binary_id: int
+ @param binary_id: the id of the binary
+ @type fullpaths: list
+ @param fullpaths: the list of paths of the file being associated with the binary
+ @type session: SQLAlchemy session
+ @param session: Optional SQLAlchemy session. If this is passed, the caller
+ is responsible for ensuring a transaction has begun and committing the
+ results or rolling back based on the result code. If not passed, a commit
+ will be performed at the end of the function, otherwise the caller is
+ responsible for commiting.
+
+ @return: True upon success
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ try:
+ # Insert paths
+ pathcache = {}
+ for fullpath in fullpaths:
+ # Get the necessary IDs ...
+ (path, file) = os.path.split(fullpath)
+
+ filepath_id = get_or_set_contents_path_id(path, session)
+ filename_id = get_or_set_contents_file_id(file, session)
+
+ pathcache[fullpath] = (filepath_id, filename_id)
+
+ for fullpath, dat in pathcache.items():
+ ca = ContentAssociation()
+ ca.binary_id = binary_id
+ ca.filepath_id = dat[0]
+ ca.filename_id = dat[1]
+ session.add(ca)
+
+ # Only commit if we set up the session ourself
+ if privatetrans:
+ session.commit()
+ session.close()
+ else:
+ session.flush()
+
+ return True
+
+ except:
+ traceback.print_exc()
+
+ # Only rollback if we set up the session ourself
+ if privatetrans:
+ session.rollback()
+ session.close()
+
+ return False
+
+__all__.append('insert_content_paths')
+
+################################################################################
+
+class DSCFile(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<DSCFile %s>' % self.dscfile_id
+
+__all__.append('DSCFile')
+
+def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
+ """
+ Returns a list of DSCFiles which may be empty
+
+ @type dscfile_id: int (optional)
+ @param dscfile_id: the dscfile_id of the DSCFiles to find
+
+ @type source_id: int (optional)
+ @param source_id: the source id related to the DSCFiles to find
+
+ @type poolfile_id: int (optional)
+ @param poolfile_id: the poolfile id related to the DSCFiles to find
+
+ @rtype: list
+ @return: Possibly empty list of DSCFiles
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(DSCFile)
+
+ if dscfile_id is not None:
+ q = q.filter_by(dscfile_id=dscfile_id)
+
+ if source_id is not None:
+ q = q.filter_by(source_id=source_id)
+
+ if poolfile_id is not None:
+ q = q.filter_by(poolfile_id=poolfile_id)
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_dscfiles')
+
+################################################################################
+
+class PoolFile(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<PoolFile %s>' % self.filename
+
+__all__.append('PoolFile')
+
+def check_poolfile(filename, filesize, md5sum, location_id, session=None):
+ """
+ Returns a tuple:
+ (ValidFileFound [boolean or None], PoolFile object or None)
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @type filesize: int
+ @param filesize: the size of the file to check against the DB
+
+ @type md5sum: string
+ @param md5sum: the md5sum of the file to check against the DB
+
+ @type location_id: int
+ @param location_id: the id of the location to look in
+
+ @rtype: tuple
+ @return: Tuple of length 2.
+ If more than one file found with that name:
+ (None, None)
+ If valid pool file found: (True, PoolFile object)
+ If valid pool file not found:
+ (False, None) if no file found
+ (False, PoolFile object) if file found with size/md5sum mismatch
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(PoolFile).filter_by(filename=filename)
+ q = q.join(Location).filter_by(location_id=location_id)
+
+ ret = None
+
+ if q.count() > 1:
+ ret = (None, None)
+ elif q.count() < 1:
+ ret = (False, None)
+ else:
+ obj = q.one()
+ if obj.md5sum != md5sum or obj.filesize != filesize:
+ ret = (False, obj)
+
+ if ret is None:
+ ret = (True, obj)
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('check_poolfile')
+
+def get_poolfile_by_id(file_id, session=None):
+ """
+ Returns a PoolFile objects or None for the given id
+
+ @type file_id: int
+ @param file_id: the id of the file to look for
+
+ @rtype: PoolFile or None
+ @return: either the PoolFile object or None
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(PoolFile).filter_by(file_id=file_id)
+
+ if q.count() > 0:
+ ret = q.one()
+ else:
+ ret = None
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_poolfile_by_id')
+
+
+def get_poolfile_by_name(filename, location_id=None, session=None):
+ """
+ Returns an array of PoolFile objects for the given filename and
+ (optionally) location_id
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @type location_id: int
+ @param location_id: the id of the location to look in (optional)
+
+ @rtype: array
+ @return: array of PoolFile objects
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(PoolFile).filter_by(filename=filename)
+
+ if location_id is not None:
+ q = q.join(Location).filter_by(location_id=location_id)
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_poolfile_by_name')
+
+def get_poolfile_like_name(filename, session=None):
+ """
+ Returns an array of PoolFile objects which are like the given name
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @rtype: array
+ @return: array of PoolFile objects
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ # TODO: There must be a way of properly using bind parameters with %FOO%
+ q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_poolfile_like_name')
+
+################################################################################
+
+class Fingerprint(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Fingerprint %s>' % self.fingerprint
+
+__all__.append('Fingerprint')
+
+def get_or_set_fingerprint(fpr, session=None):
+ """
+ Returns Fingerprint object for given fpr.
+
+ If no matching fpr is found, a row is inserted.
+
+ @type fpr: string
+ @param fpr: The fpr to find / add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+ A flush will be performed either way.
+
+ @rtype: Fingerprint
+ @return: the Fingerprint object for the given fpr
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Fingerprint).filter_by(fingerprint=fpr)
+ if q.count() < 1:
+ fingerprint = Fingerprint()
+ fingerprint.fingerprint = fpr
+ session.add(fingerprint)
+ if privatetrans:
+ session.commit()
+ else:
+ session.flush()
+ ret = fingerprint
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_or_set_fingerprint')
+
+################################################################################
+
+class Keyring(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Keyring %s>' % self.keyring_name
+
+__all__.append('Keyring')
+
+################################################################################
+
+class Location(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Location %s (%s)>' % (self.path, self.location_id)
+
+__all__.append('Location')
+
+def get_location(location, component=None, archive=None, session=None):
+ """
+ Returns Location object for the given combination of location, component
+ and archive
+
+ @type location: string
+ @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
+
+ @type component: string
+ @param component: the component name (if None, no restriction applied)
+
+ @type archive: string
+ @param archive_id: the archive name (if None, no restriction applied)
+
+ @rtype: Location / None
+ @return: Either a Location object or None if one can't be found
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Location).filter_by(path=location)
+
+ if archive is not None:
+ q = q.join(Archive).filter_by(archive_name=archive)
+
+ if component is not None:
+ q = q.join(Component).filter_by(component_name=component)
+
+ if q.count() < 1:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_location')
+
+################################################################################
+
+class Maintainer(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
+
+ def get_split_maintainer(self):
+ if not hasattr(self, 'name') or self.name is None:
+ return ('', '', '', '')
+
+ return fix_maintainer(self.name.strip())
+
+__all__.append('Maintainer')
+
+def get_or_set_maintainer(name, session=None):
+ """
+ Returns Maintainer object for given maintainer name.
+
+ If no matching maintainer name is found, a row is inserted.
+
+ @type name: string
+ @param name: The maintainer name to add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+ A flush will be performed either way.
+
+ @rtype: Maintainer
+ @return: the Maintainer object for the given maintainer
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Maintainer).filter_by(name=name)
+ if q.count() < 1:
+ maintainer = Maintainer()
+ maintainer.name = name
+ session.add(maintainer)
+ if privatetrans:
+ session.commit()
+ else:
+ session.flush()
+ ret = maintainer
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_or_set_maintainer')
+
+################################################################################
+
+class NewComment(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
+
+__all__.append('NewComment')
+
+def has_new_comment(package, version, session=None):
+ """
+ Returns true if the given combination of C{package}, C{version} has a comment.
+
+ @type package: string
+ @param package: name of the package
+
+ @type version: string
+ @param version: package version
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: boolean
+ @return: true/false
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(NewComment)
+ q = q.filter_by(package=package)
+ q = q.filter_by(version=version)
+
+ ret = q.count() > 0
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('has_new_comment')
+
+def get_new_comments(package=None, version=None, comment_id=None, session=None):
+ """
+ Returns (possibly empty) list of NewComment objects for the given
+ parameters
+
+ @type package: string (optional)
+ @param package: name of the package
+
+ @type version: string (optional)
+ @param version: package version
+
+ @type comment_id: int (optional)
+ @param comment_id: An id of a comment
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: A (possibly empty) list of NewComment objects will be returned
+
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(NewComment)
+ if package is not None: q = q.filter_by(package=package)
+ if version is not None: q = q.filter_by(version=version)
+ if comment_id is not None: q = q.filter_by(comment_id=comment_id)
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_new_comments')
+
+################################################################################
+
+class Override(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Override %s (%s)>' % (self.package, self.suite_id)
+
+__all__.append('Override')
+
+def get_override(package, suite=None, component=None, overridetype=None, session=None):
+ """
+ Returns Override object for the given parameters
+
+ @type package: string
+ @param package: The name of the package
+
+ @type suite: string, list or None
+ @param suite: The name of the suite (or suites if a list) to limit to. If
+ None, don't limit. Defaults to None.
+
+ @type component: string, list or None
+ @param component: The name of the component (or components if a list) to
+ limit to. If None, don't limit. Defaults to None.
+
+ @type overridetype: string, list or None
+ @param overridetype: The name of the overridetype (or overridetypes if a list) to
+ limit to. If None, don't limit. Defaults to None.
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: A (possibly empty) list of Override objects will be returned
+
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Override)
+ q = q.filter_by(package=package)
+
+ if suite is not None:
+ if not isinstance(suite, list): suite = [suite]
+ q = q.join(Suite).filter(Suite.suite_name.in_(suite))
+
+ if component is not None:
+ if not isinstance(component, list): component = [component]
+ q = q.join(Component).filter(Component.component_name.in_(component))
+
+ if overridetype is not None:
+ if not isinstance(overridetype, list): overridetype = [overridetype]
+ q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_override')
+
################################################################################
-class Cache(object):
- def __init__(self, hashfunc=None):
- if hashfunc:
- self.hashfunc = hashfunc
- else:
- self.hashfunc = lambda x: x['value']
+class OverrideType(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<OverrideType %s>' % self.overridetype
+
+__all__.append('OverrideType')
+
+def get_override_type(override_type, session=None):
+ """
+ Returns OverrideType object for given C{override type}.
+
+ @type override_type: string
+ @param override_type: The name of the override type
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: int
+ @return: the database id for the given override type
+
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(OverrideType).filter_by(overridetype=override_type)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
- self.data = {}
+ if privatetrans:
+ session.close()
- def SetValue(self, keys, value):
- self.data[self.hashfunc(keys)] = value
+ return ret
- def GetValue(self, keys):
- return self.data.get(self.hashfunc(keys))
+__all__.append('get_override_type')
################################################################################
-class DBConn(Singleton):
+class PendingContentAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<PendingContentAssociation %s>' % self.pca_id
+
+__all__.append('PendingContentAssociation')
+
+def insert_pending_content_paths(package, fullpaths, session=None):
"""
- database module init.
+ Make sure given paths are temporarily associated with given
+ package
+
+ @type package: dict
+ @param package: the package to associate with should have been read in from the binary control file
+ @type fullpaths: list
+ @param fullpaths: the list of paths of the file being associated with the binary
+ @type session: SQLAlchemy session
+ @param session: Optional SQLAlchemy session. If this is passed, the caller
+ is responsible for ensuring a transaction has begun and committing the
+ results or rolling back based on the result code. If not passed, a commit
+ will be performed at the end of the function
+
+ @return: True upon success, False if there is a problem
"""
+
+ privatetrans = False
+
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ try:
+ arch = get_architecture(package['Architecture'], session)
+ arch_id = arch.arch_id
+
+ # Remove any already existing recorded files for this package
+ q = session.query(PendingContentAssociation)
+ q = q.filter_by(package=package['Package'])
+ q = q.filter_by(version=package['Version'])
+ q = q.filter_by(architecture=arch_id)
+ q.delete()
+
+ # Insert paths
+ pathcache = {}
+ for fullpath in fullpaths:
+ (path, file) = os.path.split(fullpath)
+
+ if path.startswith( "./" ):
+ path = path[2:]
+
+ filepath_id = get_or_set_contents_path_id(path, session)
+ filename_id = get_or_set_contents_file_id(file, session)
+
+ pathcache[fullpath] = (filepath_id, filename_id)
+
+ for fullpath, dat in pathcache.items():
+ pca = PendingContentAssociation()
+ pca.package = package['Package']
+ pca.version = package['Version']
+ pca.filepath_id = dat[0]
+ pca.filename_id = dat[1]
+ pca.architecture = arch_id
+ session.add(pca)
+
+ # Only commit if we set up the session ourself
+ if privatetrans:
+ session.commit()
+ session.close()
+ else:
+ session.flush()
+
+ return True
+ except Exception, e:
+ traceback.print_exc()
+
+ # Only rollback if we set up the session ourself
+ if privatetrans:
+ session.rollback()
+ session.close()
+
+ return False
+
+__all__.append('insert_pending_content_paths')
+
+################################################################################
+
+class Priority(object):
def __init__(self, *args, **kwargs):
- super(DBConn, self).__init__(*args, **kwargs)
+ pass
- def _startup(self, *args, **kwargs):
- self.__createconn()
- self.__init_caches()
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.priority == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
- ## Connection functions
- def __createconn(self):
- cnf = Config()
- connstr = "dbname=%s" % cnf["DB::Name"]
- if cnf["DB::Host"]:
- connstr += " host=%s" % cnf["DB::Host"]
- if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
- connstr += " port=%s" % cnf["DB::Port"]
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.priority != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
- self.db_con = psycopg2.connect(connstr)
+ def __repr__(self):
+ return '<Priority %s (%s)>' % (self.priority, self.priority_id)
- def reconnect(self):
- try:
- self.db_con.close()
- except psycopg2.InterfaceError:
- pass
+__all__.append('Priority')
- self.db_con = None
- self.__createconn()
+def get_priority(priority, session=None):
+ """
+ Returns Priority object for given C{priority name}.
- ## Cache functions
- def __init_caches(self):
- self.caches = {'suite': Cache(),
- 'section': Cache(),
- 'priority': Cache(),
- 'override_type': Cache(),
- 'architecture': Cache(),
- 'archive': Cache(),
- 'component': Cache(),
- 'content_path_names': Cache(),
- 'content_file_names': Cache(),
- 'location': Cache(lambda x: '%s_%s_%s' % (x['location'], x['component'], x['location'])),
- 'maintainer': {}, # TODO
- 'keyring': {}, # TODO
- 'source': Cache(lambda x: '%s_%s_' % (x['source'], x['version'])),
- 'files': Cache(lambda x: '%s_%s_' % (x['filename'], x['location'])),
- 'maintainer': {}, # TODO
- 'fingerprint': {}, # TODO
- 'queue': {}, # TODO
- 'uid': {}, # TODO
- 'suite_version': Cache(lambda x: '%s_%s' % (x['source'], x['suite'])),
- }
-
- self.prepared_statements = {}
-
- def prepare(self,name,statement):
- if not self.prepared_statements.has_key(name):
- c = self.cursor()
- c.execute(statement)
- self.prepared_statements[name] = statement
-
- def clear_caches(self):
- self.__init_caches()
-
- ## Functions to pass through to the database connector
- def cursor(self):
- return self.db_con.cursor()
-
- def commit(self):
- return self.db_con.commit()
-
- ## Get functions
- def __get_single_id(self, query, values, cachename=None):
- # This is a bit of a hack but it's an internal function only
- if cachename is not None:
- res = self.caches[cachename].GetValue(values)
- if res:
- return res
-
- c = self.db_con.cursor()
- c.execute(query, values)
-
- if c.rowcount != 1:
- return None
-
- res = c.fetchone()[0]
-
- if cachename is not None:
- self.caches[cachename].SetValue(values, res)
-
- return res
-
- def __get_id(self, retfield, table, qfield, value):
- query = "SELECT %s FROM %s WHERE %s = %%(value)s" % (retfield, table, qfield)
- return self.__get_single_id(query, {'value': value}, cachename=table)
-
- def get_suite_id(self, suite):
- """
- Returns database id for given C{suite}.
- Results are kept in a cache during runtime to minimize database queries.
+ @type priority: string
+ @param priority: The name of the priority
- @type suite: string
- @param suite: The name of the suite
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
- @rtype: int
- @return: the database id for the given suite
+ @rtype: Priority
+ @return: Priority object for the given priority
- """
- suiteid = self.__get_id('id', 'suite', 'suite_name', suite)
- if suiteid is None:
- return None
- else:
- return int(suiteid)
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- def get_section_id(self, section):
- """
- Returns database id for given C{section}.
- Results are kept in a cache during runtime to minimize database queries.
+ q = session.query(Priority).filter_by(priority=priority)
- @type section: string
- @param section: The name of the section
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
- @rtype: int
- @return: the database id for the given section
+ if privatetrans:
+ session.close()
- """
- return self.__get_id('id', 'section', 'section', section)
+ return ret
- def get_priority_id(self, priority):
- """
- Returns database id for given C{priority}.
- Results are kept in a cache during runtime to minimize database queries.
+__all__.append('get_priority')
- @type priority: string
- @param priority: The name of the priority
+def get_priorities(session=None):
+ """
+ Returns dictionary of priority names -> id mappings
- @rtype: int
- @return: the database id for the given priority
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
- """
- return self.__get_id('id', 'priority', 'priority', priority)
+ @rtype: dictionary
+ @return: dictionary of priority names -> id mappings
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- def get_override_type_id(self, override_type):
- """
- Returns database id for given override C{type}.
- Results are kept in a cache during runtime to minimize database queries.
+ ret = {}
+ q = session.query(Priority)
+ for x in q.all():
+ ret[x.priority] = x.priority_id
- @type override_type: string
- @param override_type: The name of the override type
+ if privatetrans:
+ session.close()
- @rtype: int
- @return: the database id for the given override type
+ return ret
- """
- return self.__get_id('id', 'override_type', 'type', override_type)
+__all__.append('get_priorities')
- def get_architecture_id(self, architecture):
- """
- Returns database id for given C{architecture}.
- Results are kept in a cache during runtime to minimize database queries.
+################################################################################
- @type architecture: string
- @param architecture: The name of the override type
+class Queue(object):
+ def __init__(self, *args, **kwargs):
+ pass
- @rtype: int
- @return: the database id for the given architecture
+ def __repr__(self):
+ return '<Queue %s>' % self.queue_name
+ def autobuild_upload(self, changes, srcpath, session=None):
"""
- return self.__get_id('id', 'architecture', 'arch_string', architecture)
+ Update queue_build database table used for incoming autobuild support.
- def get_archive_id(self, archive):
- """
- returns database id for given c{archive}.
- results are kept in a cache during runtime to minimize database queries.
+ @type changes: Changes
+ @param changes: changes object for the upload to process
- @type archive: string
- @param archive: the name of the override type
+ @type srcpath: string
+ @param srcpath: path for the queue file entries/link destinations
- @rtype: int
- @return: the database id for the given archive
+ @type session: SQLAlchemy session
+ @param session: Optional SQLAlchemy session. If this is passed, the
+ caller is responsible for ensuring a transaction has begun and
+ committing the results or rolling back based on the result code. If
+ not passed, a commit will be performed at the end of the function,
+ otherwise the caller is responsible for commiting.
+ @rtype: NoneType or string
+ @return: None if the operation failed, a string describing the error if not
"""
- return self.__get_id('id', 'archive', 'lower(name)', archive)
- def get_component_id(self, component):
- """
- Returns database id for given C{component}.
- Results are kept in a cache during runtime to minimize database queries.
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ # TODO: Remove by moving queue config into the database
+ conf = Config()
+
+ for suitename in changes.changes["distribution"].keys():
+ # TODO: Move into database as:
+ # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
+ # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
+ # This also gets rid of the SecurityQueueBuild hack below
+ if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
+ continue
+
+ # Find suite object
+ s = get_suite(suitename, session)
+ if s is None:
+ return "INTERNAL ERROR: Could not find suite %s" % suitename
+
+ # TODO: Get from database as above
+ dest_dir = conf["Dir::QueueBuild"]
+
+ # TODO: Move into database as above
+ if conf.FindB("Dinstall::SecurityQueueBuild"):
+ dest_dir = os.path.join(dest_dir, suitename)
+
+ for file_entry in changes.files.keys():
+ src = os.path.join(srcpath, file_entry)
+ dest = os.path.join(dest_dir, file_entry)
+
+ # TODO: Move into database as above
+ if conf.FindB("Dinstall::SecurityQueueBuild"):
+ # Copy it since the original won't be readable by www-data
+ utils.copy(src, dest)
+ else:
+ # Create a symlink to it
+ os.symlink(src, dest)
+
+ qb = QueueBuild()
+ qb.suite_id = s.suite_id
+ qb.queue_id = self.queue_id
+ qb.filename = dest
+ qb.in_queue = True
+
+ session.add(qb)
+
+ # If the .orig.tar.gz is in the pool, create a symlink to
+ # it (if one doesn't already exist)
+ if changes.orig_tar_id:
+ # Determine the .orig.tar.gz file name
+ for dsc_file in changes.dsc_files.keys():
+ if dsc_file.endswith(".orig.tar.gz"):
+ filename = dsc_file
+
+ dest = os.path.join(dest_dir, filename)
+
+ # If it doesn't exist, create a symlink
+ if not os.path.exists(dest):
+ q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id",
+ {'id': changes.orig_tar_id})
+ res = q.fetchone()
+ if not res:
+ return "[INTERNAL ERROR] Couldn't find id %s in files table." % (changes.orig_tar_id)
+
+ src = os.path.join(res[0], res[1])
+ os.symlink(src, dest)
+
+ # Add it to the list of packages for later processing by apt-ftparchive
+ qb = QueueBuild()
+ qb.suite_id = s.suite_id
+ qb.queue_id = self.queue_id
+ qb.filename = dest
+ qb.in_queue = True
+ session.add(qb)
+
+ # If it does, update things to ensure it's not removed prematurely
+ else:
+ qb = get_queue_build(dest, s.suite_id, session)
+ if qb is None:
+ qb.in_queue = True
+ qb.last_used = None
+ session.add(qb)
- @type component: string
- @param component: The name of the override type
+ if privatetrans:
+ session.commit()
+ session.close()
- @rtype: int
- @return: the database id for the given component
+ return None
- """
- return self.__get_id('id', 'component', 'lower(name)', component)
+__all__.append('Queue')
- def get_location_id(self, location, component, archive):
- """
- Returns database id for the location behind the given combination of
- - B{location} - the path of the location, eg. I{/srv/ftp.debian.org/ftp/pool/}
- - B{component} - the id of the component as returned by L{get_component_id}
- - B{archive} - the id of the archive as returned by L{get_archive_id}
- Results are kept in a cache during runtime to minimize database queries.
+def get_queue(queuename, session=None):
+ """
+ Returns Queue object for given C{queue name}.
- @type location: string
- @param location: the path of the location
+ @type queuename: string
+ @param queuename: The name of the queue
- @type component: int
- @param component: the id of the component
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
- @type archive: int
- @param archive: the id of the archive
+ @rtype: Queue
+ @return: Queue object for the given queue
- @rtype: int
- @return: the database id for the location
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- """
+ q = session.query(Queue).filter_by(queue_name=queuename)
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
- archive_id = self.get_archive_id(archive)
+ if privatetrans:
+ session.close()
- if not archive_id:
- return None
+ return ret
- res = None
+__all__.append('get_queue')
- if component:
- component_id = self.get_component_id(component)
- if component_id:
- res = self.__get_single_id("SELECT id FROM location WHERE path=%(location)s AND component=%(component)s AND archive=%(archive)s",
- {'location': location,
- 'archive': int(archive_id),
- 'component': component_id}, cachename='location')
- else:
- res = self.__get_single_id("SELECT id FROM location WHERE path=%(location)s AND archive=%(archive)d",
- {'location': location, 'archive': archive_id, 'component': ''}, cachename='location')
+################################################################################
- return res
+class QueueBuild(object):
+ def __init__(self, *args, **kwargs):
+ pass
- def get_source_id(self, source, version):
- """
- Returns database id for the combination of C{source} and C{version}
- - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{version}
- Results are kept in a cache during runtime to minimize database queries.
+ def __repr__(self):
+ return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
- @type source: string
- @param source: source package name
+__all__.append('QueueBuild')
- @type version: string
- @param version: the source version
+def get_queue_build(filename, suite, session=None):
+ """
+ Returns QueueBuild object for given C{filename} and C{suite}.
- @rtype: int
- @return: the database id for the source
+ @type filename: string
+ @param filename: The name of the file
- """
- return self.__get_single_id("SELECT id FROM source s WHERE s.source=%(source)s AND s.version=%(version)s",
- {'source': source, 'version': version}, cachename='source')
+ @type suiteid: int or str
+ @param suiteid: Suite name or ID
- def get_suite_version(self, source, suite):
- """
- Returns database id for a combination of C{source} and C{suite}.
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Queue
+ @return: Queue object for the given queue
- - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{suite} - a suite name, eg. I{unstable}
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- Results are kept in a cache during runtime to minimize database queries.
+ if isinstance(suite, int):
+ q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite)
+ else:
+ q = session.query(QueueBuild).filter_by(filename=filename)
+ q = q.join(Suite).filter_by(suite_name=suite)
- @type source: string
- @param source: source package name
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
- @type suite: string
- @param suite: the suite name
+ if privatetrans:
+ session.close()
- @rtype: string
- @return: the version for I{source} in I{suite}
+ return ret
- """
- return self.__get_single_id("""
- SELECT s.version FROM source s, suite su, src_associations sa
- WHERE sa.source=s.id
- AND sa.suite=su.id
- AND su.suite_name=%(suite)s
- AND s.source=%(source)""", {'suite': suite, 'source': source}, cachename='suite_version')
+__all__.append('get_queue_build')
+################################################################################
- def get_files_id (self, filename, size, md5sum, location_id):
- """
- Returns -1, -2 or the file_id for filename, if its C{size} and C{md5sum} match an
- existing copy.
+class Section(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.section == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
- The database is queried using the C{filename} and C{location_id}. If a file does exist
- at that location, the existing size and md5sum are checked against the provided
- parameters. A size or checksum mismatch returns -2. If more than one entry is
- found within the database, a -1 is returned, no result returns None, otherwise
- the file id.
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.section != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
- Results are kept in a cache during runtime to minimize database queries.
+ def __repr__(self):
+ return '<Section %s>' % self.section
- @type filename: string
- @param filename: the filename of the file to check against the DB
+__all__.append('Section')
- @type size: int
- @param size: the size of the file to check against the DB
+def get_section(section, session=None):
+ """
+ Returns Section object for given C{section name}.
- @type md5sum: string
- @param md5sum: the md5sum of the file to check against the DB
+ @type section: string
+ @param section: The name of the section
- @type location_id: int
- @param location_id: the id of the location as returned by L{get_location_id}
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
- @rtype: int / None
- @return: Various return values are possible:
- - -2: size/checksum error
- - -1: more than one file found in database
- - None: no file found in database
- - int: file id
+ @rtype: Section
+ @return: Section object for the given section name
- """
- values = {'filename' : filename,
- 'location' : location_id}
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- res = self.caches['files'].GetValue( values )
+ q = session.query(Section).filter_by(section=section)
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
- if not res:
- query = """SELECT id, size, md5sum
- FROM files
- WHERE filename = %(filename)s AND location = %(location)s"""
+ if privatetrans:
+ session.close()
- cursor = self.db_con.cursor()
- cursor.execute( query, values )
+ return ret
- if cursor.rowcount == 0:
- res = None
+__all__.append('get_section')
- elif cursor.rowcount != 1:
- res = -1
+def get_sections(session=None):
+ """
+ Returns dictionary of section names -> id mappings
- else:
- row = cursor.fetchone()
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
- if row[1] != int(size) or row[2] != md5sum:
- res = -2
+ @rtype: dictionary
+ @return: dictionary of section names -> id mappings
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- else:
- self.caches['files'].SetValue(values, row[0])
- res = row[0]
+ ret = {}
+ q = session.query(Section)
+ for x in q.all():
+ ret[x.section] = x.section_id
- return res
+ if privatetrans:
+ session.close()
+ return ret
- def get_or_set_contents_file_id(self, filename):
- """
- Returns database id for given filename.
+__all__.append('get_sections')
- Results are kept in a cache during runtime to minimize database queries.
- If no matching file is found, a row is inserted.
+################################################################################
- @type filename: string
- @param filename: The filename
+class DBSource(object):
+ def __init__(self, *args, **kwargs):
+ pass
- @rtype: int
- @return: the database id for the given component
- """
- try:
- values={'value': filename}
- query = "SELECT id FROM content_file_names WHERE file = %(value)s"
- id = self.__get_single_id(query, values, cachename='content_file_names')
- if not id:
- c = self.db_con.cursor()
- c.execute( "INSERT INTO content_file_names VALUES (DEFAULT, %(value)s) RETURNING id",
- values )
-
- id = c.fetchone()[0]
- self.caches['content_file_names'].SetValue(values, id)
-
- return id
- except:
- traceback.print_exc()
- raise
-
- def get_or_set_contents_path_id(self, path):
- """
- Returns database id for given path.
+ def __repr__(self):
+ return '<DBSource %s (%s)>' % (self.source, self.version)
- Results are kept in a cache during runtime to minimize database queries.
- If no matching file is found, a row is inserted.
+__all__.append('DBSource')
- @type path: string
- @param path: The filename
+def source_exists(source, source_version, suites = ["any"], session=None):
+ """
+ Ensure that source exists somewhere in the archive for the binary
+ upload being processed.
+ 1. exact match => 1.0-3
+ 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
- @rtype: int
- @return: the database id for the given component
- """
- try:
- values={'value': path}
- query = "SELECT id FROM content_file_paths WHERE path = %(value)s"
- id = self.__get_single_id(query, values, cachename='content_path_names')
- if not id:
- c = self.db_con.cursor()
- c.execute( "INSERT INTO content_file_paths VALUES (DEFAULT, %(value)s) RETURNING id",
- values )
-
- id = c.fetchone()[0]
- self.caches['content_path_names'].SetValue(values, id)
-
- return id
- except:
- traceback.print_exc()
- raise
-
- def get_suite_architectures(self, suite):
- """
- Returns list of architectures for C{suite}.
+ @type package: string
+ @param package: package source name
- @type suite: string, int
- @param suite: the suite name or the suite_id
+ @type source_version: string
+ @param source_version: expected source version
- @rtype: list
- @return: the list of architectures for I{suite}
- """
+ @type suites: list
+ @param suites: list of suites to check in, default I{any}
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: int
+ @return: returns 1 if a source with expected version is found, otherwise 0
+
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ cnf = Config()
+ ret = 1
+
+ for suite in suites:
+ q = session.query(DBSource).filter_by(source=source)
+ if suite != "any":
+ # source must exist in suite X, or in some other suite that's
+ # mapped to X, recursively... silent-maps are counted too,
+ # unreleased-maps aren't.
+ maps = cnf.ValueList("SuiteMappings")[:]
+ maps.reverse()
+ maps = [ m.split() for m in maps ]
+ maps = [ (x[1], x[2]) for x in maps
+ if x[0] == "map" or x[0] == "silent-map" ]
+ s = [suite]
+ for x in maps:
+ if x[1] in s and x[0] not in s:
+ s.append(x[0])
+
+ q = q.join(SrcAssociation).join(Suite)
+ q = q.filter(Suite.suite_name.in_(s))
+
+ # Reduce the query results to a list of version numbers
+ ql = [ j.version for j in q.all() ]
+
+ # Try (1)
+ if source_version in ql:
+ continue
+
+ # Try (2)
+ from daklib.regexes import re_bin_only_nmu
+ orig_source_version = re_bin_only_nmu.sub('', source_version)
+ if orig_source_version in ql:
+ continue
+
+ # No source found so return not ok
+ ret = 0
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('source_exists')
+
+def get_suites_source_in(source, session=None):
+ """
+ Returns list of Suite objects which given C{source} name is in
+
+ @type source: str
+ @param source: DBSource package name to search for
+
+ @rtype: list
+ @return: list of Suite objects for the given source
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ ret = session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_suites_source_in')
+
+def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
+ """
+ Returns list of DBSource objects for given C{source} name and other parameters
+
+ @type source: str
+ @param source: DBSource package name to search for
+
+ @type source: str or None
+ @param source: DBSource version name to search for or None if not applicable
+
+ @type dm_upload_allowed: bool
+ @param dm_upload_allowed: If None, no effect. If True or False, only
+ return packages with that dm_upload_allowed setting
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of DBSource objects for the given name (may be empty)
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(DBSource).filter_by(source=source)
+
+ if version is not None:
+ q = q.filter_by(version=version)
+
+ if dm_upload_allowed is not None:
+ q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_sources_from_name')
+
+def get_source_in_suite(source, suite, session=None):
+ """
+ Returns list of DBSource objects for a combination of C{source} and C{suite}.
+
+ - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
+ - B{suite} - a suite name, eg. I{unstable}
+
+ @type source: string
+ @param source: source package name
+
+ @type suite: string
+ @param suite: the suite name
+
+ @rtype: string
+ @return: the version for I{source} in I{suite}
- suite_id = None
- if type(suite) == str:
- suite_id = self.get_suite_id(suite)
- elif type(suite) == int:
- suite_id = suite
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(SrcAssociation)
+ q = q.join('source').filter_by(source=source)
+ q = q.join('suite').filter_by(suite_name=suite)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ # ???: Maybe we should just return the SrcAssociation object instead
+ ret = q.one().source
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_source_in_suite')
+
+################################################################################
+
+class SrcAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
+
+__all__.append('SrcAssociation')
+
+################################################################################
+
+class SrcUploader(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SrcUploader %s>' % self.uploader_id
+
+__all__.append('SrcUploader')
+
+################################################################################
+
+SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
+ ('SuiteID', 'suite_id'),
+ ('Version', 'version'),
+ ('Origin', 'origin'),
+ ('Label', 'label'),
+ ('Description', 'description'),
+ ('Untouchable', 'untouchable'),
+ ('Announce', 'announce'),
+ ('Codename', 'codename'),
+ ('OverrideCodename', 'overridecodename'),
+ ('ValidTime', 'validtime'),
+ ('Priority', 'priority'),
+ ('NotAutomatic', 'notautomatic'),
+ ('CopyChanges', 'copychanges'),
+ ('CopyDotDak', 'copydotdak'),
+ ('CommentsDir', 'commentsdir'),
+ ('OverrideSuite', 'overridesuite'),
+ ('ChangelogBase', 'changelogbase')]
+
+
+class Suite(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Suite %s>' % self.suite_name
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.suite_name == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.suite_name != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def details(self):
+ ret = []
+ for disp, field in SUITE_FIELDS:
+ val = getattr(self, field, None)
+ if val is not None:
+ ret.append("%s: %s" % (disp, val))
+
+ return "\n".join(ret)
+
+__all__.append('Suite')
+
+def get_suite_architecture(suite, architecture, session=None):
+ """
+ Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
+ doesn't exist
+
+ @type suite: str
+ @param suite: Suite name to search for
+
+ @type architecture: str
+ @param architecture: Architecture name to search for
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: SuiteArchitecture
+ @return: the SuiteArchitecture object or None
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(SuiteArchitecture)
+ q = q.join(Architecture).filter_by(arch_string=architecture)
+ q = q.join(Suite).filter_by(suite_name=suite)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_suite_architecture')
+
+def get_suite(suite, session=None):
+ """
+ Returns Suite object for given C{suite name}.
+
+ @type suite: string
+ @param suite: The name of the suite
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Suite
+ @return: Suite object for the requested suite name (None if not presenT)
+
+ """
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Suite).filter_by(suite_name=suite)
+
+ if q.count() == 0:
+ ret = None
+ else:
+ ret = q.one()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_suite')
+
+################################################################################
+
+class SuiteArchitecture(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
+
+__all__.append('SuiteArchitecture')
+
+def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
+ """
+ Returns list of Architecture objects for given C{suite} name
+
+ @type source: str
+ @param source: Suite name to search for
+
+ @type skipsrc: boolean
+ @param skipsrc: Whether to skip returning the 'source' architecture entry
+ (Default False)
+
+ @type skipall: boolean
+ @param skipall: Whether to skip returning the 'all' architecture entry
+ (Default False)
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of Architecture objects for the given name (may be empty)
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Architecture)
+ q = q.join(SuiteArchitecture)
+ q = q.join(Suite).filter_by(suite_name=suite)
+
+ if skipsrc:
+ q = q.filter(Architecture.arch_string != 'source')
+
+ if skipall:
+ q = q.filter(Architecture.arch_string != 'all')
+
+ q = q.order_by('arch_string')
+
+ ret = q.all()
+
+ if privatetrans:
+ session.close()
+
+ return ret
+
+__all__.append('get_suite_architectures')
+
+################################################################################
+
+class Uid(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.uid == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.uid != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Uid %s (%s)>' % (self.uid, self.name)
+
+__all__.append('Uid')
+
+def add_database_user(uidname, session=None):
+ """
+ Adds a database user
+
+ @type uidname: string
+ @param uidname: The uid of the user to add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: Uid
+ @return: the uid object for the given uidname
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ session.execute("CREATE USER :uid", {'uid': uidname})
+
+ if privatetrans:
+ session.commit()
+ session.close()
+
+__all__.append('add_database_user')
+
+def get_or_set_uid(uidname, session=None):
+ """
+ Returns uid object for given uidname.
+
+ If no matching uidname is found, a row is inserted.
+
+ @type uidname: string
+ @param uidname: The uid to add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: Uid
+ @return: the uid object for the given uidname
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ q = session.query(Uid).filter_by(uid=uidname)
+
+ if q.count() < 1:
+ uid = Uid()
+ uid.uid = uidname
+ session.add(uid)
+ if privatetrans:
+ session.commit()
else:
- return None
+ session.flush()
+ ret = uid
+ else:
+ ret = q.one()
- c = self.db_con.cursor()
- c.execute( """SELECT a.arch_string FROM suite_architectures sa
- JOIN architecture a ON (a.id = sa.architecture)
- WHERE suite='%s'""" % suite_id )
+ if privatetrans:
+ session.close()
- return map(lambda x: x[0], c.fetchall())
+ return ret
- def insert_content_paths(self, bin_id, fullpaths):
- """
- Make sure given path is associated with given binary id
+__all__.append('get_or_set_uid')
- @type bin_id: int
- @param bin_id: the id of the binary
- @type fullpaths: list
- @param fullpaths: the list of paths of the file being associated with the binary
- @return: True upon success
- """
+def get_uid_from_fingerprint(fpr, session=None):
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
- c = self.db_con.cursor()
+ q = session.query(Uid)
+ q = q.join(Fingerprint).filter_by(fingerprint=fpr)
- c.execute("BEGIN WORK")
- try:
+ if q.count() != 1:
+ ret = None
+ else:
+ ret = q.one()
- for fullpath in fullpaths:
- (path, file) = os.path.split(fullpath)
+ if privatetrans:
+ session.close()
- if path.startswith( "./" ):
- path = path[2:]
- # Get the necessary IDs ...
- file_id = self.get_or_set_contents_file_id(file)
- path_id = self.get_or_set_contents_path_id(path)
+ return ret
- c.execute("""INSERT INTO content_associations
- (binary_pkg, filepath, filename)
- VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) )
+__all__.append('get_uid_from_fingerprint')
- c.execute("COMMIT")
- return True
- except:
- traceback.print_exc()
- c.execute("ROLLBACK")
- return False
+################################################################################
- def insert_pending_content_paths(self, package, fullpaths):
- """
- Make sure given paths are temporarily associated with given
- package
+class DBConn(Singleton):
+ """
+ database module init.
+ """
+ def __init__(self, *args, **kwargs):
+ super(DBConn, self).__init__(*args, **kwargs)
+
+ def _startup(self, *args, **kwargs):
+ self.debug = False
+ if kwargs.has_key('debug'):
+ self.debug = True
+ self.__createconn()
- @type package: dict
- @param package: the package to associate with should have been read in from the binary control file
- @type fullpaths: list
- @param fullpaths: the list of paths of the file being associated with the binary
+ def __setuptables(self):
+ self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
+ self.tbl_archive = Table('archive', self.db_meta, autoload=True)
+ self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
+ self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
+ self.tbl_component = Table('component', self.db_meta, autoload=True)
+ self.tbl_config = Table('config', self.db_meta, autoload=True)
+ self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
+ self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
+ self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
+ self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
+ self.tbl_files = Table('files', self.db_meta, autoload=True)
+ self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
+ self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
+ self.tbl_location = Table('location', self.db_meta, autoload=True)
+ self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
+ self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
+ self.tbl_override = Table('override', self.db_meta, autoload=True)
+ self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
+ self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
+ self.tbl_priority = Table('priority', self.db_meta, autoload=True)
+ self.tbl_queue = Table('queue', self.db_meta, autoload=True)
+ self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
+ self.tbl_section = Table('section', self.db_meta, autoload=True)
+ self.tbl_source = Table('source', self.db_meta, autoload=True)
+ self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
+ self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
+ self.tbl_suite = Table('suite', self.db_meta, autoload=True)
+ self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
+ self.tbl_uid = Table('uid', self.db_meta, autoload=True)
+
+ def __setupmappers(self):
+ mapper(Architecture, self.tbl_architecture,
+ properties = dict(arch_id = self.tbl_architecture.c.id))
+
+ mapper(Archive, self.tbl_archive,
+ properties = dict(archive_id = self.tbl_archive.c.id,
+ archive_name = self.tbl_archive.c.name))
+
+ mapper(BinAssociation, self.tbl_bin_associations,
+ properties = dict(ba_id = self.tbl_bin_associations.c.id,
+ suite_id = self.tbl_bin_associations.c.suite,
+ suite = relation(Suite),
+ binary_id = self.tbl_bin_associations.c.bin,
+ binary = relation(DBBinary)))
+
+ mapper(DBBinary, self.tbl_binaries,
+ properties = dict(binary_id = self.tbl_binaries.c.id,
+ package = self.tbl_binaries.c.package,
+ version = self.tbl_binaries.c.version,
+ maintainer_id = self.tbl_binaries.c.maintainer,
+ maintainer = relation(Maintainer),
+ source_id = self.tbl_binaries.c.source,
+ source = relation(DBSource),
+ arch_id = self.tbl_binaries.c.architecture,
+ architecture = relation(Architecture),
+ poolfile_id = self.tbl_binaries.c.file,
+ poolfile = relation(PoolFile),
+ binarytype = self.tbl_binaries.c.type,
+ fingerprint_id = self.tbl_binaries.c.sig_fpr,
+ fingerprint = relation(Fingerprint),
+ install_date = self.tbl_binaries.c.install_date,
+ binassociations = relation(BinAssociation,
+ primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
+
+ mapper(Component, self.tbl_component,
+ properties = dict(component_id = self.tbl_component.c.id,
+ component_name = self.tbl_component.c.name))
+
+ mapper(DBConfig, self.tbl_config,
+ properties = dict(config_id = self.tbl_config.c.id))
+
+ mapper(ContentAssociation, self.tbl_content_associations,
+ properties = dict(ca_id = self.tbl_content_associations.c.id,
+ filename_id = self.tbl_content_associations.c.filename,
+ filename = relation(ContentFilename),
+ filepath_id = self.tbl_content_associations.c.filepath,
+ filepath = relation(ContentFilepath),
+ binary_id = self.tbl_content_associations.c.binary_pkg,
+ binary = relation(DBBinary)))
+
+
+ mapper(ContentFilename, self.tbl_content_file_names,
+ properties = dict(cafilename_id = self.tbl_content_file_names.c.id,
+ filename = self.tbl_content_file_names.c.file))
+
+ mapper(ContentFilepath, self.tbl_content_file_paths,
+ properties = dict(cafilepath_id = self.tbl_content_file_paths.c.id,
+ filepath = self.tbl_content_file_paths.c.path))
+
+ mapper(DSCFile, self.tbl_dsc_files,
+ properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
+ source_id = self.tbl_dsc_files.c.source,
+ source = relation(DBSource),
+ poolfile_id = self.tbl_dsc_files.c.file,
+ poolfile = relation(PoolFile)))
+
+ mapper(PoolFile, self.tbl_files,
+ properties = dict(file_id = self.tbl_files.c.id,
+ filesize = self.tbl_files.c.size,
+ location_id = self.tbl_files.c.location,
+ location = relation(Location)))
+
+ mapper(Fingerprint, self.tbl_fingerprint,
+ properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
+ uid_id = self.tbl_fingerprint.c.uid,
+ uid = relation(Uid),
+ keyring_id = self.tbl_fingerprint.c.keyring,
+ keyring = relation(Keyring)))
+
+ mapper(Keyring, self.tbl_keyrings,
+ properties = dict(keyring_name = self.tbl_keyrings.c.name,
+ keyring_id = self.tbl_keyrings.c.id))
+
+ mapper(Location, self.tbl_location,
+ properties = dict(location_id = self.tbl_location.c.id,
+ component_id = self.tbl_location.c.component,
+ component = relation(Component),
+ archive_id = self.tbl_location.c.archive,
+ archive = relation(Archive),
+ archive_type = self.tbl_location.c.type))
+
+ mapper(Maintainer, self.tbl_maintainer,
+ properties = dict(maintainer_id = self.tbl_maintainer.c.id))
+
+ mapper(NewComment, self.tbl_new_comments,
+ properties = dict(comment_id = self.tbl_new_comments.c.id))
+
+ mapper(Override, self.tbl_override,
+ properties = dict(suite_id = self.tbl_override.c.suite,
+ suite = relation(Suite),
+ component_id = self.tbl_override.c.component,
+ component = relation(Component),
+ priority_id = self.tbl_override.c.priority,
+ priority = relation(Priority),
+ section_id = self.tbl_override.c.section,
+ section = relation(Section),
+ overridetype_id = self.tbl_override.c.type,
+ overridetype = relation(OverrideType)))
+
+ mapper(OverrideType, self.tbl_override_type,
+ properties = dict(overridetype = self.tbl_override_type.c.type,
+ overridetype_id = self.tbl_override_type.c.id))
+
+ mapper(PendingContentAssociation, self.tbl_pending_content_associations,
+ properties = dict(pca_id = self.tbl_pending_content_associations.c.id,
+ filepath_id = self.tbl_pending_content_associations.c.filepath,
+ filepath = relation(ContentFilepath),
+ filename_id = self.tbl_pending_content_associations.c.filename,
+ filename = relation(ContentFilename)))
+
+ mapper(Priority, self.tbl_priority,
+ properties = dict(priority_id = self.tbl_priority.c.id))
+
+ mapper(Queue, self.tbl_queue,
+ properties = dict(queue_id = self.tbl_queue.c.id))
+
+ mapper(QueueBuild, self.tbl_queue_build,
+ properties = dict(suite_id = self.tbl_queue_build.c.suite,
+ queue_id = self.tbl_queue_build.c.queue,
+ queue = relation(Queue, backref='queuebuild')))
+
+ mapper(Section, self.tbl_section,
+ properties = dict(section_id = self.tbl_section.c.id))
+
+ mapper(DBSource, self.tbl_source,
+ properties = dict(source_id = self.tbl_source.c.id,
+ version = self.tbl_source.c.version,
+ maintainer_id = self.tbl_source.c.maintainer,
+ maintainer = relation(Maintainer,
+ primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
+ poolfile_id = self.tbl_source.c.file,
+ poolfile = relation(PoolFile),
+ fingerprint_id = self.tbl_source.c.sig_fpr,
+ fingerprint = relation(Fingerprint),
+ changedby_id = self.tbl_source.c.changedby,
+ changedby = relation(Maintainer,
+ primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
+ srcfiles = relation(DSCFile,
+ primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
+ srcassociations = relation(SrcAssociation,
+ primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source))))
+
+ mapper(SrcAssociation, self.tbl_src_associations,
+ properties = dict(sa_id = self.tbl_src_associations.c.id,
+ suite_id = self.tbl_src_associations.c.suite,
+ suite = relation(Suite),
+ source_id = self.tbl_src_associations.c.source,
+ source = relation(DBSource)))
+
+ mapper(SrcUploader, self.tbl_src_uploaders,
+ properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
+ source_id = self.tbl_src_uploaders.c.source,
+ source = relation(DBSource,
+ primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
+ maintainer_id = self.tbl_src_uploaders.c.maintainer,
+ maintainer = relation(Maintainer,
+ primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
+
+ mapper(Suite, self.tbl_suite,
+ properties = dict(suite_id = self.tbl_suite.c.id))
+
+ mapper(SuiteArchitecture, self.tbl_suite_architectures,
+ properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
+ suite = relation(Suite, backref='suitearchitectures'),
+ arch_id = self.tbl_suite_architectures.c.architecture,
+ architecture = relation(Architecture)))
+
+ mapper(Uid, self.tbl_uid,
+ properties = dict(uid_id = self.tbl_uid.c.id,
+ fingerprint = relation(Fingerprint)))
+
+ ## Connection functions
+ def __createconn(self):
+ from config import Config
+ cnf = Config()
+ if cnf["DB::Host"]:
+ # TCP/IP
+ connstr = "postgres://%s" % cnf["DB::Host"]
+ if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
+ connstr += ":%s" % cnf["DB::Port"]
+ connstr += "/%s" % cnf["DB::Name"]
+ else:
+ # Unix Socket
+ connstr = "postgres:///%s" % cnf["DB::Name"]
+ if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
+ connstr += "?port=%s" % cnf["DB::Port"]
+
+ self.db_pg = create_engine(connstr, echo=self.debug)
+ self.db_meta = MetaData()
+ self.db_meta.bind = self.db_pg
+ self.db_smaker = sessionmaker(bind=self.db_pg,
+ autoflush=True,
+ autocommit=False)
+
+ self.__setuptables()
+ self.__setupmappers()
+
+ def session(self):
+ return self.db_smaker()
+
+__all__.append('DBConn')
- @return: True upon success
- """
- c = self.db_con.cursor()
-
- c.execute("BEGIN WORK")
- try:
- arch_id = self.get_architecture_id(package['Architecture'])
-
- # Remove any already existing recorded files for this package
- c.execute("""DELETE FROM pending_content_associations
- WHERE package=%(Package)s
- AND version=%(Version)s
- AND architecture=%(ArchID)s""", {'Package': package['Package'],
- 'Version': package['Version'],
- 'ArchID': arch_id})
-
- for fullpath in fullpaths:
- (path, file) = os.path.split(fullpath)
-
- if path.startswith( "./" ):
- path = path[2:]
- # Get the necessary IDs ...
- file_id = self.get_or_set_contents_file_id(file)
- path_id = self.get_or_set_contents_path_id(path)
-
- c.execute("""INSERT INTO pending_content_associations
- (package, version, architecture, filepath, filename)
- VALUES (%%(Package)s, %%(Version)s, '%d', '%d', '%d')"""
- % (arch_id, path_id, file_id), package )
-
- c.execute("COMMIT")
- return True
- except:
- traceback.print_exc()
- c.execute("ROLLBACK")
- return False
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Simple singleton class for storing info about Holding directory
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+import os
+from errno import ENOENT, EEXIST, EACCES
+import shutil
+
+from singleton import Singleton
+from config import Config
+from utils import fubar
+
+###############################################################################
+
+class Holding(Singleton):
+ def __init__(self, *args, **kwargs):
+ super(Holding, self).__init__(*args, **kwargs)
+
+ def _startup(self):
+ self.in_holding = {}
+ self.holding_dir = Config()["Dir::Queue::Holding"]
+
+ def copy_to_holding(self, filename):
+ base_filename = os.path.basename(filename)
+
+ dest = os.path.join(self.holding_dir, base_filename)
+ try:
+ fd = os.open(dest, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0640)
+ os.close(fd)
+ except OSError, e:
+ # Shouldn't happen, but will if, for example, someone lists a
+ # file twice in the .changes.
+ if e.errno == EEXIST:
+ return "%s: already exists in holding area; can not overwrite." % (base_filename)
+
+ try:
+ shutil.copy(filename, dest)
+ except IOError, e:
+ # In either case (ENOENT or EACCES) we want to remove the
+ # O_CREAT | O_EXCLed ghost file, so add the file to the list
+ # of 'in holding' even if it's not the real file.
+ if e.errno == ENOENT:
+ os.unlink(dest)
+ return "%s: can not copy to holding area: file not found." % (base_filename)
+
+ elif e.errno == EACCES:
+ os.unlink(dest)
+ return "%s: can not copy to holding area: read permission denied." % (base_filename)
+
+ self.in_holding[base_filename] = ""
+
+ return None
+
+ def clean(self):
+ cwd = os.getcwd()
+ os.chdir(self.holding_dir)
+ for f in self.in_holding.keys():
+ # TODO: Sanitize path in a much better manner...
+ if os.path.exists(f):
+ if f.find('/') != -1:
+ fubar("WTF? clean_holding() got a file ('%s') with / in it!" % (f))
+ else:
+ os.unlink(f)
+ self.in_holding = {}
+ os.chdir(cwd)
+
+++ /dev/null
-#!/usr/bin/env python
-
-"""
-Logging functions
-
-@contact: Debian FTP Master <ftpmaster@debian.org>
-@copyright: 2001, 2002, 2006 James Troup <james@nocrew.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-################################################################################
-
-import os
-import pwd
-import time
-import sys
-import utils
-
-################################################################################
-
-class Logger:
- "Logger object"
- Cnf = None
- logfile = None
- program = None
-
- def __init__ (self, Cnf, program, debug=0):
- "Initialize a new Logger object"
- self.Cnf = Cnf
- self.program = program
- # Create the log directory if it doesn't exist
- logdir = Cnf["Dir::Log"]
- if not os.path.exists(logdir):
- umask = os.umask(00000)
- os.makedirs(logdir, 02775)
- os.umask(umask)
- # Open the logfile
- logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m"))
- logfile = None
- if debug:
- logfile = sys.stderr
- else:
- umask = os.umask(00002)
- logfile = utils.open_file(logfilename, 'a')
- os.umask(umask)
- self.logfile = logfile
- # Log the start of the program
- user = pwd.getpwuid(os.getuid())[0]
- self.log(["program start", user])
-
- def log (self, details):
- "Log an event"
- # Prepend the timestamp and program name
- details.insert(0, self.program)
- timestamp = time.strftime("%Y%m%d%H%M%S")
- details.insert(0, timestamp)
- # Force the contents of the list to be string.join-able
- details = [ str(i) for i in details ]
- # Write out the log in TSV
- self.logfile.write("|".join(details)+'\n')
- # Flush the output to enable tail-ing
- self.logfile.flush()
-
- def close (self):
- "Close a Logger object"
- self.log(["program end"])
- self.logfile.flush()
- self.logfile.close()
import apt_inst
import apt_pkg
import utils
-import database
-from dak_exceptions import *
-from regexes import re_default_answer, re_fdnic, re_bin_only_nmu
-
+import commands
+import shutil
from types import *
+from dak_exceptions import *
+from changes import *
+from regexes import *
+from config import Config
+from holding import Holding
+from dbconn import *
+from summarystats import SummaryStats
+from utils import parse_changes
+from textutils import fix_maintainer
+from binary import Binary
+
###############################################################################
+def get_type(f, session):
+ """
+ Get the file type of C{f}
+
+ @type f: dict
+ @param f: file entry from Changes object
+
+ @type session: SQLA Session
+ @param session: SQL Alchemy session object
+
+ @rtype: string
+ @return: filetype
+
+ """
+ # Determine the type
+ if f.has_key("dbtype"):
+ file_type = file["dbtype"]
+ elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
+ file_type = "dsc"
+ else:
+ utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
+
+ # Validate the override type
+ type_id = get_override_type(file_type, session)
+ if type_id is None:
+ utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
+
+ return file_type
+
+################################################################################
+
# Determine what parts in a .changes are NEW
-def determine_new(changes, files, projectB, warn=1):
+def determine_new(changes, files, warn=1):
"""
Determine what parts in a C{changes} file are NEW.
@type files: Upload.Pkg.files dict
@param files: Files dictionary
- @type projectB: pgobject
- @param projectB: DB handle
-
@type warn: bool
@param warn: Warn if overrides are added for (old)stable
"""
new = {}
+ session = DBConn().session()
+
# Build up a list of potentially new things
- for file_entry in files.keys():
- f = files[file_entry]
+ for name, f in files.items():
# Skip byhand elements
if f["type"] == "byhand":
continue
pkg = f["package"]
priority = f["priority"]
section = f["section"]
- file_type = get_type(f)
+ file_type = get_type(f, session)
component = f["component"]
if file_type == "dsc":
priority = "source"
+
if not new.has_key(pkg):
new[pkg] = {}
new[pkg]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["type"] = file_type
new[pkg]["component"] = component
- new[pkg]["files"].append(file_entry)
+
+ new[pkg]["files"].append(name)
+
if f.has_key("othercomponents"):
new[pkg]["othercomponents"] = f["othercomponents"]
for suite in changes["suite"].keys():
- suite_id = database.get_suite_id(suite)
for pkg in new.keys():
- component_id = database.get_component_id(new[pkg]["component"])
- type_id = database.get_override_type_id(new[pkg]["type"])
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
- ql = q.getresult()
- if ql:
+ ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
+ if len(ql) > 0:
for file_entry in new[pkg]["files"]:
if files[file_entry].has_key("new"):
del files[file_entry]["new"]
del new[pkg]
if warn:
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!"
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!"
+ for s in ['stable', 'oldstable']:
+ if changes["suite"].has_key(s):
+ print "WARNING: overrides will be added for %s!" % s
for pkg in new.keys():
if new[pkg].has_key("othercomponents"):
print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
- return new
-
-################################################################################
-
-def get_type(file):
- """
- Get the file type of C{file}
-
- @type file: dict
- @param file: file entry
-
- @rtype: string
- @return: filetype
+ session.close()
- """
- # Determine the type
- if file.has_key("dbtype"):
- file_type = file["dbtype"]
- elif file["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
- file_type = "dsc"
- else:
- utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
-
- # Validate the override type
- type_id = database.get_override_type_id(file_type)
- if type_id == -1:
- utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
-
- return file_type
+ return new
################################################################################
-
-
def check_valid(new):
"""
Check if section and priority for NEW packages exist in database.
"""
for pkg in new.keys():
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
+ section_name = new[pkg]["section"]
+ priority_name = new[pkg]["priority"]
file_type = new[pkg]["type"]
- new[pkg]["section id"] = database.get_section_id(section)
- new[pkg]["priority id"] = database.get_priority_id(new[pkg]["priority"])
+
+ section = get_section(section_name)
+ if section is None:
+ new[pkg]["section id"] = -1
+ else:
+ new[pkg]["section id"] = section.section_id
+
+ priority = get_priority(priority_name)
+ if priority is None:
+ new[pkg]["priority id"] = -1
+ else:
+ new[pkg]["priority id"] = priority.priority_id
+
# Sanity checks
- di = section.find("debian-installer") != -1
- if (di and file_type not in ("udeb", "dsc")) or (not di and file_type == "udeb"):
+ di = section_name.find("debian-installer") != -1
+
+ # If d-i, we must be udeb and vice-versa
+ if (di and file_type not in ("udeb", "dsc")) or \
+ (not di and file_type == "udeb"):
new[pkg]["section id"] = -1
+
+ # If dsc we need to be source and vice-versa
if (priority == "source" and file_type != "dsc") or \
(priority != "source" and file_type == "dsc"):
new[pkg]["priority id"] = -1
+###############################################################################
+
+def lookup_uid_from_fingerprint(fpr, session):
+ uid = None
+ uid_name = ""
+ # This is a stupid default, but see the comments below
+ is_dm = False
+
+ user = get_uid_from_fingerprint(fpr, session)
+
+ if user is not None:
+ uid = user.uid
+ if user.name is None:
+ uid_name = ''
+ else:
+ uid_name = user.name
+
+ # Check the relevant fingerprint (which we have to have)
+ for f in user.fingerprint:
+ if f.fingerprint == fpr:
+ is_dm = f.keyring.debian_maintainer
+ break
+
+ return (uid, uid_name, is_dm)
###############################################################################
-class Pkg:
- """ Convenience wrapper to carry around all the package information """
- def __init__(self, **kwds):
- self.__dict__.update(kwds)
+# Used by Upload.check_timestamps
+class TarTime(object):
+ def __init__(self, future_cutoff, past_cutoff):
+ self.reset()
+ self.future_cutoff = future_cutoff
+ self.past_cutoff = past_cutoff
- def update(self, **kwds):
- self.__dict__.update(kwds)
+ def reset(self):
+ self.future_files = {}
+ self.ancient_files = {}
+
+ def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
+ if MTime > self.future_cutoff:
+ self.future_files[Name] = MTime
+ if MTime < self.past_cutoff:
+ self.ancient_files[Name] = MTime
###############################################################################
-class Upload:
+class Upload(object):
"""
Everything that has to do with an upload processed.
"""
- def __init__(self, Cnf):
- """
- Initialize various variables and the global substitution template mappings.
- Also connect to the DB and initialize the Database module.
-
- """
- self.Cnf = Cnf
- self.accept_count = 0
- self.accept_bytes = 0L
- self.reject_message = ""
- self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {})
-
- # Initialize the substitution template mapping global
- Subst = self.Subst = {}
- Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
- Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
- Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
- Subst["__DAK_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
-
- self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, self.projectB)
+ def __init__(self):
+ self.logger = None
+ self.pkg = Changes()
+ self.reset()
###########################################################################
- def init_vars (self):
- """ Reset a number of entries from our Pkg object. """
- self.pkg.changes.clear()
- self.pkg.dsc.clear()
- self.pkg.files.clear()
- self.pkg.dsc_files.clear()
- self.pkg.orig_tar_id = None
- self.pkg.orig_tar_location = ""
- self.pkg.orig_tar_gz = None
+ def reset (self):
+ """ Reset a number of internal variables."""
+
+ # Initialize the substitution template map
+ cnf = Config()
+ self.Subst = {}
+ self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
+ self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
+ self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+
+ self.rejects = []
+ self.warnings = []
+ self.notes = []
+
+ self.pkg.reset()
+
+ def package_info(self):
+ msg = ''
+
+ if len(self.rejects) > 0:
+ msg += "Reject Reasons:\n"
+ msg += "\n".join(self.rejects)
+
+ if len(self.warnings) > 0:
+ msg += "Warnings:\n"
+ msg += "\n".join(self.warnings)
+
+ if len(self.notes) > 0:
+ msg += "Notes:\n"
+ msg += "\n".join(self.notes)
+
+ return msg
###########################################################################
+ def update_subst(self):
+ """ Set up the per-package template substitution mappings """
+
+ cnf = Config()
- def update_vars (self):
+ # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
+ if not self.pkg.changes.has_key("architecture") or not \
+ isinstance(self.pkg.changes["architecture"], DictType):
+ self.pkg.changes["architecture"] = { "Unknown" : "" }
+
+ # and maintainer2047 may not exist.
+ if not self.pkg.changes.has_key("maintainer2047"):
+ self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
+
+ self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
+ self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
+ self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
+
+ # For source uploads the Changed-By field wins; otherwise Maintainer wins.
+ if self.pkg.changes["architecture"].has_key("source") and \
+ self.pkg.changes["changedby822"] != "" and \
+ (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
+
+ self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
+ self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
+ self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
+ else:
+ self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
+ self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
+ self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
+
+ if "sponsoremail" in self.pkg.changes:
+ self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
+
+ if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
+ self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
+
+ # Apply any global override of the Maintainer field
+ if cnf.get("Dinstall::OverrideMaintainer"):
+ self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
+ self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
+
+ self.Subst["__REJECT_MESSAGE__"] = self.package_info()
+ self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
+ self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
+
+ ###########################################################################
+ def load_changes(self, filename):
"""
- Update our Pkg object by reading a previously created cPickle .dak dumpfile.
+ @rtype boolean
+ @rvalue: whether the changes file was valid or not. We may want to
+ reject even if this is True (see what gets put in self.rejects).
+ This is simply to prevent us even trying things later which will
+ fail because we couldn't properly parse the file.
"""
- dump_filename = self.pkg.changes_file[:-8]+".dak"
- dump_file = utils.open_file(dump_filename)
- p = cPickle.Unpickler(dump_file)
+ Cnf = Config()
+ self.pkg.changes_file = filename
- self.pkg.changes.update(p.load())
- self.pkg.dsc.update(p.load())
- self.pkg.files.update(p.load())
- self.pkg.dsc_files.update(p.load())
+ # Parse the .changes field into a dictionary
+ try:
+ self.pkg.changes.update(parse_changes(filename))
+ except CantOpenError:
+ self.rejects.append("%s: can't read file." % (filename))
+ return False
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
+ return False
+ except ChangesUnicodeError:
+ self.rejects.append("%s: changes file not proper utf-8" % (filename))
+ return False
+
+ # Parse the Files field from the .changes into another dictionary
+ try:
+ self.pkg.files.update(utils.build_file_list(self.pkg.changes))
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
+ return False
+ except UnknownFormatError, format:
+ self.rejects.append("%s: unknown format '%s'." % (filename, format))
+ return False
+
+ # Check for mandatory fields
+ for i in ("distribution", "source", "binary", "architecture",
+ "version", "maintainer", "files", "changes", "description"):
+ if not self.pkg.changes.has_key(i):
+ # Avoid undefined errors later
+ self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
+ return False
+
+ # Strip a source version in brackets from the source field
+ if re_strip_srcver.search(self.pkg.changes["source"]):
+ self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
+
+ # Ensure the source field is a valid package name.
+ if not re_valid_pkg_name.match(self.pkg.changes["source"]):
+ self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
+
+ # Split multi-value fields into a lower-level dictionary
+ for i in ("architecture", "distribution", "binary", "closes"):
+ o = self.pkg.changes.get(i, "")
+ if o != "":
+ del self.pkg.changes[i]
+
+ self.pkg.changes[i] = {}
+
+ for j in o.split():
+ self.pkg.changes[i][j] = 1
+
+ # Fix the Maintainer: field to be RFC822/2047 compatible
+ try:
+ (self.pkg.changes["maintainer822"],
+ self.pkg.changes["maintainer2047"],
+ self.pkg.changes["maintainername"],
+ self.pkg.changes["maintaineremail"]) = \
+ fix_maintainer (self.pkg.changes["maintainer"])
+ except ParseMaintError, msg:
+ self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
+ % (filename, changes["maintainer"], msg))
+
+ # ...likewise for the Changed-By: field if it exists.
+ try:
+ (self.pkg.changes["changedby822"],
+ self.pkg.changes["changedby2047"],
+ self.pkg.changes["changedbyname"],
+ self.pkg.changes["changedbyemail"]) = \
+ fix_maintainer (self.pkg.changes.get("changed-by", ""))
+ except ParseMaintError, msg:
+ self.pkg.changes["changedby822"] = ""
+ self.pkg.changes["changedby2047"] = ""
+ self.pkg.changes["changedbyname"] = ""
+ self.pkg.changes["changedbyemail"] = ""
+
+ self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
+ % (filename, changes["changed-by"], msg))
+
+ # Ensure all the values in Closes: are numbers
+ if self.pkg.changes.has_key("closes"):
+ for i in self.pkg.changes["closes"].keys():
+ if re_isanum.match (i) == None:
+ self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
+
+ # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
+ self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
+ self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
+
+ # Check there isn't already a changes file of the same name in one
+ # of the queue directories.
+ base_filename = os.path.basename(filename)
+ for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
+ if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
+ self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
+
+ # Check the .changes is non-empty
+ if not self.pkg.files:
+ self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
+ return False
+
+ # Changes was syntactically valid even if we'll reject
+ return True
- self.pkg.orig_tar_id = p.load()
- self.pkg.orig_tar_location = p.load()
+ ###########################################################################
- dump_file.close()
+ def check_distributions(self):
+ "Check and map the Distribution field"
+
+ Cnf = Config()
+
+ # Handle suite mappings
+ for m in Cnf.ValueList("SuiteMappings"):
+ args = m.split()
+ mtype = args[0]
+ if mtype == "map" or mtype == "silent-map":
+ (source, dest) = args[1:3]
+ if self.pkg.changes["distribution"].has_key(source):
+ del self.pkg.changes["distribution"][source]
+ self.pkg.changes["distribution"][dest] = 1
+ if mtype != "silent-map":
+ self.notes.append("Mapping %s to %s." % (source, dest))
+ if self.pkg.changes.has_key("distribution-version"):
+ if self.pkg.changes["distribution-version"].has_key(source):
+ self.pkg.changes["distribution-version"][source]=dest
+ elif mtype == "map-unreleased":
+ (source, dest) = args[1:3]
+ if self.pkg.changes["distribution"].has_key(source):
+ for arch in self.pkg.changes["architecture"].keys():
+ if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
+ self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
+ del self.pkg.changes["distribution"][source]
+ self.pkg.changes["distribution"][dest] = 1
+ break
+ elif mtype == "ignore":
+ suite = args[1]
+ if self.pkg.changes["distribution"].has_key(suite):
+ del self.pkg.changes["distribution"][suite]
+ self.warnings.append("Ignoring %s as a target suite." % (suite))
+ elif mtype == "reject":
+ suite = args[1]
+ if self.pkg.changes["distribution"].has_key(suite):
+ self.rejects.append("Uploads to %s are not accepted." % (suite))
+ elif mtype == "propup-version":
+ # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
+ #
+ # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
+ if self.pkg.changes["distribution"].has_key(args[1]):
+ self.pkg.changes.setdefault("distribution-version", {})
+ for suite in args[2:]:
+ self.pkg.changes["distribution-version"][suite] = suite
+
+ # Ensure there is (still) a target distribution
+ if len(self.pkg.changes["distribution"].keys()) < 1:
+ self.rejects.append("No valid distribution remaining.")
+
+ # Ensure target distributions exist
+ for suite in self.pkg.changes["distribution"].keys():
+ if not Cnf.has_key("Suite::%s" % (suite)):
+ self.rejects.append("Unknown distribution `%s'." % (suite))
###########################################################################
+ def binary_file_checks(self, f, session):
+ cnf = Config()
+ entry = self.pkg.files[f]
- def dump_vars(self, dest_dir):
- """
- Dump our Pkg object into a cPickle file.
+ # Extract package control information
+ deb_file = utils.open_file(f)
+ try:
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
+ except:
+ self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
+ deb_file.close()
+ # Can't continue, none of the checks on control would work.
+ return
- @type dest_dir: string
- @param dest_dir: Path where the dumpfile should be stored
+ # Check for mandantory "Description:"
+ deb_file.seek(0)
+ try:
+ apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
+ except:
+ self.rejects.append("%s: Missing Description in binary package" % (f))
+ return
- @note: This could just dump the dictionaries as is, but I'd like to avoid this so
- there's some idea of what process-accepted & process-new use from
- process-unchecked. (JT)
+ deb_file.close()
+
+ # Check for mandatory fields
+ for field in [ "Package", "Architecture", "Version" ]:
+ if control.Find(field) == None:
+ # Can't continue
+ self.rejects.append("%s: No %s field in control." % (f, field))
+ return
+
+ # Ensure the package name matches the one give in the .changes
+ if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
+ self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
+
+ # Validate the package field
+ package = control.Find("Package")
+ if not re_valid_pkg_name.match(package):
+ self.rejects.append("%s: invalid package name '%s'." % (f, package))
+
+ # Validate the version field
+ version = control.Find("Version")
+ if not re_valid_version.match(version):
+ self.rejects.append("%s: invalid version number '%s'." % (f, version))
+
+ # Ensure the architecture of the .deb is one we know about.
+ default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
+ architecture = control.Find("Architecture")
+ upload_suite = self.pkg.changes["distribution"].keys()[0]
+
+ if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
+ and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
+ self.rejects.append("Unknown architecture '%s'." % (architecture))
+
+ # Ensure the architecture of the .deb is one of the ones
+ # listed in the .changes.
+ if not self.pkg.changes["architecture"].has_key(architecture):
+ self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
+
+ # Sanity-check the Depends field
+ depends = control.Find("Depends")
+ if depends == '':
+ self.rejects.append("%s: Depends field is empty." % (f))
+
+ # Sanity-check the Provides field
+ provides = control.Find("Provides")
+ if provides:
+ provide = re_spacestrip.sub('', provides)
+ if provide == '':
+ self.rejects.append("%s: Provides field is empty." % (f))
+ prov_list = provide.split(",")
+ for prov in prov_list:
+ if not re_valid_pkg_name.match(prov):
+ self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
+
+ # Check the section & priority match those given in the .changes (non-fatal)
+ if control.Find("Section") and entry["section"] != "" \
+ and entry["section"] != control.Find("Section"):
+ self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
+ (f, control.Find("Section", ""), entry["section"]))
+ if control.Find("Priority") and entry["priority"] != "" \
+ and entry["priority"] != control.Find("Priority"):
+ self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
+ (f, control.Find("Priority", ""), entry["priority"]))
+
+ entry["package"] = package
+ entry["architecture"] = architecture
+ entry["version"] = version
+ entry["maintainer"] = control.Find("Maintainer", "")
+
+ if f.endswith(".udeb"):
+ self.pkg.files[f]["dbtype"] = "udeb"
+ elif f.endswith(".deb"):
+ self.pkg.files[f]["dbtype"] = "deb"
+ else:
+ self.rejects.append("%s is neither a .deb or a .udeb." % (f))
+
+ entry["source"] = control.Find("Source", entry["package"])
+
+ # Get the source version
+ source = entry["source"]
+ source_version = ""
+
+ if source.find("(") != -1:
+ m = re_extract_src_version.match(source)
+ source = m.group(1)
+ source_version = m.group(2)
+
+ if not source_version:
+ source_version = self.pkg.files[f]["version"]
+
+ entry["source package"] = source
+ entry["source version"] = source_version
+
+ # Ensure the filename matches the contents of the .deb
+ m = re_isadeb.match(f)
+
+ # package name
+ file_package = m.group(1)
+ if entry["package"] != file_package:
+ self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
+ (f, file_package, entry["dbtype"], entry["package"]))
+ epochless_version = re_no_epoch.sub('', control.Find("Version"))
+
+ # version
+ file_version = m.group(2)
+ if epochless_version != file_version:
+ self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
+ (f, file_version, entry["dbtype"], epochless_version))
+
+ # architecture
+ file_architecture = m.group(3)
+ if entry["architecture"] != file_architecture:
+ self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
+ (f, file_architecture, entry["dbtype"], entry["architecture"]))
+
+ # Check for existent source
+ source_version = entry["source version"]
+ source_package = entry["source package"]
+ if self.pkg.changes["architecture"].has_key("source"):
+ if source_version != self.pkg.changes["version"]:
+ self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
+ (source_version, f, self.pkg.changes["version"]))
+ else:
+ # Check in the SQL database
+ if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
+ # Check in one of the other directories
+ source_epochless_version = re_no_epoch.sub('', source_version)
+ dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
+ if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
+ entry["byhand"] = 1
+ elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
+ entry["new"] = 1
+ else:
+ dsc_file_exists = False
+ for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
+ if cnf.has_key("Dir::Queue::%s" % (myq)):
+ if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
+ dsc_file_exists = True
+ break
+
+ if not dsc_file_exists:
+ self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
+
+ # Check the version and for file overwrites
+ self.check_binary_against_db(f, session)
+
+ b = Binary(f)
+ b.scan_package()
+ if len(b.rejects) > 0:
+ for j in b.rejects:
+ self.rejects.append(j)
+
+ def source_file_checks(self, f, session):
+ entry = self.pkg.files[f]
+
+ m = re_issource.match(f)
+ if not m:
+ return
- """
+ entry["package"] = m.group(1)
+ entry["version"] = m.group(2)
+ entry["type"] = m.group(3)
- changes = self.pkg.changes
- dsc = self.pkg.dsc
- files = self.pkg.files
- dsc_files = self.pkg.dsc_files
- orig_tar_id = self.pkg.orig_tar_id
- orig_tar_location = self.pkg.orig_tar_location
+ # Ensure the source package name matches the Source filed in the .changes
+ if self.pkg.changes["source"] != entry["package"]:
+ self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
- dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".dak")
- dump_file = utils.open_file(dump_filename, 'w')
- try:
- os.chmod(dump_filename, 0664)
- except OSError, e:
- # chmod may fail when the dumpfile is not owned by the user
- # invoking dak (like e.g. when NEW is processed by a member
- # of ftpteam)
- if errno.errorcode[e.errno] == 'EPERM':
- perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE])
- # security precaution, should never happen unless a weird
- # umask is set anywhere
- if perms & stat.S_IWOTH:
- utils.fubar("%s is world writable and chmod failed." % \
- (dump_filename,))
- # ignore the failed chmod otherwise as the file should
- # already have the right privileges and is just, at worst,
- # unreadable for world
+ # Ensure the source version matches the version in the .changes file
+ if entry["type"] == "orig.tar.gz":
+ changes_version = self.pkg.changes["chopversion2"]
+ else:
+ changes_version = self.pkg.changes["chopversion"]
+
+ if changes_version != entry["version"]:
+ self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
+
+ # Ensure the .changes lists source in the Architecture field
+ if not self.pkg.changes["architecture"].has_key("source"):
+ self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
+
+ # Check the signature of a .dsc file
+ if entry["type"] == "dsc":
+ # check_signature returns either:
+ # (None, [list, of, rejects]) or (signature, [])
+ (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
+ for j in rejects:
+ self.rejects.append(j)
+
+ entry["architecture"] = "source"
+
+ def per_suite_file_checks(self, f, suite, session):
+ cnf = Config()
+ entry = self.pkg.files[f]
+ archive = utils.where_am_i()
+
+ # Skip byhand
+ if entry.has_key("byhand"):
+ return
+
+ # Check we have fields we need to do these checks
+ oktogo = True
+ for m in ['component', 'package', 'priority', 'size', 'md5sum']:
+ if not entry.has_key(m):
+ self.rejects.append("file '%s' does not have field %s set" % (f, m))
+ oktogo = False
+
+ if not oktogo:
+ return
+
+ # Handle component mappings
+ for m in cnf.ValueList("ComponentMappings"):
+ (source, dest) = m.split()
+ if entry["component"] == source:
+ entry["original component"] = source
+ entry["component"] = dest
+
+ # Ensure the component is valid for the target suite
+ if cnf.has_key("Suite:%s::Components" % (suite)) and \
+ entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
+ self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
+ return
+
+ # Validate the component
+ if not get_component(entry["component"], session):
+ self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
+ return
+
+ # See if the package is NEW
+ if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
+ entry["new"] = 1
+
+ # Validate the priority
+ if entry["priority"].find('/') != -1:
+ self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
+
+ # Determine the location
+ location = cnf["Dir::Pool"]
+ l = get_location(location, entry["component"], archive, session)
+ if l is None:
+ self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
+ entry["location id"] = -1
+ else:
+ entry["location id"] = l.location_id
+
+ # Check the md5sum & size against existing files (if any)
+ entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
+
+ found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
+ entry["size"], entry["md5sum"], entry["location id"])
+
+ if found is None:
+ self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
+ elif found is False and poolfile is not None:
+ self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
+ else:
+ if poolfile is None:
+ entry["files id"] = None
else:
- raise
-
- p = cPickle.Pickler(dump_file, 1)
- d_changes = {}
- d_dsc = {}
- d_files = {}
- d_dsc_files = {}
-
- ## files
- for file_entry in files.keys():
- d_files[file_entry] = {}
- for i in [ "package", "version", "architecture", "type", "size",
- "md5sum", "sha1sum", "sha256sum", "component",
- "location id", "source package", "source version",
- "maintainer", "dbtype", "files id", "new",
- "section", "priority", "othercomponents",
- "pool name", "original component" ]:
- if files[file_entry].has_key(i):
- d_files[file_entry][i] = files[file_entry][i]
- ## changes
- # Mandatory changes fields
- for i in [ "distribution", "source", "architecture", "version",
- "maintainer", "urgency", "fingerprint", "changedby822",
- "changedby2047", "changedbyname", "maintainer822",
- "maintainer2047", "maintainername", "maintaineremail",
- "closes", "changes" ]:
- d_changes[i] = changes[i]
- # Optional changes fields
- for i in [ "changed-by", "filecontents", "format", "process-new note", "adv id", "distribution-version",
- "sponsoremail" ]:
- if changes.has_key(i):
- d_changes[i] = changes[i]
- ## dsc
- for i in [ "source", "version", "maintainer", "fingerprint",
- "uploaders", "bts changelog", "dm-upload-allowed" ]:
- if dsc.has_key(i):
- d_dsc[i] = dsc[i]
- ## dsc_files
- for file_entry in dsc_files.keys():
- d_dsc_files[file_entry] = {}
- # Mandatory dsc_files fields
- for i in [ "size", "md5sum" ]:
- d_dsc_files[file_entry][i] = dsc_files[file_entry][i]
- # Optional dsc_files fields
- for i in [ "files id" ]:
- if dsc_files[file_entry].has_key(i):
- d_dsc_files[file_entry][i] = dsc_files[file_entry][i]
-
- for i in [ d_changes, d_dsc, d_files, d_dsc_files,
- orig_tar_id, orig_tar_location ]:
- p.dump(i)
- dump_file.close()
+ entry["files id"] = poolfile.file_id
+
+ # Check for packages that have moved from one component to another
+ entry['suite'] = suite
+ res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
+ if res.rowcount > 0:
+ entry["othercomponents"] = res.fetchone()[0]
+
+ def check_files(self, action=True):
+ archive = utils.where_am_i()
+ file_keys = self.pkg.files.keys()
+ holding = Holding()
+ cnf = Config()
+
+ # XXX: As far as I can tell, this can no longer happen - see
+ # comments by AJ in old revisions - mhy
+ # if reprocess is 2 we've already done this and we're checking
+ # things again for the new .orig.tar.gz.
+ # [Yes, I'm fully aware of how disgusting this is]
+ if action and self.reprocess < 2:
+ cwd = os.getcwd()
+ os.chdir(self.pkg.directory)
+ for f in file_keys:
+ ret = holding.copy_to_holding(f)
+ if ret is not None:
+ # XXX: Should we bail out here or try and continue?
+ self.rejects.append(ret)
+
+ os.chdir(cwd)
+
+ # Check there isn't already a .changes or .dak file of the same name in
+ # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
+ # [NB: this check must be done post-suite mapping]
+ base_filename = os.path.basename(self.pkg.changes_file)
+ dot_dak_filename = base_filename[:-8] + ".dak"
+
+ for suite in self.pkg.changes["distribution"].keys():
+ copychanges = "Suite::%s::CopyChanges" % (suite)
+ if cnf.has_key(copychanges) and \
+ os.path.exists(os.path.join(cnf[copychanges], base_filename)):
+ self.rejects.append("%s: a file with this name already exists in %s" \
+ % (base_filename, cnf[copychanges]))
+
+ copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
+ if cnf.has_key(copy_dot_dak) and \
+ os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
+ self.rejects.append("%s: a file with this name already exists in %s" \
+ % (dot_dak_filename, Cnf[copy_dot_dak]))
+
+ self.reprocess = 0
+ has_binaries = False
+ has_source = False
+
+ session = DBConn().session()
+
+ for f, entry in self.pkg.files.items():
+ # Ensure the file does not already exist in one of the accepted directories
+ for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
+ if not cnf.has_key("Dir::Queue::%s" % (d)): continue
+ if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
+ self.rejects.append("%s file already exists in the %s directory." % (f, d))
+
+ if not re_taint_free.match(f):
+ self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
+
+ # Check the file is readable
+ if os.access(f, os.R_OK) == 0:
+ # When running in -n, copy_to_holding() won't have
+ # generated the reject_message, so we need to.
+ if action:
+ if os.path.exists(f):
+ self.rejects.append("Can't read `%s'. [permission denied]" % (f))
+ else:
+ self.rejects.append("Can't read `%s'. [file not found]" % (f))
+ entry["type"] = "unreadable"
+ continue
+
+ # If it's byhand skip remaining checks
+ if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
+ entry["byhand"] = 1
+ entry["type"] = "byhand"
+
+ # Checks for a binary package...
+ elif re_isadeb.match(f):
+ has_binaries = True
+ entry["type"] = "deb"
+
+ # This routine appends to self.rejects/warnings as appropriate
+ self.binary_file_checks(f, session)
+
+ # Checks for a source package...
+ elif re_issource.match(f):
+ has_source = True
+
+ # This routine appends to self.rejects/warnings as appropriate
+ self.source_file_checks(f, session)
+
+ # Not a binary or source package? Assume byhand...
+ else:
+ entry["byhand"] = 1
+ entry["type"] = "byhand"
+
+ # Per-suite file checks
+ entry["oldfiles"] = {}
+ for suite in self.pkg.changes["distribution"].keys():
+ self.per_suite_file_checks(f, suite, session)
+
+ session.close()
+
+ # If the .changes file says it has source, it must have source.
+ if self.pkg.changes["architecture"].has_key("source"):
+ if not has_source:
+ self.rejects.append("no source found and Architecture line in changes mention source.")
+
+ if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
+ self.rejects.append("source only uploads are not supported.")
###########################################################################
+ def check_dsc(self, action=True):
+ """Returns bool indicating whether or not the source changes are valid"""
+ # Ensure there is source to check
+ if not self.pkg.changes["architecture"].has_key("source"):
+ return True
+
+ # Find the .dsc
+ dsc_filename = None
+ for f, entry in self.pkg.files.items():
+ if entry["type"] == "dsc":
+ if dsc_filename:
+ self.rejects.append("can not process a .changes file with multiple .dsc's.")
+ return False
+ else:
+ dsc_filename = f
- # Set up the per-package template substitution mappings
+ # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
+ if not dsc_filename:
+ self.rejects.append("source uploads must contain a dsc file")
+ return False
- def update_subst (self, reject_message = ""):
- """ Set up the per-package template substitution mappings """
+ # Parse the .dsc file
+ try:
+ self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
+ except CantOpenError:
+ # if not -n copy_to_holding() will have done this for us...
+ if not action:
+ self.rejects.append("%s: can't read file." % (dsc_filename))
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
+ except InvalidDscError, line:
+ self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
+ except ChangesUnicodeError:
+ self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
+
+ # Build up the file list of files mentioned by the .dsc
+ try:
+ self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
+ except NoFilesFieldError:
+ self.rejects.append("%s: no Files: field." % (dsc_filename))
+ return False
+ except UnknownFormatError, format:
+ self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
+ return False
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
+ return False
+
+ # Enforce mandatory fields
+ for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
+ if not self.pkg.dsc.has_key(i):
+ self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
+ return False
+
+ # Validate the source and version fields
+ if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
+ self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
+ if not re_valid_version.match(self.pkg.dsc["version"]):
+ self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
+
+ # Bumping the version number of the .dsc breaks extraction by stable's
+ # dpkg-source. So let's not do that...
+ if self.pkg.dsc["format"] != "1.0":
+ self.rejects.append("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename))
+
+ # Validate the Maintainer field
+ try:
+ # We ignore the return value
+ fix_maintainer(self.pkg.dsc["maintainer"])
+ except ParseMaintError, msg:
+ self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
+ % (dsc_filename, self.pkg.dsc["maintainer"], msg))
+
+ # Validate the build-depends field(s)
+ for field_name in [ "build-depends", "build-depends-indep" ]:
+ field = self.pkg.dsc.get(field_name)
+ if field:
+ # Check for broken dpkg-dev lossage...
+ if field.startswith("ARRAY"):
+ self.rejects.append("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % \
+ (dsc_filename, field_name.title()))
+
+ # Have apt try to parse them...
+ try:
+ apt_pkg.ParseSrcDepends(field)
+ except:
+ self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
+
+ # Ensure the version number in the .dsc matches the version number in the .changes
+ epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
+ changes_version = self.pkg.files[dsc_filename]["version"]
+
+ if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
+ self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
+
+ # Ensure there is a .tar.gz in the .dsc file
+ has_tar = False
+ for f in self.pkg.dsc_files.keys():
+ m = re_issource.match(f)
+ if not m:
+ self.rejects.append("%s: %s in Files field not recognised as source." % (dsc_filename, f))
+ continue
+ ftype = m.group(3)
+ if ftype == "orig.tar.gz" or ftype == "tar.gz":
+ has_tar = True
- Subst = self.Subst
- changes = self.pkg.changes
- # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
- if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
- changes["architecture"] = { "Unknown" : "" }
- # and maintainer2047 may not exist.
- if not changes.has_key("maintainer2047"):
- changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"]
+ if not has_tar:
+ self.rejects.append("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
- Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys())
- Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
- Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "")
+ # Ensure source is newer than existing source in target suites
+ session = DBConn().session()
+ self.check_source_against_db(dsc_filename, session)
+ self.check_dsc_against_db(dsc_filename, session)
+ session.close()
- # For source uploads the Changed-By field wins; otherwise Maintainer wins.
- if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
- Subst["__MAINTAINER_FROM__"] = changes["changedby2047"]
- Subst["__MAINTAINER_TO__"] = "%s, %s" % (changes["changedby2047"],
- changes["maintainer2047"])
- Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown")
+ return True
+
+ ###########################################################################
+
+ def get_changelog_versions(self, source_dir):
+ """Extracts a the source package and (optionally) grabs the
+ version history out of debian/changelog for the BTS."""
+
+ cnf = Config()
+
+ # Find the .dsc (again)
+ dsc_filename = None
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f]["type"] == "dsc":
+ dsc_filename = f
+
+ # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
+ if not dsc_filename:
+ return
+
+ # Create a symlink mirror of the source files in our temporary directory
+ for f in self.pkg.files.keys():
+ m = re_issource.match(f)
+ if m:
+ src = os.path.join(source_dir, f)
+ # If a file is missing for whatever reason, give up.
+ if not os.path.exists(src):
+ return
+ ftype = m.group(3)
+ if ftype == "orig.tar.gz" and self.pkg.orig_tar_gz:
+ continue
+ dest = os.path.join(os.getcwd(), f)
+ os.symlink(src, dest)
+
+ # If the orig.tar.gz is not a part of the upload, create a symlink to the
+ # existing copy.
+ if self.pkg.orig_tar_gz:
+ dest = os.path.join(os.getcwd(), os.path.basename(self.pkg.orig_tar_gz))
+ os.symlink(self.pkg.orig_tar_gz, dest)
+
+ # Extract the source
+ cmd = "dpkg-source -sn -x %s" % (dsc_filename)
+ (result, output) = commands.getstatusoutput(cmd)
+ if (result != 0):
+ self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
+ self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
+ return
+
+ if not cnf.Find("Dir::Queue::BTSVersionTrack"):
+ return
+
+ # Get the upstream version
+ upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
+ if re_strip_revision.search(upstr_version):
+ upstr_version = re_strip_revision.sub('', upstr_version)
+
+ # Ensure the changelog file exists
+ changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
+ if not os.path.exists(changelog_filename):
+ self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
+ return
+
+ # Parse the changelog
+ self.pkg.dsc["bts changelog"] = ""
+ changelog_file = utils.open_file(changelog_filename)
+ for line in changelog_file.readlines():
+ m = re_changelog_versions.match(line)
+ if m:
+ self.pkg.dsc["bts changelog"] += line
+ changelog_file.close()
+
+ # Check we found at least one revision in the changelog
+ if not self.pkg.dsc["bts changelog"]:
+ self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
+
+ def check_source(self):
+ # XXX: I'm fairly sure reprocess == 2 can never happen
+ # AJT disabled the is_incoming check years ago - mhy
+ # We should probably scrap or rethink the whole reprocess thing
+ # Bail out if:
+ # a) there's no source
+ # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
+ # or c) the orig.tar.gz is MIA
+ if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
+ or self.pkg.orig_tar_gz == -1:
+ return
+
+ tmpdir = utils.temp_dirname()
+
+ # Move into the temporary directory
+ cwd = os.getcwd()
+ os.chdir(tmpdir)
+
+ # Get the changelog version history
+ self.get_changelog_versions(cwd)
+
+ # Move back and cleanup the temporary tree
+ os.chdir(cwd)
+
+ try:
+ shutil.rmtree(tmpdir)
+ except OSError, e:
+ if e.errno != errno.EACCES:
+ print "foobar"
+ utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
+
+ self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
+ # We probably have u-r or u-w directories so chmod everything
+ # and try again.
+ cmd = "chmod -R u+rwx %s" % (tmpdir)
+ result = os.system(cmd)
+ if result != 0:
+ utils.fubar("'%s' failed with result %s." % (cmd, result))
+ shutil.rmtree(tmpdir)
+ except Exception, e:
+ print "foobar2 (%s)" % e
+ utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
+
+ ###########################################################################
+ def ensure_hashes(self):
+ # Make sure we recognise the format of the Files: field in the .changes
+ format = self.pkg.changes.get("format", "0.0").split(".", 1)
+ if len(format) == 2:
+ format = int(format[0]), int(format[1])
else:
- Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"]
- Subst["__MAINTAINER_TO__"] = changes["maintainer2047"]
- Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown")
+ format = int(float(format[0])), 0
- if "sponsoremail" in changes:
- Subst["__MAINTAINER_TO__"] += ", %s"%changes["sponsoremail"]
+ # We need to deal with the original changes blob, as the fields we need
+ # might not be in the changes dict serialised into the .dak anymore.
+ orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
- if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
- Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
+ # Copy the checksums over to the current changes dict. This will keep
+ # the existing modifications to it intact.
+ for field in orig_changes:
+ if field.startswith('checksums-'):
+ self.pkg.changes[field] = orig_changes[field]
- # Apply any global override of the Maintainer field
- if self.Cnf.get("Dinstall::OverrideMaintainer"):
- Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"]
- Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"]
+ # Check for unsupported hashes
+ for j in utils.check_hash_fields(".changes", self.pkg.changes):
+ self.rejects.append(j)
+
+ for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
+ self.rejects.append(j)
+
+ # We have to calculate the hash if we have an earlier changes version than
+ # the hash appears in rather than require it exist in the changes file
+ for hashname, hashfunc, version in utils.known_hashes:
+ # TODO: Move _ensure_changes_hash into this class
+ for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
+ self.rejects.append(j)
+ if "source" in self.pkg.changes["architecture"]:
+ # TODO: Move _ensure_dsc_hash into this class
+ for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
+ self.rejects.append(j)
+
+ def check_hashes(self):
+ for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
+ self.rejects.append(m)
+
+ for m in utils.check_size(".changes", self.pkg.files):
+ self.rejects.append(m)
+
+ for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
+ self.rejects.append(m)
- Subst["__REJECT_MESSAGE__"] = reject_message
- Subst["__SOURCE__"] = changes.get("source", "Unknown")
- Subst["__VERSION__"] = changes.get("version", "Unknown")
+ for m in utils.check_size(".dsc", self.pkg.dsc_files):
+ self.rejects.append(m)
+
+ self.ensure_hashes()
+
+ ###########################################################################
+ def check_urgency(self):
+ cnf = Config()
+ if self.pkg.changes["architecture"].has_key("source"):
+ if not self.pkg.changes.has_key("urgency"):
+ self.pkg.changes["urgency"] = cnf["Urgency::Default"]
+ self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
+ if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
+ self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
+ (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
+ self.pkg.changes["urgency"] = cnf["Urgency::Default"]
###########################################################################
+ # Sanity check the time stamps of files inside debs.
+ # [Files in the near future cause ugly warnings and extreme time
+ # travel can cause errors on extraction]
+
+ def check_timestamps(self):
+ Cnf = Config()
+
+ future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
+ past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
+ tar = TarTime(future_cutoff, past_cutoff)
+
+ for filename, entry in self.pkg.files.items():
+ if entry["type"] == "deb":
+ tar.reset()
+ try:
+ deb_file = utils.open_file(filename)
+ apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
+ deb_file.seek(0)
+ try:
+ apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
+ except SystemError, e:
+ # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
+ if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
+ raise
+ deb_file.seek(0)
+ apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
+
+ deb_file.close()
+
+ future_files = tar.future_files.keys()
+ if future_files:
+ num_future_files = len(future_files)
+ future_file = future_files[0]
+ future_date = tar.future_files[future_file]
+ self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
+ % (filename, num_future_files, future_file, time.ctime(future_date)))
+
+ ancient_files = tar.ancient_files.keys()
+ if ancient_files:
+ num_ancient_files = len(ancient_files)
+ ancient_file = ancient_files[0]
+ ancient_date = tar.ancient_files[ancient_file]
+ self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
+ % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
+ except:
+ self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
+
+ ###########################################################################
+ def check_signed_by_key(self):
+ """Ensure the .changes is signed by an authorized uploader."""
+ session = DBConn().session()
+
+ (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
+
+ # match claimed name with actual name:
+ if uid is None:
+ # This is fundamentally broken but need us to refactor how we get
+ # the UIDs/Fingerprints in order for us to fix it properly
+ uid, uid_email = self.pkg.changes["fingerprint"], uid
+ may_nmu, may_sponsor = 1, 1
+ # XXX by default new dds don't have a fingerprint/uid in the db atm,
+ # and can't get one in there if we don't allow nmu/sponsorship
+ elif is_dm is False:
+ # If is_dm is False, we allow full upload rights
+ uid_email = "%s@debian.org" % (uid)
+ may_nmu, may_sponsor = 1, 1
+ else:
+ # Assume limited upload rights unless we've discovered otherwise
+ uid_email = uid
+ may_nmu, may_sponsor = 0, 0
+
+ if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
+ sponsored = 0
+ elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
+ sponsored = 0
+ if uid_name == "": sponsored = 1
+ else:
+ sponsored = 1
+ if ("source" in self.pkg.changes["architecture"] and
+ uid_email and utils.is_email_alias(uid_email)):
+ sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
+ if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
+ self.pkg.changes["changedbyemail"] not in sponsor_addresses):
+ self.pkg.changes["sponsoremail"] = uid_email
+
+ if sponsored and not may_sponsor:
+ self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
+
+ if not sponsored and not may_nmu:
+ should_reject = True
+ highest_sid, highest_version = None, None
+
+ # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
+ # It ignores higher versions with the dm_upload_allowed flag set to false
+ # I'm keeping the existing behaviour for now until I've gone back and
+ # checked exactly what the GR says - mhy
+ for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
+ if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
+ highest_sid = si.source_id
+ highest_version = si.version
+
+ if highest_sid is None:
+ self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
+ else:
+ for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
+ (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
+ if email == uid_email or name == uid_name:
+ should_reject = False
+ break
+
+ if should_reject is True:
+ self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
+
+ for b in self.pkg.changes["binary"].keys():
+ for suite in self.pkg.changes["distribution"].keys():
+ q = session.query(DBSource)
+ q = q.join(DBBinary).filter_by(package=b)
+ q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
+
+ for s in q.all():
+ if s.source != self.pkg.changes["source"]:
+ self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
+
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f].has_key("byhand"):
+ self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
+ if self.pkg.files[f].has_key("new"):
+ self.rejects.append("%s may not upload NEW file %s" % (uid, f))
+
+ session.close()
+
+ ###########################################################################
def build_summaries(self):
""" Build a summary of changes the upload introduces. """
- changes = self.pkg.changes
- files = self.pkg.files
-
- byhand = summary = new = ""
-
- # changes["distribution"] may not exist in corner cases
- # (e.g. unreadable changes files)
- if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
- changes["distribution"] = {}
-
- override_summary =""
- file_keys = files.keys()
- file_keys.sort()
- for file_entry in file_keys:
- if files[file_entry].has_key("byhand"):
- byhand = 1
- summary += file_entry + " byhand\n"
- elif files[file_entry].has_key("new"):
- new = 1
- summary += "(new) %s %s %s\n" % (file_entry, files[file_entry]["priority"], files[file_entry]["section"])
- if files[file_entry].has_key("othercomponents"):
- summary += "WARNING: Already present in %s distribution.\n" % (files[file_entry]["othercomponents"])
- if files[file_entry]["type"] == "deb":
- deb_fh = utils.open_file(file_entry)
- summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n'
- deb_fh.close()
- else:
- files[file_entry]["pool name"] = utils.poolify (changes.get("source",""), files[file_entry]["component"])
- destination = self.Cnf["Dir::PoolRoot"] + files[file_entry]["pool name"] + file_entry
- summary += file_entry + "\n to " + destination + "\n"
- if not files[file_entry].has_key("type"):
- files[file_entry]["type"] = "unknown"
- if files[file_entry]["type"] in ["deb", "udeb", "dsc"]:
- # (queue/unchecked), there we have override entries already, use them
- # (process-new), there we dont have override entries, use the newly generated ones.
- override_prio = files[file_entry].get("override priority", files[file_entry]["priority"])
- override_sect = files[file_entry].get("override section", files[file_entry]["section"])
- override_summary += "%s - %s %s\n" % (file_entry, override_prio, override_sect)
+
+ (byhand, new, summary, override_summary) = self.pkg.file_summary()
short_summary = summary
# This is for direport's benefit...
- f = re_fdnic.sub("\n .\n", changes.get("changes",""))
+ f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
if byhand or new:
summary += "Changes: " + f
###########################################################################
- def close_bugs (self, summary, action):
+ def close_bugs(self, summary, action):
"""
Send mail to close bugs as instructed by the closes field in the changes file.
Also add a line to summary if any work was done.
@return: summary. If action was taken, extended by the list of closed bugs.
"""
- changes = self.pkg.changes
- Subst = self.Subst
- Cnf = self.Cnf
- bugs = changes["closes"].keys()
+ template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
+
+ bugs = self.pkg.changes["closes"].keys()
if not bugs:
return summary
for bug in bugs:
summary += "%s " % (bug)
if action:
- Subst["__BUG_NUMBER__"] = bug
- if changes["distribution"].has_key("stable"):
- Subst["__STABLE_WARNING__"] = """
+ self.Subst["__BUG_NUMBER__"] = bug
+ if self.pkg.changes["distribution"].has_key("stable"):
+ self.Subst["__STABLE_WARNING__"] = """
Note that this package is not part of the released stable Debian
distribution. It may have dependencies on other unreleased software,
or other instabilities. Please take care if you wish to install it.
The update will eventually make its way into the next released Debian
distribution."""
else:
- Subst["__STABLE_WARNING__"] = ""
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.bug-close")
- utils.send_mail (mail_message)
- if action:
- self.Logger.log(["closing bugs"]+bugs)
+ self.Subst["__STABLE_WARNING__"] = ""
+ mail_message = utils.TemplateSubst(self.Subst, template)
+ utils.send_mail(mail_message)
+
+ # Clear up after ourselves
+ del self.Subst["__BUG_NUMBER__"]
+ del self.Subst["__STABLE_WARNING__"]
+
+ if action and self.logger:
+ self.logger.log(["closing bugs"] + bugs)
+
summary += "\n"
return summary
###########################################################################
- def announce (self, short_summary, action):
+ def announce(self, short_summary, action):
"""
Send an announce mail about a new upload.
@return: Textstring about action taken.
"""
- Subst = self.Subst
- Cnf = self.Cnf
- changes = self.pkg.changes
+
+ cnf = Config()
+ announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
# Only do announcements for source uploads with a recent dpkg-dev installed
- if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
+ if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
+ self.pkg.changes["architecture"].has_key("source"):
return ""
lists_done = {}
summary = ""
- Subst["__SHORT_SUMMARY__"] = short_summary
- for dist in changes["distribution"].keys():
- announce_list = Cnf.Find("Suite::%s::Announce" % (dist))
+ self.Subst["__SHORT_SUMMARY__"] = short_summary
+
+ for dist in self.pkg.changes["distribution"].keys():
+ announce_list = cnf.Find("Suite::%s::Announce" % (dist))
if announce_list == "" or lists_done.has_key(announce_list):
continue
+
lists_done[announce_list] = 1
summary += "Announcing to %s\n" % (announce_list)
if action:
- Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
- if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
- Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"])
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.announce")
- utils.send_mail (mail_message)
+ self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
+ if cnf.get("Dinstall::TrackingServer") and \
+ self.pkg.changes["architecture"].has_key("source"):
+ trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
+ self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
+
+ mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
+ utils.send_mail(mail_message)
+
+ del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
- if Cnf.FindB("Dinstall::CloseBugs"):
+ if cnf.FindB("Dinstall::CloseBugs"):
summary = self.close_bugs(summary, action)
+ del self.Subst["__SHORT_SUMMARY__"]
+
return summary
###########################################################################
"""
- Cnf = self.Cnf
- Subst = self.Subst
- files = self.pkg.files
- changes = self.pkg.changes
- changes_file = self.pkg.changes_file
- dsc = self.pkg.dsc
+ cnf = Config()
+ stats = SummaryStats()
+
+ accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
if targetdir is None:
- targetdir = Cnf["Dir::Queue::Accepted"]
+ targetdir = cnf["Dir::Queue::Accepted"]
print "Accepting."
- self.Logger.log(["Accepting changes",changes_file])
+ if self.logger:
+ self.logger.log(["Accepting changes", self.pkg.changes_file])
- self.dump_vars(targetdir)
+ self.pkg.write_dot_dak(targetdir)
# Move all the files into the accepted directory
- utils.move(changes_file, targetdir)
- file_keys = files.keys()
- for file_entry in file_keys:
- utils.move(file_entry, targetdir)
- self.accept_bytes += float(files[file_entry]["size"])
- self.accept_count += 1
+ utils.move(self.pkg.changes_file, targetdir)
+
+ for name, entry in sorted(self.pkg.files.items()):
+ utils.move(name, targetdir)
+ stats.accept_bytes += float(entry["size"])
+
+ stats.accept_count += 1
# Send accept mail, announce to lists, close bugs and check for
# override disparities
- if not Cnf["Dinstall::Options::No-Mail"]:
- Subst["__SUITE__"] = ""
- Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
+ if not cnf["Dinstall::Options::No-Mail"]:
+ self.Subst["__SUITE__"] = ""
+ self.Subst["__SUMMARY__"] = summary
+ mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
utils.send_mail(mail_message)
self.announce(short_summary, 1)
-
## Helper stuff for DebBugs Version Tracking
- if Cnf.Find("Dir::Queue::BTSVersionTrack"):
+ if cnf.Find("Dir::Queue::BTSVersionTrack"):
# ??? once queue/* is cleared on *.d.o and/or reprocessed
# the conditionalization on dsc["bts changelog"] should be
# dropped.
# Write out the version history from the changelog
- if changes["architecture"].has_key("source") and \
- dsc.has_key("bts changelog"):
+ if self.pkg.changes["architecture"].has_key("source") and \
+ self.pkg.dsc.has_key("bts changelog"):
- (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
version_history = os.fdopen(fd, 'w')
- version_history.write(dsc["bts changelog"])
+ version_history.write(self.pkg.dsc["bts changelog"])
version_history.close()
- filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
- changes_file[:-8]+".versions")
+ filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+ self.pkg.changes_file[:-8]+".versions")
os.rename(temp_filename, filename)
os.chmod(filename, 0644)
# Write out the binary -> source mapping.
- (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
debinfo = os.fdopen(fd, 'w')
- for file_entry in file_keys:
- f = files[file_entry]
- if f["type"] == "deb":
- line = " ".join([f["package"], f["version"],
- f["architecture"], f["source package"],
- f["source version"]])
+ for name, entry in sorted(self.pkg.files.items()):
+ if entry["type"] == "deb":
+ line = " ".join([entry["package"], entry["version"],
+ entry["architecture"], entry["source package"],
+ entry["source version"]])
debinfo.write(line+"\n")
debinfo.close()
- filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
- changes_file[:-8]+".debinfo")
+ filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+ self.pkg.changes_file[:-8]+".debinfo")
os.rename(temp_filename, filename)
os.chmod(filename, 0644)
# <Ganneff> so it will work out, as unchecked move it over
# <mhy> that's all completely sick
# <Ganneff> yes
- self.queue_build("accepted", Cnf["Dir::Queue::Accepted"])
- ###########################################################################
-
- def queue_build (self, queue, path):
- """
- Prepare queue_build database table used for incoming autobuild support.
+ # This routine returns None on success or an error on failure
+ res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
+ if res:
+ utils.fubar(res)
- @type queue: string
- @param queue: queue name
- @type path: string
- @param path: path for the queue file entries/link destinations
- """
-
- Cnf = self.Cnf
- Subst = self.Subst
- files = self.pkg.files
- changes = self.pkg.changes
- changes_file = self.pkg.changes_file
- dsc = self.pkg.dsc
- file_keys = files.keys()
-
- ## Special support to enable clean auto-building of queued packages
- queue_id = database.get_or_set_queue_id(queue)
-
- self.projectB.query("BEGIN WORK")
- for suite in changes["distribution"].keys():
- if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
- continue
- suite_id = database.get_suite_id(suite)
- dest_dir = Cnf["Dir::QueueBuild"]
- if Cnf.FindB("Dinstall::SecurityQueueBuild"):
- dest_dir = os.path.join(dest_dir, suite)
- for file_entry in file_keys:
- src = os.path.join(path, file_entry)
- dest = os.path.join(dest_dir, file_entry)
- if Cnf.FindB("Dinstall::SecurityQueueBuild"):
- # Copy it since the original won't be readable by www-data
- utils.copy(src, dest)
- else:
- # Create a symlink to it
- os.symlink(src, dest)
- # Add it to the list of packages for later processing by apt-ftparchive
- self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest))
- # If the .orig.tar.gz is in the pool, create a symlink to
- # it (if one doesn't already exist)
- if self.pkg.orig_tar_id:
- # Determine the .orig.tar.gz file name
- for dsc_file in self.pkg.dsc_files.keys():
- if dsc_file.endswith(".orig.tar.gz"):
- filename = dsc_file
- dest = os.path.join(dest_dir, filename)
- # If it doesn't exist, create a symlink
- if not os.path.exists(dest):
- # Find the .orig.tar.gz in the pool
- q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id))
- ql = q.getresult()
- if not ql:
- utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id))
- src = os.path.join(ql[0][0], ql[0][1])
- os.symlink(src, dest)
- # Add it to the list of packages for later processing by apt-ftparchive
- self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest))
- # if it does, update things to ensure it's not removed prematurely
- else:
- self.projectB.query("UPDATE queue_build SET in_queue = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id))
-
- self.projectB.query("COMMIT WORK")
-
- ###########################################################################
-
- def check_override (self):
+ def check_override(self):
"""
Checks override entries for validity. Mails "Override disparity" warnings,
if that feature is enabled.
Abandons the check if
- - this is a non-sourceful upload
- override disparity checks are disabled
- mail sending is disabled
-
"""
- Subst = self.Subst
- changes = self.pkg.changes
- files = self.pkg.files
- Cnf = self.Cnf
+
+ cnf = Config()
# Abandon the check if:
- # a) it's a non-sourceful upload
- # b) override disparity checks have been disabled
- # c) we're not sending mail
- if not changes["architecture"].has_key("source") or \
- not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
- Cnf["Dinstall::Options::No-Mail"]:
+ # a) override disparity checks have been disabled
+ # b) we're not sending mail
+ if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
+ cnf["Dinstall::Options::No-Mail"]:
return
- summary = ""
- file_keys = files.keys()
- file_keys.sort()
- for file_entry in file_keys:
- if not files[file_entry].has_key("new") and files[file_entry]["type"] == "deb":
- section = files[file_entry]["section"]
- override_section = files[file_entry]["override section"]
- if section.lower() != override_section.lower() and section != "-":
- summary += "%s: package says section is %s, override says %s.\n" % (file_entry, section, override_section)
- priority = files[file_entry]["priority"]
- override_priority = files[file_entry]["override priority"]
- if priority != override_priority and priority != "-":
- summary += "%s: package says priority is %s, override says %s.\n" % (file_entry, priority, override_priority)
+ summary = self.pkg.check_override()
if summary == "":
return
- Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/process-unchecked.override-disparity")
+ overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
+
+ self.Subst["__SUMMARY__"] = summary
+ mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
utils.send_mail(mail_message)
+ del self.Subst["__SUMMARY__"]
+
+ ###########################################################################
+
+ def remove(self, dir=None):
+ """
+ Used (for instance) in p-u to remove the package from unchecked
+ """
+ if dir is None:
+ os.chdir(self.pkg.directory)
+ else:
+ os.chdir(dir)
+
+ for f in self.pkg.files.keys():
+ os.unlink(f)
+ os.unlink(self.pkg.changes_file)
+
+ ###########################################################################
+
+ def move_to_dir (self, dest, perms=0660, changesperms=0664):
+ """
+ Move files to dest with certain perms/changesperms
+ """
+ utils.move(self.pkg.changes_file, dest, perms=changesperms)
+ for f in self.pkg.files.keys():
+ utils.move(f, dest, perms=perms)
###########################################################################
- def force_reject (self, files):
+ def force_reject(self, reject_files):
"""
Forcefully move files from the current directory to the
reject directory. If any file already exists in the reject
"""
- Cnf = self.Cnf
+ cnf = Config()
- for file_entry in files:
+ for file_entry in reject_files:
# Skip any files which don't exist or which we don't have permission to copy.
- if os.access(file_entry,os.R_OK) == 0:
+ if os.access(file_entry, os.R_OK) == 0:
continue
- dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file_entry)
+
+ dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
+
try:
- dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
except OSError, e:
# File exists? Let's try and move it to the morgue
- if errno.errorcode[e.errno] == 'EEXIST':
- morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file_entry)
+ if e.errno == errno.EEXIST:
+ morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
try:
morgue_file = utils.find_next_free(morgue_file)
except NoFreeFilenameError:
os.close(dest_fd)
###########################################################################
-
- def do_reject (self, manual = 0, reject_message = "", note = ""):
+ def do_reject (self, manual=0, reject_message="", note=""):
"""
Reject an upload. If called without a reject message or C{manual} is
true, spawn an editor so the user can write one.
print "Rejecting.\n"
- Cnf = self.Cnf
- Subst = self.Subst
- pkg = self.pkg
+ cnf = Config()
- reason_filename = pkg.changes_file[:-8] + ".reason"
- reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename
+ reason_filename = self.pkg.changes_file[:-8] + ".reason"
+ reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
# Move all the files into the reject directory
- reject_files = pkg.files.keys() + [pkg.changes_file]
+ reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
self.force_reject(reject_files)
# If we fail here someone is probably trying to exploit the race
os.unlink(reason_filename)
reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
+
if not manual:
- Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
- Subst["__MANUAL_REJECT_MESSAGE__"] = ""
- Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
+ self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
+ self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
os.write(reason_fd, reject_message)
- reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/queue.rejected")
+ reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
else:
# Build up the rejection email
- user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"])
-
- Subst["__REJECTOR_ADDRESS__"] = user_email_address
- Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
- reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/queue.rejected")
+ user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
+ self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
+ self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
+ self.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
+ reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
# Write the rejection email out as the <foo>.reason file
os.write(reason_fd, reject_mail_message)
+ del self.Subst["__REJECTOR_ADDRESS__"]
+ del self.Subst["__MANUAL_REJECT_MESSAGE__"]
+ del self.Subst["__CC__"]
+
os.close(reason_fd)
# Send the rejection mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
+ if not cnf["Dinstall::Options::No-Mail"]:
utils.send_mail(reject_mail_message)
- self.Logger.log(["rejected", pkg.changes_file])
- return 0
-
- ################################################################################
-
- def source_exists (self, package, source_version, suites = ["any"]):
- """
- Ensure that source exists somewhere in the archive for the binary
- upload being processed.
- 1. exact match => 1.0-3
- 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
-
- @type package: string
- @param package: package source name
-
- @type source_version: string
- @param source_version: expected source version
-
- @type suites: list
- @param suites: list of suites to check in, default I{any}
-
- @rtype: int
- @return: returns 1 if a source with expected version is found, otherwise 0
+ if self.logger:
+ self.logger.log(["rejected", self.pkg.changes_file])
- """
- okay = 1
- for suite in suites:
- if suite == "any":
- que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
- (package)
- else:
- # source must exist in suite X, or in some other suite that's
- # mapped to X, recursively... silent-maps are counted too,
- # unreleased-maps aren't.
- maps = self.Cnf.ValueList("SuiteMappings")[:]
- maps.reverse()
- maps = [ m.split() for m in maps ]
- maps = [ (x[1], x[2]) for x in maps
- if x[0] == "map" or x[0] == "silent-map" ]
- s = [suite]
- for x in maps:
- if x[1] in s and x[0] not in s:
- s.append(x[0])
-
- que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, " OR ".join(["su.suite_name = '%s'" % a for a in s]))
- q = self.projectB.query(que)
-
- # Reduce the query results to a list of version numbers
- ql = [ i[0] for i in q.getresult() ]
-
- # Try (1)
- if source_version in ql:
- continue
-
- # Try (2)
- orig_source_version = re_bin_only_nmu.sub('', source_version)
- if orig_source_version in ql:
- continue
-
- # No source found...
- okay = 0
- break
- return okay
+ return 0
################################################################################
-
- def in_override_p (self, package, component, suite, binary_type, file):
+ def in_override_p(self, package, component, suite, binary_type, file, session):
"""
Check if a package already has override entries in the DB
@param package: package name
@type component: string
- @param component: database id of the component, as returned by L{database.get_component_id}
+ @param component: database id of the component
@type suite: int
- @param suite: database id of the suite, as returned by L{database.get_suite_id}
+ @param suite: database id of the suite
@type binary_type: string
@param binary_type: type of the package
@return: the database result. But noone cares anyway.
"""
- files = self.pkg.files
+
+ cnf = Config()
if binary_type == "": # must be source
file_type = "dsc"
file_type = binary_type
# Override suite name; used for example with proposed-updates
- if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
- suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)]
-
- # Avoid <undef> on unknown distributions
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
- return None
- component_id = database.get_component_id(component)
- type_id = database.get_override_type_id(file_type)
-
- q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
- % (package, suite_id, component_id, type_id))
- result = q.getresult()
+ if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
+ suite = cnf["Suite::%s::OverrideSuite" % (suite)]
+
+ result = get_override(package, suite, component, file_type, session)
+
# If checking for a source package fall back on the binary override type
- if file_type == "dsc" and not result:
- deb_type_id = database.get_override_type_id("deb")
- udeb_type_id = database.get_override_type_id("udeb")
- q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
- % (package, suite_id, component_id, deb_type_id, udeb_type_id))
- result = q.getresult()
+ if file_type == "dsc" and len(result) < 1:
+ result = get_override(package, suite, component, ['deb', 'udeb'], session)
# Remember the section and priority so we can check them later if appropriate
- if result:
- files[file]["override section"] = result[0][0]
- files[file]["override priority"] = result[0][1]
+ if len(result) > 0:
+ result = result[0]
+ self.pkg.files[file]["override section"] = result.section.section
+ self.pkg.files[file]["override priority"] = result.priority.priority
+ return result
- return result
+ return None
################################################################################
-
- def reject (self, str, prefix="Rejected: "):
+ def get_anyversion(self, sv_list, suite):
"""
- Add C{str} to reject_message. Adds C{prefix}, by default "Rejected: "
-
- @type str: string
- @param str: Reject text
+ @type sv_list: list
+ @param sv_list: list of (suite, version) tuples to check
- @type prefix: string
- @param prefix: Prefix text, default Rejected:
+ @type suite: string
+ @param suite: suite name
+ Description: TODO
"""
- if str:
- # Unlike other rejects we add new lines first to avoid trailing
- # new lines when this message is passed back up to a caller.
- if self.reject_message:
- self.reject_message += "\n"
- self.reject_message += prefix + str
-
- ################################################################################
-
- def get_anyversion(self, query_result, suite):
- """ """
- anyversion=None
- anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
- for (v, s) in query_result:
+ Cnf = Config()
+ anyversion = None
+ anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
+ for (s, v) in sv_list:
if s in [ x.lower() for x in anysuite ]:
if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
- anyversion=v
+ anyversion = v
+
return anyversion
################################################################################
- def cross_suite_version_check(self, query_result, file, new_version,
- sourceful=False):
+ def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
"""
+ @type sv_list: list
+ @param sv_list: list of (suite, version) tuples to check
+
+ @type file: string
+ @param file: XXX
+
+ @type new_version: string
+ @param new_version: XXX
+
Ensure versions are newer than existing packages in target
suites and that cross-suite version checking rules as
set out in the conf file are satisfied.
-
"""
+ cnf = Config()
+
# Check versions for each target suite
for target_suite in self.pkg.changes["distribution"].keys():
- must_be_newer_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
- must_be_older_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+ must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
+ must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+
# Enforce "must be newer than target suite" even if conffile omits it
if target_suite not in must_be_newer_than:
must_be_newer_than.append(target_suite)
- for entry in query_result:
- existent_version = entry[0]
- suite = entry[1]
- if suite in must_be_newer_than and sourceful and \
- apt_pkg.VersionCompare(new_version, existent_version) < 1:
- self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
- if suite in must_be_older_than and \
- apt_pkg.VersionCompare(new_version, existent_version) > -1:
- ch = self.pkg.changes
+
+ for (suite, existent_version) in sv_list:
+ vercmp = apt_pkg.VersionCompare(new_version, existent_version)
+
+ if suite in must_be_newer_than and sourceful and vercmp < 1:
+ self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+
+ if suite in must_be_older_than and vercmp > -1:
cansave = 0
- if ch.get('distribution-version', {}).has_key(suite):
- # we really use the other suite, ignoring the conflicting one ...
- addsuite = ch["distribution-version"][suite]
- add_version = self.get_anyversion(query_result, addsuite)
- target_version = self.get_anyversion(query_result, target_suite)
+ if self.pkg.changes.get('distribution-version', {}).has_key(suite):
+ # we really use the other suite, ignoring the conflicting one ...
+ addsuite = self.pkg.changes["distribution-version"][suite]
+
+ add_version = self.get_anyversion(sv_list, addsuite)
+ target_version = self.get_anyversion(sv_list, target_suite)
if not add_version:
# not add_version can only happen if we map to a suite
# than complaining. either way, this isn't a REJECT issue
#
# And - we really should complain to the dorks who configured dak
- self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ")
+ self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
self.pkg.changes.setdefault("propdistribution", {})
self.pkg.changes["propdistribution"][addsuite] = 1
cansave = 1
# not targets_version is true when the package is NEW
# we could just stick with the "...old version..." REJECT
# for this, I think.
- self.reject("Won't propogate NEW packages.")
+ self.rejects.append("Won't propogate NEW packages.")
elif apt_pkg.VersionCompare(new_version, add_version) < 0:
# propogation would be redundant. no need to reject though.
- self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ")
+ self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
cansave = 1
elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
apt_pkg.VersionCompare(add_version, target_version) >= 0:
# propogate!!
- self.reject("Propogating upload to %s" % (addsuite), "Warning: ")
+ self.warnings.append("Propogating upload to %s" % (addsuite))
self.pkg.changes.setdefault("propdistribution", {})
self.pkg.changes["propdistribution"][addsuite] = 1
cansave = 1
if not cansave:
- self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
################################################################################
-
- def check_binary_against_db(self, file):
- """
-
- """
- self.reject_message = ""
- files = self.pkg.files
-
+ def check_binary_against_db(self, file, session):
# Ensure version is sane
- q = self.projectB.query("""
-SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
- architecture a
- WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
- AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
- % (files[file]["package"],
- files[file]["architecture"]))
- self.cross_suite_version_check(q.getresult(), file,
- files[file]["version"], sourceful=False)
+ q = session.query(BinAssociation)
+ q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
+ q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
+
+ self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
+ file, self.pkg.files[file]["version"], sourceful=False)
# Check for any existing copies of the file
- q = self.projectB.query("""
-SELECT b.id FROM binaries b, architecture a
- WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
- AND a.id = b.architecture"""
- % (files[file]["package"],
- files[file]["version"],
- files[file]["architecture"]))
- if q.getresult():
- self.reject("%s: can not overwrite existing copy already in the archive." % (file))
-
- return self.reject_message
+ q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
+ q = q.filter_by(version=self.pkg.files[file]["version"])
+ q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
+
+ if q.count() > 0:
+ self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
################################################################################
- def check_source_against_db(self, file):
+ def check_source_against_db(self, file, session):
"""
"""
- self.reject_message = ""
- dsc = self.pkg.dsc
+ source = self.pkg.dsc.get("source")
+ version = self.pkg.dsc.get("version")
# Ensure version is sane
- q = self.projectB.query("""
-SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
- WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")))
- self.cross_suite_version_check(q.getresult(), file, dsc.get("version"),
- sourceful=True)
+ q = session.query(SrcAssociation)
+ q = q.join(DBSource).filter(DBSource.source==source)
- return self.reject_message
+ self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
+ file, version, sourceful=True)
################################################################################
-
-
- def check_dsc_against_db(self, file):
+ def check_dsc_against_db(self, file, session):
"""
@warning: NB: this function can remove entries from the 'files' index [if
ensure you haven't just tried to dereference the deleted entry.
"""
- self.reject_message = ""
- files = self.pkg.files
- dsc_files = self.pkg.dsc_files
+
+ Cnf = Config()
self.pkg.orig_tar_gz = None
# Try and find all files mentioned in the .dsc. This has
# locations of an .orig.tar.gz.
# The ordering on the select is needed to pick the newest orig
# when it exists in multiple places.
- for dsc_file in dsc_files.keys():
+ for dsc_name, dsc_entry in self.pkg.dsc_files.items():
found = None
- if files.has_key(dsc_file):
- actual_md5 = files[dsc_file]["md5sum"]
- actual_size = int(files[dsc_file]["size"])
- found = "%s in incoming" % (dsc_file)
+ if self.pkg.files.has_key(dsc_name):
+ actual_md5 = self.pkg.files[dsc_name]["md5sum"]
+ actual_size = int(self.pkg.files[dsc_name]["size"])
+ found = "%s in incoming" % (dsc_name)
+
# Check the file does not already exist in the archive
- q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location ORDER BY f.id DESC" % (dsc_file))
- ql = q.getresult()
+ ql = get_poolfile_like_name(dsc_name, session)
+
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
- if i[3] != dsc_file and i[3][-(len(dsc_file)+1):] != '/'+dsc_file:
+ if not i.filename.endswith(dsc_name):
ql.remove(i)
# "[dak] has not broken them. [dak] has fixed a
# the same name and version.)"
# -- ajk@ on d-devel@l.d.o
- if ql:
+ if len(ql) > 0:
# Ignore exact matches for .orig.tar.gz
match = 0
- if dsc_file.endswith(".orig.tar.gz"):
+ if dsc_name.endswith(".orig.tar.gz"):
for i in ql:
- if files.has_key(dsc_file) and \
- int(files[dsc_file]["size"]) == int(i[0]) and \
- files[dsc_file]["md5sum"] == i[1]:
- self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ")
- del files[dsc_file]
- self.pkg.orig_tar_gz = i[2] + i[3]
+ if self.pkg.files.has_key(dsc_name) and \
+ int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
+ self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
+ self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
+ # TODO: Don't delete the entry, just mark it as not needed
+ # This would fix the stupidity of changing something we often iterate over
+ # whilst we're doing it
+ del self.pkg.files[dsc_name]
+ self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename)
match = 1
if not match:
- self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file))
- elif dsc_file.endswith(".orig.tar.gz"):
+ self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
+
+ elif dsc_name.endswith(".orig.tar.gz"):
# Check in the pool
- q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file))
- ql = q.getresult()
+ ql = get_poolfile_like_name(dsc_name, session)
+
# Strip out anything that isn't '%s' or '/%s$'
+ # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
for i in ql:
- if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
+ if not i.filename.endswith(dsc_name):
ql.remove(i)
- if ql:
+ if len(ql) > 0:
# Unfortunately, we may get more than one match here if,
# for example, the package was in potato but had an -sa
# upload in woody. So we need to choose the right one.
if len(ql) > 1:
for i in ql:
- old_file = i[0] + i[1]
+ old_file = os.path.join(i.location.path, i.filename)
old_file_fh = utils.open_file(old_file)
actual_md5 = apt_pkg.md5sum(old_file_fh)
old_file_fh.close()
actual_size = os.stat(old_file)[stat.ST_SIZE]
- if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
+ if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
x = i
- old_file = x[0] + x[1]
+ old_file = os.path.join(i.location.path, i.filename)
old_file_fh = utils.open_file(old_file)
actual_md5 = apt_pkg.md5sum(old_file_fh)
old_file_fh.close()
actual_size = os.stat(old_file)[stat.ST_SIZE]
found = old_file
- suite_type = x[2]
+ suite_type = x.location.archive_type
# need this for updating dsc_files in install()
- dsc_files[dsc_file]["files id"] = x[3]
+ dsc_entry["files id"] = x.file_id
# See install() in process-accepted...
- self.pkg.orig_tar_id = x[3]
+ self.pkg.orig_tar_id = x.file_id
self.pkg.orig_tar_gz = old_file
- self.pkg.orig_tar_location = x[4]
+ self.pkg.orig_tar_location = x.location.location_id
else:
+ # TODO: Record the queues and info in the DB so we don't hardcode all this crap
# Not there? Check the queue directories...
-
- in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file)
- # See process_it() in 'dak process-unchecked' for explanation of this
- # in_unchecked check dropped by ajt 2007-08-28, how did that
- # ever make sense?
- if os.path.exists(in_unchecked) and False:
- return (self.reject_message, in_unchecked)
- else:
- for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
- in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (directory)],dsc_file)
- if os.path.exists(in_otherdir):
- in_otherdir_fh = utils.open_file(in_otherdir)
- actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
- in_otherdir_fh.close()
- actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
- found = in_otherdir
- self.pkg.orig_tar_gz = in_otherdir
+ for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
+ if not Cnf.has_key("Dir::Queue::%s" % (directory)):
+ continue
+ in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
+ if os.path.exists(in_otherdir):
+ in_otherdir_fh = utils.open_file(in_otherdir)
+ actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
+ in_otherdir_fh.close()
+ actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
+ found = in_otherdir
+ self.pkg.orig_tar_gz = in_otherdir
if not found:
- self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file))
+ self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
self.pkg.orig_tar_gz = -1
continue
else:
- self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file))
+ self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
continue
- if actual_md5 != dsc_files[dsc_file]["md5sum"]:
- self.reject("md5sum for %s doesn't match %s." % (found, file))
- if actual_size != int(dsc_files[dsc_file]["size"]):
- self.reject("size for %s doesn't match %s." % (found, file))
+ if actual_md5 != dsc_entry["md5sum"]:
+ self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
+ if actual_size != int(dsc_entry["size"]):
+ self.rejects.append("size for %s doesn't match %s." % (found, file))
- return (self.reject_message, None)
+ ################################################################################
+ def accepted_checks(self, overwrite_checks, session):
+ # Recheck anything that relies on the database; since that's not
+ # frozen between accept and our run time when called from p-a.
- def do_query(self, query):
- """
- Executes a database query. Writes statistics / timing to stderr.
+ # overwrite_checks is set to False when installing to stable/oldstable
- @type query: string
- @param query: database query string, passed unmodified
+ propogate={}
+ nopropogate={}
- @return: db result
+ # Find the .dsc (again)
+ dsc_filename = None
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f]["type"] == "dsc":
+ dsc_filename = f
- @warning: The query is passed B{unmodified}, so be careful what you use this for.
- """
- sys.stderr.write("query: \"%s\" ... " % (query))
- before = time.time()
- r = self.projectB.query(query)
- time_diff = time.time()-before
- sys.stderr.write("took %.3f seconds.\n" % (time_diff))
- return r
+ for checkfile in self.pkg.files.keys():
+ # The .orig.tar.gz can disappear out from under us is it's a
+ # duplicate of one in the archive.
+ if not self.pkg.files.has_key(checkfile):
+ continue
+
+ entry = self.pkg.files[checkfile]
+
+ # Check that the source still exists
+ if entry["type"] == "deb":
+ source_version = entry["source version"]
+ source_package = entry["source package"]
+ if not self.pkg.changes["architecture"].has_key("source") \
+ and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
+ self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
+
+ # Version and file overwrite checks
+ if overwrite_checks:
+ if entry["type"] == "deb":
+ self.check_binary_against_db(checkfile, session)
+ elif entry["type"] == "dsc":
+ self.check_source_against_db(checkfile, session)
+ self.check_dsc_against_db(dsc_filename, session)
+
+ # propogate in the case it is in the override tables:
+ for suite in self.pkg.changes.get("propdistribution", {}).keys():
+ if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
+ propogate[suite] = 1
+ else:
+ nopropogate[suite] = 1
+
+ for suite in propogate.keys():
+ if suite in nopropogate:
+ continue
+ self.pkg.changes["distribution"][suite] = 1
+
+ for checkfile in self.pkg.files.keys():
+ # Check the package is still in the override tables
+ for suite in self.pkg.changes["distribution"].keys():
+ if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
+ self.rejects.append("%s is NEW for %s." % (checkfile, suite))
+
+ ################################################################################
+ # This is not really a reject, but an unaccept, but since a) the code for
+ # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
+ # extremely rare, for now we'll go with whining at our admin folks...
+
+ def do_unaccept(self):
+ cnf = Config()
+
+ self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__REJECT_MESSAGE__"] = self.package_info()
+ self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__BCC__"] = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
+ if cnf.has_key("Dinstall::Bcc"):
+ self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+
+ template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
+
+ reject_mail_message = utils.TemplateSubst(self.Subst, template)
+
+ # Write the rejection email out as the <foo>.reason file
+ reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
+ reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
+
+ # If we fail here someone is probably trying to exploit the race
+ # so let's just raise an exception ...
+ if os.path.exists(reject_filename):
+ os.unlink(reject_filename)
+
+ fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ os.write(fd, reject_mail_message)
+ os.close(fd)
+
+ utils.send_mail(reject_mail_message)
+
+ del self.Subst["__REJECTOR_ADDRESS__"]
+ del self.Subst["__REJECT_MESSAGE__"]
+ del self.Subst["__CC__"]
+
+ ################################################################################
+ # If any file of an upload has a recent mtime then chances are good
+ # the file is still being uploaded.
+
+ def upload_too_new(self):
+ cnf = Config()
+ too_new = False
+ # Move back to the original directory to get accurate time stamps
+ cwd = os.getcwd()
+ os.chdir(self.pkg.directory)
+ file_list = self.pkg.files.keys()
+ file_list.extend(self.pkg.dsc_files.keys())
+ file_list.append(self.pkg.changes_file)
+ for f in file_list:
+ try:
+ last_modified = time.time()-os.path.getmtime(f)
+ if last_modified < int(cnf["Dinstall::SkipTime"]):
+ too_new = True
+ break
+ except:
+ pass
+
+ os.chdir(cwd)
+ return too_new
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Simple summary class for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+from singleton import Singleton
+
+###############################################################################
+
+class SummaryStats(Singleton):
+ def __init__(self, *args, **kwargs):
+ super(SummaryStats, self).__init__(*args, **kwargs)
+
+ def _startup(self):
+ self.reset_accept()
+
+ def reset_accept(self):
+ self.accept_count = 0
+ self.accept_bytes = 0
+
--- /dev/null
+#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
+
+"""Text utility functions
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import email.Header
+
+from dak_exceptions import *
+from regexes import re_parse_maintainer
+
+################################################################################
+
+def force_to_utf8(s):
+ """
+ Forces a string to UTF-8. If the string isn't already UTF-8,
+ it's assumed to be ISO-8859-1.
+ """
+ try:
+ unicode(s, 'utf-8')
+ return s
+ except UnicodeError:
+ latin1_s = unicode(s,'iso8859-1')
+ return latin1_s.encode('utf-8')
+
+def rfc2047_encode(s):
+ """
+ Encodes a (header) string per RFC2047 if necessary. If the
+ string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.
+ """
+ for enc in ['ascii', 'utf-8', 'iso-8859-1']:
+ try:
+ h = email.Header.Header(s, enc, 998)
+ return str(h)
+ except UnicodeError:
+ pass
+
+ # If we get here, we're boned beyond belief
+ return ''
+
+################################################################################
+
+# <Culus> 'The standard sucks, but my tool is supposed to interoperate
+# with it. I know - I'll fix the suckage and make things
+# incompatible!'
+
+def fix_maintainer(maintainer):
+ """
+ Parses a Maintainer or Changed-By field and returns:
+ 1. an RFC822 compatible version,
+ 2. an RFC2047 compatible version,
+ 3. the name
+ 4. the email
+
+ The name is forced to UTF-8 for both 1. and 3.. If the name field
+ contains '.' or ',' (as allowed by Debian policy), 1. and 2. are
+ switched to 'email (name)' format.
+
+ """
+ maintainer = maintainer.strip()
+ if not maintainer:
+ return ('', '', '', '')
+
+ if maintainer.find("<") == -1:
+ email = maintainer
+ name = ""
+ elif (maintainer[0] == "<" and maintainer[-1:] == ">"):
+ email = maintainer[1:-1]
+ name = ""
+ else:
+ m = re_parse_maintainer.match(maintainer)
+ if not m:
+ raise ParseMaintError, "Doesn't parse as a valid Maintainer field."
+ name = m.group(1)
+ email = m.group(2)
+
+ # Get an RFC2047 compliant version of the name
+ rfc2047_name = rfc2047_encode(name)
+
+ # Force the name to be UTF-8
+ name = force_to_utf8(name)
+
+ if name.find(',') != -1 or name.find('.') != -1:
+ rfc822_maint = "%s (%s)" % (email, name)
+ rfc2047_maint = "%s (%s)" % (email, rfc2047_name)
+ else:
+ rfc822_maint = "%s <%s>" % (name, email)
+ rfc2047_maint = "%s <%s>" % (rfc2047_name, email)
+
+ if email.find("@") == -1 and email.find("buildd_") != 0:
+ raise ParseMaintError, "No @ found in email address part."
+
+ return (rfc822_maint, rfc2047_maint, name, email)
+
+################################################################################
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Urgency Logger class for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+import os
+import time
+
+from singleton import Singleton
+from config import Config
+from utils import warn, open_file, move
+
+###############################################################################
+
+class UrgencyLog(Singleton):
+ "Urgency Logger object"
+ def __init__(self, *args, **kwargs):
+ super(UrgencyLog, self).__init__(*args, **kwargs)
+
+ def _startup(self):
+ "Initialize a new Urgency Logger object"
+
+ self.timestamp = time.strftime("%Y%m%d%H%M%S")
+
+ # Create the log directory if it doesn't exist
+ self.log_dir = Config()["Dir::UrgencyLog"]
+
+ if not os.path.exists(self.log_dir) or not os.access(self.log_dir, os.W_OK):
+ warn("UrgencyLog directory %s does not exist or is not writeable, using /srv/ftp.debian.org/tmp/ instead" % (self.log_dir))
+ self.log_dir = '/srv/ftp.debian.org/tmp/'
+
+ # Open the logfile
+ self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp)
+ self.log_file = open_file(self.log_filename, 'w')
+ self.writes = 0
+
+ def log(self, source, version, urgency):
+ "Log an event"
+ self.log_file.write(" ".join([source, version, urgency])+'\n')
+ self.log_file.flush()
+ self.writes += 1
+
+ def close(self):
+ "Close a Logger object"
+ self.log_file.flush()
+ self.log_file.close()
+
+ if self.writes:
+ new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp)
+ move(self.log_filename, new_filename)
+ else:
+ os.unlink(self.log_filename)
+
import traceback
import stat
import apt_pkg
-import database
import time
import re
import string
import email as modemail
+
+from dbconn import DBConn, get_architecture, get_component, get_suite
from dak_exceptions import *
+from textutils import fix_maintainer
from regexes import re_html_escaping, html_escaping, re_single_line_field, \
re_multi_line_field, re_srchasver, re_verwithext, \
re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark, \
################################################################################
-def ensure_hashes(changes, dsc, files, dsc_files):
- rejmsg = []
-
- # Make sure we recognise the format of the Files: field in the .changes
- format = changes.get("format", "0.0").split(".", 1)
- if len(format) == 2:
- format = int(format[0]), int(format[1])
- else:
- format = int(float(format[0])), 0
-
- # We need to deal with the original changes blob, as the fields we need
- # might not be in the changes dict serialised into the .dak anymore.
- orig_changes = parse_deb822(changes['filecontents'])
-
- # Copy the checksums over to the current changes dict. This will keep
- # the existing modifications to it intact.
- for field in orig_changes:
- if field.startswith('checksums-'):
- changes[field] = orig_changes[field]
-
- # Check for unsupported hashes
- rejmsg.extend(check_hash_fields(".changes", changes))
- rejmsg.extend(check_hash_fields(".dsc", dsc))
-
- # We have to calculate the hash if we have an earlier changes version than
- # the hash appears in rather than require it exist in the changes file
- for hashname, hashfunc, version in known_hashes:
- rejmsg.extend(_ensure_changes_hash(changes, format, version, files,
- hashname, hashfunc))
- if "source" in changes["architecture"]:
- rejmsg.extend(_ensure_dsc_hash(dsc, dsc_files, hashname,
- hashfunc))
-
- return rejmsg
-
def parse_checksums(where, files, manifest, hashname):
rejmsg = []
field = 'checksums-%s' % hashname
for line in manifest[field].split('\n'):
if not line:
break
- checksum, size, checkfile = line.strip().split(' ')
+ clist = line.strip().split(' ')
+ if len(clist) == 3:
+ checksum, size, checkfile = clist
+ else:
+ rejmsg.append("Cannot parse checksum line [%s]" % (line))
+ continue
if not files.has_key(checkfile):
# TODO: check for the file's entry in the original files dict, not
# the one modified by (auto)byhand and other weird stuff
################################################################################
-def force_to_utf8(s):
- """
- Forces a string to UTF-8. If the string isn't already UTF-8,
- it's assumed to be ISO-8859-1.
- """
- try:
- unicode(s, 'utf-8')
- return s
- except UnicodeError:
- latin1_s = unicode(s,'iso8859-1')
- return latin1_s.encode('utf-8')
-
-def rfc2047_encode(s):
- """
- Encodes a (header) string per RFC2047 if necessary. If the
- string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.
- """
- try:
- codecs.lookup('ascii')[1](s)
- return s
- except UnicodeError:
- pass
- try:
- codecs.lookup('utf-8')[1](s)
- h = email.Header.Header(s, 'utf-8', 998)
- return str(h)
- except UnicodeError:
- h = email.Header.Header(s, 'iso-8859-1', 998)
- return str(h)
-
-################################################################################
-
-# <Culus> 'The standard sucks, but my tool is supposed to interoperate
-# with it. I know - I'll fix the suckage and make things
-# incompatible!'
-
-def fix_maintainer (maintainer):
- """
- Parses a Maintainer or Changed-By field and returns:
- 1. an RFC822 compatible version,
- 2. an RFC2047 compatible version,
- 3. the name
- 4. the email
-
- The name is forced to UTF-8 for both 1. and 3.. If the name field
- contains '.' or ',' (as allowed by Debian policy), 1. and 2. are
- switched to 'email (name)' format.
-
- """
- maintainer = maintainer.strip()
- if not maintainer:
- return ('', '', '', '')
-
- if maintainer.find("<") == -1:
- email = maintainer
- name = ""
- elif (maintainer[0] == "<" and maintainer[-1:] == ">"):
- email = maintainer[1:-1]
- name = ""
- else:
- m = re_parse_maintainer.match(maintainer)
- if not m:
- raise ParseMaintError, "Doesn't parse as a valid Maintainer field."
- name = m.group(1)
- email = m.group(2)
-
- # Get an RFC2047 compliant version of the name
- rfc2047_name = rfc2047_encode(name)
-
- # Force the name to be UTF-8
- name = force_to_utf8(name)
-
- if name.find(',') != -1 or name.find('.') != -1:
- rfc822_maint = "%s (%s)" % (email, name)
- rfc2047_maint = "%s (%s)" % (email, rfc2047_name)
- else:
- rfc822_maint = "%s <%s>" % (name, email)
- rfc2047_maint = "%s <%s>" % (rfc2047_name, email)
-
- if email.find("@") == -1 and email.find("buildd_") != 0:
- raise ParseMaintError, "No @ found in email address part."
-
- return (rfc822_maint, rfc2047_maint, name, email)
-
-################################################################################
-
def send_mail (message, filename=""):
"""sendmail wrapper, takes _either_ a message string or a file as arguments"""
################################################################################
-# Escape characters which have meaning to SQL's regex comparison operator ('~')
-# (woefully incomplete)
-
-def regex_safe (s):
- s = s.replace('+', '\\\\+')
- s = s.replace('.', '\\\\.')
- return s
-
-################################################################################
-
def TemplateSubst(map, filename):
""" Perform a substition of template """
templatefile = open_file(filename)
template = templatefile.read()
for x in map.keys():
- template = template.replace(x,map[x])
+ template = template.replace(x, str(map[x]))
templatefile.close()
return template
def parse_args(Options):
""" Handle -a, -c and -s arguments; returns them as SQL constraints """
+ # XXX: This should go away and everything which calls it be converted
+ # to use SQLA properly. For now, we'll just fix it not to use
+ # the old Pg interface though
+ session = DBConn().session()
# Process suite
if Options["Suite"]:
suite_ids_list = []
- for suite in split_args(Options["Suite"]):
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
- warn("suite '%s' not recognised." % (suite))
+ for suitename in split_args(Options["Suite"]):
+ suite = get_suite(suitename, session=session)
+ if suite.suite_id is None:
+ warn("suite '%s' not recognised." % (suite.suite_name))
else:
- suite_ids_list.append(suite_id)
+ suite_ids_list.append(suite.suite_id)
if suite_ids_list:
con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
else:
# Process component
if Options["Component"]:
component_ids_list = []
- for component in split_args(Options["Component"]):
- component_id = database.get_component_id(component)
- if component_id == -1:
- warn("component '%s' not recognised." % (component))
+ for componentname in split_args(Options["Component"]):
+ component = get_component(componentname, session=session)
+ if component is None:
+ warn("component '%s' not recognised." % (componentname))
else:
- component_ids_list.append(component_id)
+ component_ids_list.append(component.component_id)
if component_ids_list:
con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
else:
# Process architecture
con_architectures = ""
+ check_source = 0
if Options["Architecture"]:
arch_ids_list = []
- check_source = 0
- for architecture in split_args(Options["Architecture"]):
- if architecture == "source":
+ for archname in split_args(Options["Architecture"]):
+ if archname == "source":
check_source = 1
else:
- architecture_id = database.get_architecture_id(architecture)
- if architecture_id == -1:
- warn("architecture '%s' not recognised." % (architecture))
+ arch = get_architecture(archname, session=session)
+ if arch is None:
+ warn("architecture '%s' not recognised." % (archname))
else:
- arch_ids_list.append(architecture_id)
+ arch_ids_list.append(arch.arch_id)
if arch_ids_list:
con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
else:
################################################################################
-def check_signature (sig_filename, reject, data_filename="", keyrings=None, autofetch=None):
+def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None):
"""
Check the signature of a file and return the fingerprint if the
signature is valid or 'None' if it's not. The first argument is the
used.
"""
+ rejects = []
+
# Ensure the filename contains no shell meta-characters or other badness
if not re_taint_free.match(sig_filename):
- reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
- return None
+ rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
+ return (None, rejects)
if data_filename and not re_taint_free.match(data_filename):
- reject("!!WARNING!! tainted data filename: '%s'." % (data_filename))
- return None
+ rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
+ return (None, rejects)
if not keyrings:
keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
if autofetch:
error_msg = retrieve_key(sig_filename)
if error_msg:
- reject(error_msg)
- return None
+ rejects.append(error_msg)
+ return (None, rejects)
# Build the command line
status_read, status_write = os.pipe()
# If we failed to parse the status-fd output, let's just whine and bail now
if internal_error:
- reject("internal error while performing signature check on %s." % (sig_filename))
- reject(internal_error, "")
- reject("Please report the above errors to the Archive maintainers by replying to this mail.", "")
- return None
+ rejects.append("internal error while performing signature check on %s." % (sig_filename))
+ rejects.append(internal_error, "")
+ rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
+ return (None, rejects)
- bad = ""
# Now check for obviously bad things in the processed output
if keywords.has_key("KEYREVOKED"):
- reject("The key used to sign %s has been revoked." % (sig_filename))
- bad = 1
+ rejects.append("The key used to sign %s has been revoked." % (sig_filename))
if keywords.has_key("BADSIG"):
- reject("bad signature on %s." % (sig_filename))
- bad = 1
+ rejects.append("bad signature on %s." % (sig_filename))
if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
- reject("failed to check signature on %s." % (sig_filename))
- bad = 1
+ rejects.append("failed to check signature on %s." % (sig_filename))
if keywords.has_key("NO_PUBKEY"):
args = keywords["NO_PUBKEY"]
if len(args) >= 1:
key = args[0]
- reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
- bad = 1
+ rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
if keywords.has_key("BADARMOR"):
- reject("ASCII armour of signature was corrupt in %s." % (sig_filename))
- bad = 1
+ rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))
if keywords.has_key("NODATA"):
- reject("no signature found in %s." % (sig_filename))
- bad = 1
+ rejects.append("no signature found in %s." % (sig_filename))
if keywords.has_key("EXPKEYSIG"):
args = keywords["EXPKEYSIG"]
if len(args) >= 1:
key = args[0]
- reject("Signature made by expired key 0x%s" % (key))
- bad = 1
+ rejects.append("Signature made by expired key 0x%s" % (key))
if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
args = keywords["KEYEXPIRED"]
expiredate=""
expiredate = "unknown (%s)" % (timestamp)
else:
expiredate = timestamp
- reject("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
- bad = 1
+ rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
- if bad:
- return None
+ if len(rejects) > 0:
+ return (None, rejects)
# Next check gpgv exited with a zero return code
if exit_status:
- reject("gpgv failed while checking %s." % (sig_filename))
+ rejects.append("gpgv failed while checking %s." % (sig_filename))
if status.strip():
- reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
+ rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
else:
- reject(prefix_multi_line_string(output, " [GPG output:] "), "")
- return None
+ rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "")
+ return (None, rejects)
# Sanity check the good stuff we expect
if not keywords.has_key("VALIDSIG"):
- reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
- bad = 1
+ rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
else:
args = keywords["VALIDSIG"]
if len(args) < 1:
- reject("internal error while checking signature on %s." % (sig_filename))
- bad = 1
+ rejects.append("internal error while checking signature on %s." % (sig_filename))
else:
fingerprint = args[0]
if not keywords.has_key("GOODSIG"):
- reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
- bad = 1
+ rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
if not keywords.has_key("SIG_ID"):
- reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
- bad = 1
+ rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
# Finally ensure there's not something we don't recognise
known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
for keyword in keywords.keys():
if not known_keywords.has_key(keyword):
- reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
- bad = 1
+ rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
- if bad:
- return None
+ if len(rejects) > 0:
+ return (None, rejects)
else:
- return fingerprint
+ return (fingerprint, [])
################################################################################
projects / dak.git / commitdiff