FileList "/srv/ftp.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
SourceFileList "/srv/ftp.debian.org/database/dists/testing_$(SECTION)_source.list";
Sections "main contrib non-free";
- Architectures "alpha amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 source";
+ Architectures "amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 source";
BinOverride "override.squeeze.$(SECTION)";
ExtraOverride "override.squeeze.extra.$(SECTION)";
SrcOverride "override.squeeze.$(SECTION).src";
FileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
SourceFileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
Sections "main contrib non-free";
- Architectures "alpha amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 source";
+ Architectures "amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64 source";
BinOverride "override.squeeze.$(SECTION)";
ExtraOverride "override.squeeze.extra.$(SECTION)";
SrcOverride "override.squeeze.$(SECTION).src";
{
FileList "/srv/ftp.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
Sections "debian-installer";
- Architectures "alpha amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64";
+ Architectures "amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64";
BinOverride "override.squeeze.main.$(SECTION)";
SrcOverride "override.squeeze.main.src";
BinCacheDB "packages-debian-installer-$(ARCH).db";
{
FileList "/srv/ftp.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
Sections "debian-installer";
- Architectures "alpha amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64";
+ Architectures "amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64";
BinOverride "override.squeeze.main.$(SECTION)";
SrcOverride "override.squeeze.main.src";
BinCacheDB "packages-debian-installer-$(ARCH).db";
{
FileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
Sections "debian-installer";
- Architectures "alpha amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64";
+ Architectures "amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc kfreebsd-i386 kfreebsd-amd64";
BinOverride "override.squeeze.main.$(SECTION)";
SrcOverride "override.squeeze.main.src";
BinCacheDB "packages-debian-installer-$(ARCH).db";
ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1
}
-function runparts() {
- log "Using run-parts to run scripts in $base/scripts/distmnt"
- run-parts --report $base/scripts/distmnt
+function mirrorpush() {
+ log "Starting the mirrorpush"
+ date -u > /srv/ftp.debian.org/web/mirrorstart
+ echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
+ echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
+ sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
}
function i18n2() {
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
FUNC="punew"
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
lockfile "$LOCK_ACCEPTED"
lockfile "$LOCK_NEW"
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
FUNC="overrides"
ARGS=""
ERR=""
)
-stage $GO
+stage $GO &
rm -f "${NOTICE}"
rm -f "${LOCK_DAILY}"
ARGS=""
ERR=""
)
-stage $GO
+stage $GO &
GO=(
FUNC="expire"
ARGS=""
ERR=""
)
-stage $GO
+stage $GO &
GO=(
FUNC="transitionsclean"
ARGS=""
ERR=""
)
-stage $GO
+stage $GO &
GO=(
FUNC="reports"
ARGS=""
ERR=""
)
-stage $GO
+stage $GO &
GO=(
FUNC="dm"
ARGS=""
ERR=""
)
-stage $GO
+stage $GO &
GO=(
FUNC="bts"
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
FUNC="merkel2"
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
- FUNC="runparts"
- TIME="run-parts"
+ FUNC="mirrorpush"
+ TIME="mirrorpush"
ARGS=""
ERR="false"
)
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
FUNC="testingsourcelist"
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
FUNC="merkel3"
ARGS=""
ERR="false"
)
-stage $GO
+stage $GO &
GO=(
FUNC="compress"
SigningKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg";
SigningPubKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg";
SigningKeyIds "55BE302B";
- SendmailCommand "/usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org";
+// SendmailCommand "/usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org";
+ SendmailCommand "/usr/sbin/sendmail -oi -t -f envelope@ftp-master.debian.org";
MyEmailAddress "Archive Administrator <installer@ftp-master.debian.org>";
MyAdminAddress "ftpmaster@debian.org";
MyHost "debian.org"; // used for generating user@my_host addresses in e.g. manual_reject()
CloseBugs "true";
OverrideDisparityCheck "true";
DefaultSuite "unstable";
- UserExtensions "/srv/ftp.debian.org/dak/config/debian/extensions.py";
+ LintianTags "/srv/ftp.debian.org/dak/config/debian/lintian.tags";
QueueBuildSuites
{
unstable;
{
ValidGID "800";
// Comma separated list of users who are in Postgres but not the passwd file
- KnownPostgres "postgres,dak,katie,release";
+ KnownPostgres "postgres,dak,katie,release,qa,www-data,guest";
};
Clean-Queues
+++ /dev/null
-import sys, os, textwrap
-
-import apt_pkg
-import daklib.utils, daklib.database
-import yaml
-
-import daklib.extensions
-from daklib.extensions import replace_dak_function
-
-def check_transition():
- changes = dak_module.changes
- reject = dak_module.reject
- Cnf = dak_module.Cnf
-
- sourcepkg = changes["source"]
-
- # No sourceful upload -> no need to do anything else, direct return
- # We also work with unstable uploads, not experimental or those going to some
- # proposed-updates queue
- if "source" not in changes["architecture"] or "unstable" not in changes["distribution"]:
- return
-
- # Also only check if there is a file defined (and existant) with
- # checks.
- transpath = Cnf.get("Dinstall::Reject::ReleaseTransitions", "")
- if transpath == "" or not os.path.exists(transpath):
- return
-
- # Parse the yaml file
- sourcefile = file(transpath, 'r')
- sourcecontent = sourcefile.read()
- try:
- transitions = yaml.load(sourcecontent)
- except yaml.YAMLError, msg:
- # This shouldn't happen, there is a wrapper to edit the file which
- # checks it, but we prefer to be safe than ending up rejecting
- # everything.
- daklib.utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
- return
-
- # Now look through all defined transitions
- for trans in transitions:
- t = transitions[trans]
- source = t["source"]
- expected = t["new"]
-
- # Will be None if nothing is in testing.
- current = daklib.database.get_suite_version(source, "testing")
- if current is not None:
- compare = apt_pkg.VersionCompare(current, expected)
-
- if current is None or compare < 0:
- # This is still valid, the current version in testing is older than
- # the new version we wait for, or there is none in testing yet
-
- # Check if the source we look at is affected by this.
- if sourcepkg in t['packages']:
- # The source is affected, lets reject it.
-
- rejectmsg = "%s: part of the %s transition.\n\n" % (
- sourcepkg, trans)
-
- if current is not None:
- currentlymsg = "at version %s" % (current)
- else:
- currentlymsg = "not present in testing"
-
- rejectmsg += "Transition description: %s\n\n" % (t["reason"])
-
- rejectmsg += "\n".join(textwrap.wrap("""Your package
-is part of a testing transition designed to get %s migrated (it is
-currently %s, we need version %s). This transition is managed by the
-Release Team, and %s is the Release-Team member responsible for it.
-Please mail debian-release@lists.debian.org or contact %s directly if you
-need further assistance. You might want to upload to experimental until this
-transition is done."""
- % (source, currentlymsg, expected,t["rm"], t["rm"])))
-
- reject(rejectmsg + "\n")
- return
-
-@replace_dak_function("process-unchecked", "check_signed_by_key")
-def check_signed_by_key(oldfn):
- changes = dak_module.changes
- reject = dak_module.reject
-
- if changes["source"] == "dpkg":
- fpr = changes["fingerprint"]
- (uid, uid_name, is_dm) = dak_module.lookup_uid_from_fingerprint(fpr)
- if fpr == "5906F687BD03ACAD0D8E602EFCF37657" or uid == "iwj":
- reject("Upload blocked due to hijack attempt 2008/03/19")
-
- # NB: 1.15.0, 1.15.2 signed by this key targetted at unstable
- # have been made available in the wild, and should remain
- # blocked until Debian's dpkg has revved past those version
- # numbers
-
- oldfn()
-
- check_transition()
--- /dev/null
+lintian:
+ warning:
+ - statically-linked-binary
+ - arch-independent-package-contains-binary-or-object
+ - arch-dependent-file-in-usr-share
+ - missing-build-dependency
+ - arch-dependent-file-in-usr-share
+ - missing-dependency-on-libc
+ - usr-share-doc-symlink-without-dependency
+ - binary-with-bad-dynamic-table
+ - usr-share-doc-symlink-without-dependency
+ - mknod-in-maintainer-script
+ error:
+ - binary-in-etc
+ - missing-dependency-on-perlapi
+ - copyright-lists-upstream-authors-with-dh_make-boilerplate
+ - section-is-dh_make-template
+ - package-installs-python-pyc
+ - library-in-debug-or-profile-should-not-be-stripped
+ - binary-file-compressed-with-upx
+ - html-changelog-without-text-version
+ - file-in-usr-marked-as-conffile
+ - build-info-in-binary-control-file-section
+ - debian-control-with-duplicate-fields
+ - not-allowed-control-file
+ - control-file-has-bad-permissions
+ - control-file-has-bad-owner
+ - no-copyright-file
+ - copyright-refers-to-old-directory
+ - copyright-file-compressed
+ - copyright-file-is-symlink
+ - usr-share-doc-symlink-to-foreign-package
+ - old-style-copyright-file
+ - copyright-refers-to-incorrect-directory
+ - package-has-no-description
+ - description-synopsis-is-empty
+ - extended-description-is-empty
+ - description-is-dh_make-template
+ - file-in-etc-not-marked-as-conffile
+ - no-package-name
+ - bad-package-name
+ - package-not-lowercase
+ - no-version-field
+ - bad-version-number
+ - upstream-version-not-numeric
+ - no-architecture-field
+ - magic-arch-in-arch-list
+ - too-many-architectures
+ - arch-any-in-binary-pkg
+ - no-maintainer-field
+ - maintainer-name-missing
+ - maintainer-address-missing
+ - maintainer-address-malformed
+ - maintainer-address-is-on-localhost
+ - uploader-name-missing
+ - uploader-address-malformed
+ - uploader-address-is-on-localhost
+ - no-source-field
+ - source-field-does-not-match-pkg-name
+ - section-is-dh_make-template
+ - build-depends-on-essential-package-without-using-version
+ - depends-on-build-essential-package-without-using-version
+ - build-depends-on-build-essential
+ - executable-in-usr-share-doc
+ - symlink-has-too-many-up-segments
+ - debian-rules-is-symlink
+ - debian-rules-not-a-makefile
+ - debian-rules-missing-required-target
+ - maintainer-script-removes-device-files
+ - no-standards-version-field
+ - invalid-standards-version
+ - dir-or-file-in-var-www
+ - dir-or-file-in-tmp
+ - dir-or-file-in-mnt
+ - dir-or-file-in-opt
+ - dir-or-file-in-srv
# I know what I say. I dont know python and I wrote it. So go and read some other stuff.
import commands
-import pg
-import re
import sys
-import time
-import os
import apt_pkg
-from daklib import database
-from daklib import logging
-from daklib import queue
+
from daklib import utils
+from daklib.dbconn import DBConn, add_database_user, get_or_set_uid
from daklib.regexes import re_gpg_fingerprint, re_user_address, re_user_mails, re_user_name
################################################################################
Cnf = None
-projectB = None
Logger = None
-Upload = None
-Subst = None
################################################################################
################################################################################
def main():
- global Cnf, projectB
+ global Cnf
keyrings = None
Cnf = utils.get_conf()
if Options["help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
if not keyrings:
keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
utils.warn("Could not prepare password information for mail, not sending password.")
# Now add user to the database.
- projectB.query("BEGIN WORK")
- uid_id = database.get_or_set_uid_id(uid)
- projectB.query('CREATE USER "%s"' % (uid))
- projectB.query("COMMIT WORK")
+ # Note that we provide a session, so we're responsible for committing
+ uidobj = get_or_set_uid(uid, session=session)
+ uid_id = uidobj.uid_id
+ add_database_user(uid)
+ session.commit()
# The following two are kicked out in rhona, so we don't set them. kelly adds
# them as soon as she installs a package with unknown ones, so no problems to expect here.
# Just leave the comment in, to not think about "Why the hell aren't they added" in
# a year, if we ever touch uma again.
# maint_id = database.get_or_set_maintainer_id(name)
-# projectB.query("INSERT INTO fingerprint (fingerprint, uid) VALUES ('%s', '%s')" % (primary_key, uid_id))
+# session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)",
+# {'fingerprint': primary_key, 'uid': uid_id})
# Lets add user to the email-whitelist file if its configured.
if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "":
# Should we send mail to the newly added user?
if Cnf.FindB("Add-User::SendEmail"):
mail = name + "<" + emails[0] +">"
- Upload = queue.Upload(Cnf)
- Subst = Upload.Subst
+ Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
--- /dev/null
+#!/usr/bin/env python
+
+"""Configure dak parameters in the database"""
+# Copyright (C) 2009 Mark Hymers <mhy@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import sys
+
+import apt_pkg
+
+from daklib import utils
+from daklib.dbconn import *
+
+################################################################################
+
+dispatch = {}
+dryrun = False
+
+################################################################################
+def warn(msg):
+ print >> sys.stderr, msg
+
+def die(msg, exit_code=1):
+ print >> sys.stderr, msg
+ sys.exit(exit_code)
+
+def die_arglen(args, args_needed, msg):
+ if len(args) < args_needed:
+ die(msg)
+
+def usage(exit_code=0):
+ """Perform administrative work on the dak database."""
+
+ print """Usage: dak admin COMMAND
+Perform administrative work on the dak database.
+
+ -h, --help show this help and exit.
+ -n, --dry-run don't do anything, just show what would have been done
+ (only applies to add or rm operations).
+
+ Commands can use a long or abbreviated form:
+
+ architecture / a:
+ a list show a list of architectures
+ a rm ARCH remove an architecture (will only work if
+ no longer linked to any suites)
+ a add ARCH DESCRIPTION [SUITELIST]
+ add architecture ARCH with DESCRIPTION.
+ If SUITELIST is given, add to each of the
+ suites at the same time
+
+ suite / s:
+ s list show a list of suites
+ s show SUITE show config details for a suite
+
+ suite-architecture / s-a:
+ s-a list-suite ARCH show the suites an ARCH is in
+ s-a list-arch SUITE show the architectures in a SUITE
+ s-a add SUITE ARCH add ARCH to suite
+ s-a rm SUITE ARCH remove ARCH from suite (will only work if
+ no packages remain for the arch in the suite)
+"""
+ sys.exit(exit_code)
+
+################################################################################
+
+def __architecture_list(d, args):
+ q = d.session().query(Architecture).order_by('arch_string')
+ for j in q.all():
+ # HACK: We should get rid of source from the arch table
+ if j.arch_string == 'source': continue
+ print j.arch_string
+ sys.exit(0)
+
+def __architecture_add(d, args):
+ die_arglen(args, 3, "E: adding an architecture requires a name and a description")
+ print "Adding architecture %s" % args[2]
+ suites = [str(x) for x in args[4:]]
+ print suites
+ if not dryrun:
+ try:
+ s = d.session()
+ a = Architecture()
+ a.arch_string = str(args[2]).lower()
+ a.description = str(args[3])
+ s.add(a)
+ for sn in suites:
+ su = get_suite(sn ,s)
+ if su is not None:
+ archsu = SuiteArchitecture()
+ archsu.arch_id = a.arch_id
+ archsu.suite_id = su.suite_id
+ s.add(archsu)
+ else:
+ warn("W: Cannot find suite %s" % su)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Integrity error adding architecture %s (it probably already exists)" % args[2])
+ except SQLAlchemyError, e:
+ die("E: Error adding architecture %s (%s)" % (args[2], e))
+ print "Architecture %s added" % (args[2])
+
+def __architecture_rm(d, args):
+ die_arglen(args, 3, "E: removing an architecture requires at least a name")
+ print "Removing architecture %s" % args[2]
+ if not dryrun:
+ try:
+ s = d.session()
+ a = get_architecture(args[2].lower(), s)
+ if a is None:
+ die("E: Cannot find architecture %s" % args[2])
+ s.delete(a)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Integrity error removing architecture %s (suite-arch entries probably still exist)" % args[2])
+ except SQLAlchemyError, e:
+ die("E: Error removing architecture %s (%s)" % (args[2], e))
+ print "Architecture %s removed" % args[2]
+
+def architecture(command):
+ args = [str(x) for x in command]
+ Cnf = utils.get_conf()
+ d = DBConn()
+
+ die_arglen(args, 2, "E: architecture needs at least a command")
+
+ mode = args[1].lower()
+ if mode == 'list':
+ __architecture_list(d, args)
+ elif mode == 'add':
+ __architecture_add(d, args)
+ elif mode == 'rm':
+ __architecture_rm(d, args)
+ else:
+ die("E: architecture command unknown")
+
+dispatch['architecture'] = architecture
+dispatch['a'] = architecture
+
+################################################################################
+
+def __suite_list(d, args):
+ s = d.session()
+ for j in s.query(Suite).order_by('suite_name').all():
+ print j.suite_name
+
+def __suite_show(d, args):
+ if len(args) < 2:
+ die("E: showing an suite entry requires a suite")
+
+ s = d.session()
+ su = get_suite(args[2].lower())
+ if su is None:
+ die("E: can't find suite entry for %s" % (args[2].lower()))
+
+ print su.details()
+
+def suite(command):
+ args = [str(x) for x in command]
+ Cnf = utils.get_conf()
+ d = DBConn()
+
+ die_arglen(args, 2, "E: suite needs at least a command")
+
+ mode = args[1].lower()
+
+ if mode == 'list':
+ __suite_list(d, args)
+ elif mode == 'show':
+ __suite_show(d, args)
+ else:
+ die("E: suite command unknown")
+
+dispatch['suite'] = suite
+dispatch['s'] = suite
+
+################################################################################
+
+def __suite_architecture_list(d, args):
+ s = d.session()
+ for j in s.query(Suite).order_by('suite_name').all():
+ print j.suite_name + ' ' + \
+ ','.join([a.architecture.arch_string for a in j.suitearchitectures])
+
+def __suite_architecture_listarch(d, args):
+ die_arglen(args, 3, "E: suite-architecture list-arch requires a suite")
+ a = get_suite_architectures(args[2].lower())
+ for j in a:
+ # HACK: We should get rid of source from the arch table
+ if j.arch_string != 'source':
+ print j.arch_string
+
+
+def __suite_architecture_listsuite(d, args):
+ die_arglen(args, 3, "E: suite-architecture list-suite requires an arch")
+ for j in get_architecture_suites(args[2].lower()):
+ print j.suite_name
+
+
+def __suite_architecture_add(d, args):
+ if len(args) < 3:
+ die("E: adding a suite-architecture entry requires a suite and arch")
+
+ s = d.session()
+
+ suite = get_suite(args[2].lower(), s)
+ if suite is None: die("E: Can't find suite %s" % args[2].lower())
+
+ arch = get_architecture(args[3].lower(), s)
+ if arch is None: die("E: Can't find architecture %s" % args[3].lower())
+
+ if not dryrun:
+ try:
+ sa = SuiteArchitecture()
+ sa.arch_id = arch.arch_id
+ sa.suite_id = suite.suite_id
+ s.add(sa)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Can't add suite-architecture entry (%s, %s) - probably already exists" % (args[2].lower(), args[3].lower()))
+ except SQLAlchemyError, e:
+ die("E: Can't add suite-architecture entry (%s, %s) - %s" % (args[2].lower(), args[3].lower(), e))
+
+ print "Added suite-architecture entry for %s, %s" % (args[2].lower(), args[3].lower())
+
+
+def __suite_architecture_rm(d, args):
+ if len(args) < 3:
+ die("E: removing an suite-architecture entry requires a suite and arch")
+
+ s = d.session()
+ if not dryrun:
+ try:
+ sa = get_suite_architecture(args[2].lower(), args[3].lower(), s)
+ if sa is None:
+ die("E: can't find suite-architecture entry for %s, %s" % (args[2].lower(), args[3].lower()))
+ s.delete(sa)
+ s.commit()
+ except IntegrityError, e:
+ die("E: Can't remove suite-architecture entry (%s, %s) - it's probably referenced" % (args[2].lower(), args[3].lower()))
+ except SQLAlchemyError, e:
+ die("E: Can't remove suite-architecture entry (%s, %s) - %s" % (args[2].lower(), args[3].lower(), e))
+
+ print "Removed suite-architecture entry for %s, %s" % (args[2].lower(), args[3].lower())
+
+
+def suite_architecture(command):
+ args = [str(x) for x in command]
+ Cnf = utils.get_conf()
+ d = DBConn()
+
+ die_arglen(args, 2, "E: suite-architecture needs at least a command")
+
+ mode = args[1].lower()
+
+ if mode == 'list':
+ __suite_architecture_list(d, args)
+ elif mode == 'list-arch':
+ __suite_architecture_listarch(d, args)
+ elif mode == 'list-suite':
+ __suite_architecture_listsuite(d, args)
+ elif mode == 'add':
+ __suite_architecture_add(d, args)
+ elif mode == 'rm':
+ __suite_architecture_rm(d, args)
+ else:
+ die("E: suite-architecture command unknown")
+
+dispatch['suite-architecture'] = suite_architecture
+dispatch['s-a'] = suite_architecture
+
+################################################################################
+
+def main():
+ """Perform administrative work on the dak database"""
+ global dryrun
+ Cnf = utils.get_conf()
+ arguments = [('h', "help", "Admin::Options::Help"),
+ ('n', "dry-run", "Admin::Options::Dry-Run")]
+ for i in [ "help", "dry-run" ]:
+ if not Cnf.has_key("Admin::Options::%s" % (i)):
+ Cnf["Admin::Options::%s" % (i)] = ""
+
+ arguments = apt_pkg.ParseCommandLine(Cnf, arguments, sys.argv)
+
+ options = Cnf.SubTree("Admin::Options")
+ if options["Help"] or len(arguments) < 1:
+ usage()
+ if options["Dry-Run"]:
+ dryrun = True
+
+ subcommand = str(arguments[0])
+
+ if subcommand in dispatch.keys():
+ dispatch[subcommand](arguments)
+ else:
+ die("E: Unknown command")
+
+################################################################################
+
+if __name__ == '__main__':
+ main()
import commands
import os
-import pg
import stat
import sys
import time
import apt_pkg
import apt_inst
-from daklib import database
+
+from daklib.dbconn import *
from daklib import utils
-from daklib.regexes import re_issource
+from daklib.config import Config
################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
db_files = {} #: Cache of filenames as known by the database
waste = 0.0 #: How many bytes are "wasted" by files not referenced in database
excluded = {} #: List of files which are excluded from files check
missing-overrides - check for missing overrides
source-in-one-dir - ensure the source for each package is in one directory
timestamps - check for future timestamps in .deb's
- tar-gz-in-dsc - ensure each .dsc lists a .tar.gz file
+ files-in-dsc - ensure each .dsc references appropriate Files
validate-indices - ensure files mentioned in Packages & Sources exist
files-not-symlinks - check files in the database aren't symlinks
validate-builddeps - validate build-dependencies of .dsc files in the archive
"""
global db_files
+ cnf = Config()
+
print "Building list of database files..."
- q = projectB.query("SELECT l.path, f.filename, f.last_used FROM files f, location l WHERE f.location = l.id ORDER BY l.path, f.filename")
- ql = q.getresult()
+ q = DBConn().session().query(PoolFile).join(Location).order_by('path', 'location')
print "Missing files:"
db_files.clear()
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
+
+ for f in q.all():
+ filename = os.path.abspath(f.location.path, f.filename)
db_files[filename] = ""
if os.access(filename, os.R_OK) == 0:
- if i[2]:
- print "(last used: %s) %s" % (i[2], filename)
+ if f.last_used:
+ print "(last used: %s) %s" % (f.last_used, filename)
else:
print "%s" % (filename)
- filename = Cnf["Dir::Override"]+'override.unreferenced'
+ filename = os.path.join(cnf["Dir::Override"], 'override.unreferenced')
if os.path.exists(filename):
f = utils.open_file(filename)
for filename in f.readlines():
print "Existent files not in db:"
- os.path.walk(Cnf["Dir::Root"]+'pool/', process_dir, None)
+ os.path.walk(cnf["Dir::Root"] + 'pool/', process_dir, None)
print
print "%s wasted..." % (utils.size_type(waste))
"""
Parse every .dsc file in the archive and check for it's validity.
"""
+
+ cnf = Config()
+
count = 0
suite = 'unstable'
- for component in Cnf.SubTree("Component").List():
+
+ for component in cnf.SubTree("Component").List():
component = component.lower()
- list_filename = '%s%s_%s_source.list' % (Cnf["Dir::Lists"], suite, component)
+ list_filename = '%s%s_%s_source.list' % (cnf["Dir::Lists"], suite, component)
list_file = utils.open_file(list_filename)
+
for line in list_file.readlines():
f = line[:-1]
try:
"""
Check for missing overrides in stable and unstable.
"""
- for suite in [ "stable", "unstable" ]:
- print suite
- print "-"*len(suite)
+ session = DBConn().session()
+
+ for suite_name in [ "stable", "unstable" ]:
+ print suite_name
+ print "-" * len(suite_name)
print
- suite_id = database.get_suite_id(suite)
- q = projectB.query("""
+ suite = get_suite(suite)
+ q = s.execute("""
SELECT DISTINCT b.package FROM binaries b, bin_associations ba
- WHERE b.id = ba.bin AND ba.suite = %s AND NOT EXISTS
- (SELECT 1 FROM override o WHERE o.suite = %s AND o.package = b.package)"""
- % (suite_id, suite_id))
- print q
- q = projectB.query("""
+ WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS
+ (SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)"""
+ % {'suiteid': suite.suite_id})
+
+ for j in q.fetchall():
+ print j[0]
+
+ q = s.execute("""
SELECT DISTINCT s.source FROM source s, src_associations sa
- WHERE s.id = sa.source AND sa.suite = %s AND NOT EXISTS
- (SELECT 1 FROM override o WHERE o.suite = %s and o.package = s.source)"""
- % (suite_id, suite_id))
- print q
+ WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS
+ (SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)"""
+ % {'suiteid': suite.suite_id})
+ for j in q.fetchall():
+ print j[0]
################################################################################
# Not the most enterprising method, but hey...
broken_count = 0
- q = projectB.query("SELECT id FROM source;")
- for i in q.getresult():
- source_id = i[0]
- q2 = projectB.query("""
-SELECT l.path, f.filename FROM files f, dsc_files df, location l WHERE df.source = %s AND f.id = df.file AND l.id = f.location"""
- % (source_id))
+
+ session = DBConn().session()
+
+ q = session.query(DBSource)
+ for s in q.all():
first_path = ""
first_filename = ""
- broken = 0
- for j in q2.getresult():
- filename = j[0] + j[1]
+ broken = False
+
+ qf = session.query(PoolFile).join(Location).join(DSCFile).filter_by(source_id=s.source_id)
+ for f in qf.all():
+ # 0: path
+ # 1: filename
+ filename = os.path.join(f.location.path, f.filename)
path = os.path.dirname(filename)
+
if first_path == "":
first_path = path
first_filename = filename
elif first_path != path:
symlink = path + '/' + os.path.basename(first_filename)
if not os.path.exists(symlink):
- broken = 1
- print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, source_id, symlink)
+ broken = True
+ print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, s.source_id, symlink)
if broken:
broken_count += 1
+
print "Found %d source packages where the source is not all in one directory." % (broken_count)
################################################################################
-
def check_checksums():
"""
Validate all files
"""
print "Getting file information from database..."
- q = projectB.query("SELECT l.path, f.filename, f.md5sum, f.sha1sum, f.sha256sum, f.size FROM files f, location l WHERE f.location = l.id")
- ql = q.getresult()
+ q = DBConn().session().query(PoolFile)
print "Checking file checksums & sizes..."
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
- db_md5sum = i[2]
- db_sha1sum = i[3]
- db_sha256sum = i[4]
- db_size = int(i[5])
+ for f in q:
+ filename = os.path.abspath(os.path.join(f.location.path, f.filename))
+
try:
- f = utils.open_file(filename)
+ fi = utils.open_file(filename)
except:
utils.warn("can't open '%s'." % (filename))
continue
- md5sum = apt_pkg.md5sum(f)
+
size = os.stat(filename)[stat.ST_SIZE]
- if md5sum != db_md5sum:
- utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, db_md5sum))
- if size != db_size:
- utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, db_size))
- f.seek(0)
- sha1sum = apt_pkg.sha1sum(f)
- if sha1sum != db_sha1sum:
- utils.warn("**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha1sum, db_sha1sum))
-
- f.seek(0)
- sha256sum = apt_pkg.sha256sum(f)
- if sha256sum != db_sha256sum:
- utils.warn("**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha256sum, db_sha256sum))
+ if size != f.filesize:
+ utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, f.filesize))
+
+ md5sum = apt_pkg.md5sum(fi)
+ if md5sum != f.md5sum:
+ utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, f.md5sum))
+
+ fi.seek(0)
+ sha1sum = apt_pkg.sha1sum(fi)
+ if sha1sum != f.sha1sum:
+ utils.warn("**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha1sum, f.sha1sum))
+
+ fi.seek(0)
+ sha256sum = apt_pkg.sha256sum(fi)
+ if sha256sum != f.sha256sum:
+ utils.warn("**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha256sum, f.sha256sum))
print "Done."
global current_file
- q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.deb$'")
- ql = q.getresult()
+ q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like('.deb$'))
+
db_files.clear()
count = 0
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
+
+ for pf in q.all():
+ filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))
if os.access(filename, os.R_OK):
f = utils.open_file(filename)
current_file = filename
f.seek(0)
apt_inst.debExtract(f, Ent, "data.tar.gz")
count += 1
+
print "Checked %d files (out of %d)." % (count, len(db_files.keys()))
################################################################################
-def check_missing_tar_gz_in_dsc():
+def check_files_in_dsc():
"""
- Ensure each .dsc lists a .tar.gz file
+ Ensure each .dsc lists appropriate files in its Files field (according
+ to the format announced in its Format field).
"""
count = 0
print "Building list of database files..."
- q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.dsc$'")
- ql = q.getresult()
- if ql:
+ q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like('.dsc$'))
+
+ if q.count() > 0:
print "Checking %d files..." % len(ql)
else:
print "No files to check."
- for i in ql:
- filename = os.path.abspath(i[0] + i[1])
+
+ for pf in q.all():
+ filename = os.path.abspath(os.path.join(pf.location.path + pf.filename))
+
try:
# NB: don't enforce .dsc syntax
dsc = utils.parse_changes(filename)
except:
utils.fubar("error parsing .dsc file '%s'." % (filename))
- dsc_files = utils.build_file_list(dsc, is_a_dsc=1)
- has_tar = 0
- for f in dsc_files.keys():
- m = re_issource.match(f)
- if not m:
- utils.fubar("%s not recognised as source." % (f))
- ftype = m.group(3)
- if ftype == "orig.tar.gz" or ftype == "tar.gz":
- has_tar = 1
- if not has_tar:
- utils.warn("%s has no .tar.gz in the .dsc file." % (f))
+
+ reasons = utils.check_dsc_files(filename, dsc)
+ for r in reasons:
+ utils.warn(r)
+
+ if len(reasons) > 0:
count += 1
if count:
"""
print "Building list of database files... ",
before = time.time()
- q = projectB.query("SELECT l.path, f.filename, f.id FROM files f, location l WHERE f.location = l.id")
- print "done. (%d seconds)" % (int(time.time()-before))
- q_files = q.getresult()
+ q = DBConn().session().query(PoolFile).filter(PoolFile.filename.like('.dsc$'))
- for i in q_files:
- filename = os.path.normpath(i[0] + i[1])
+ for pf in q.all():
+ filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))
if os.access(filename, os.R_OK) == 0:
utils.warn("%s: doesn't exist." % (filename))
else:
def check_build_depends():
""" Validate build-dependencies of .dsc files in the archive """
- os.path.walk(Cnf["Dir::Root"], chk_bd_process_dir, None)
+ os.path.walk(cnf["Dir::Root"], chk_bd_process_dir, None)
################################################################################
def main ():
- global Cnf, projectB, db_files, waste, excluded
+ global db_files, waste, excluded
+
+ cnf = Config()
- Cnf = utils.get_conf()
Arguments = [('h',"help","Check-Archive::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Check-Archive::Options::%s" % (i)):
- Cnf["Check-Archive::Options::%s" % (i)] = ""
+ if not cnf.has_key("Check-Archive::Options::%s" % (i)):
+ cnf["Check-Archive::Options::%s" % (i)] = ""
- args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ args = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Check-Archive::Options")
+ Options = cnf.SubTree("Check-Archive::Options")
if Options["Help"]:
usage()
usage(1)
mode = args[0].lower()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ # Initialize DB
+ DBConn()
if mode == "checksums":
check_checksums()
check_source_in_one_dir()
elif mode == "timestamps":
check_timestamps()
- elif mode == "tar-gz-in-dsc":
- check_missing_tar_gz_in_dsc()
+ elif mode == "files-in-dsc":
+ check_files_in_dsc()
elif mode == "validate-indices":
check_indices_files_exist()
elif mode == "files-not-symlinks":
################################################################################
-import pg, sys, os
+import sys, os
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
from daklib import utils
################################################################################
Options = None
-projectB = None
Logger = None
sections = {}
priorities = {}
entry = entry.split('_')[0]
blacklist[entry] = 1
-def process(osuite, affected_suites, originosuite, component, type):
- global Logger, Options, projectB, sections, priorities
+def process(osuite, affected_suites, originosuite, component, otype, session):
+ global Logger, Options, sections, priorities
- osuite_id = database.get_suite_id(osuite)
- if osuite_id == -1:
+ o = get_suite(osuite, session)
+ if o is None:
utils.fubar("Suite '%s' not recognised." % (osuite))
+ osuite_id = o.suite_id
+
originosuite_id = None
if originosuite:
- originosuite_id = database.get_suite_id(originosuite)
- if originosuite_id == -1:
+ oo = get_suite(originosuite, session)
+ if oo is None:
utils.fubar("Suite '%s' not recognised." % (originosuite))
+ originosuite_id = oo.suite_id
- component_id = database.get_component_id(component)
- if component_id == -1:
+ c = get_component(component, session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
+ component_id = c.component_id
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
- dsc_type_id = database.get_override_type_id("dsc")
+ ot = get_override_type(otype, session)
+ if ot is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
+ type_id = ot.overridetype_id
+ dsc_type_id = get_override_type("dsc", session).overridetype_id
- source_priority_id = database.get_priority_id("source")
+ source_priority_id = get_priority("source", session).priority_id
- if type == "deb" or type == "udeb":
+ if otype == "deb" or otype == "udeb":
packages = {}
- q = projectB.query("""
+ # TODO: Fix to use placeholders (check how to with arrays)
+ q = session.execute("""
SELECT b.package FROM binaries b, bin_associations ba, files f,
location l, component c
- WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
- AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s
-""" % (type, ",".join([ str(i) for i in affected_suites ]), component_id))
- for i in q.getresult():
+ WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
+ AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
+""" % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
+ for i in q.fetchall():
packages[i[0]] = 0
src_packages = {}
- q = projectB.query("""
+ q = session.execute("""
SELECT s.source FROM source s, src_associations sa, files f, location l,
component c
WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
- AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s
-""" % (",".join([ str(i) for i in affected_suites]), component_id))
- for i in q.getresult():
+ AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
+""" % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
+ for i in q.fetchall():
src_packages[i[0]] = 0
# -----------
# Drop unused overrides
- q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id))
- projectB.query("BEGIN WORK")
- if type == "dsc":
- for i in q.getresult():
+ q = session.execute("""SELECT package, priority, section, maintainer
+ FROM override WHERE suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'suite_id': osuite_id, 'component_id': component_id,
+ 'type_id': type_id})
+ # We're already within a transaction
+ if otype == "dsc":
+ for i in q.fetchall():
package = i[0]
if src_packages.has_key(package):
src_packages[package] = 1
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(["removing unused override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""DELETE FROM override WHERE package =
- '%s' AND suite = %s AND component = %s AND type =
- %s""" % (package, osuite_id, component_id, type_id))
+ session.execute("""DELETE FROM override WHERE package = :package
+ AND suite = :suite_id AND component = :component_id
+ AND type = :type_id""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': type_id})
# create source overrides based on binary overrides, as source
# overrides not always get created
- q = projectB.query(""" SELECT package, priority, section,
- maintainer FROM override WHERE suite = %s AND component = %s
- """ % (osuite_id, component_id))
- for i in q.getresult():
+ q = session.execute("""SELECT package, priority, section, maintainer
+ FROM override WHERE suite = :suite_id AND component = :component_id""",
+ {'suite_id': osuite_id, 'component_id': component_id})
+ for i in q.fetchall():
package = i[0]
if not src_packages.has_key(package) or src_packages[package]:
continue
src_packages[package] = 1
Logger.log(["add missing override", osuite, component,
- type, package, "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
- osuite_id, component_id, source_priority_id, i[2],
- dsc_type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id, :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': source_priority_id,
+ 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
# Check whether originosuite has an override for us we can
# copy
if originosuite:
- q = projectB.query("""SELECT origin.package, origin.priority,
- origin.section, origin.maintainer, target.priority,
- target.section, target.maintainer FROM override origin LEFT
- JOIN override target ON (origin.package = target.package AND
- target.suite=%s AND origin.component = target.component AND origin.type =
- target.type) WHERE origin.suite = %s AND origin.component = %s
- AND origin.type = %s""" %
- (osuite_id, originosuite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT origin.package, origin.priority, origin.section,
+ origin.maintainer, target.priority, target.section,
+ target.maintainer
+ FROM override origin
+ LEFT JOIN override target ON (origin.package = target.package
+ AND target.suite = :suite_id
+ AND origin.component = target.component
+ AND origin.type = target.type)
+ WHERE origin.suite = :originsuite_id
+ AND origin.component = :component_id
+ AND origin.type = :type_id""",
+ {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
+ 'component_id': component_id, 'type_id': type_id})
+ for i in q.fetchall():
package = i[0]
if not src_packages.has_key(package) or src_packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(["syncing override", osuite, component,
- type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""UPDATE override SET section=%s,
- maintainer='%s' WHERE package='%s' AND
- suite=%s AND component=%s AND type=%s""" %
- (i[2], i[3], package, osuite_id, component_id,
- dsc_type_id))
+ session.execute("""UPDATE override
+ SET section = :section,
+ maintainer = :maintainer
+ WHERE package = :package AND suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'section': i[2], 'maintainer': i[3],
+ 'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': dsc_type_id})
continue
+
# we can copy
src_packages[package] = 1
Logger.log(["copying missing override", osuite, component,
- type, package, "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
- osuite_id, component_id, source_priority_id, i[2],
- dsc_type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id,
+ :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': source_priority_id,
+ 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
for package, hasoverride in src_packages.items():
if not hasoverride:
utils.warn("%s has no override!" % package)
else: # binary override
- for i in q.getresult():
+ for i in q.fetchall():
package = i[0]
if packages.has_key(package):
packages[package] = 1
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(["removing unused override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""DELETE FROM override WHERE package =
- '%s' AND suite = %s AND component = %s AND type =
- %s""" % (package, osuite_id, component_id, type_id))
+ session.execute("""DELETE FROM override
+ WHERE package = :package AND suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': type_id})
# Check whether originosuite has an override for us we can
# copy
if originosuite:
- q = projectB.query("""SELECT origin.package, origin.priority,
- origin.section, origin.maintainer, target.priority,
- target.section, target.maintainer FROM override origin LEFT
- JOIN override target ON (origin.package = target.package AND
- target.suite=%s AND origin.component = target.component AND
- origin.type = target.type) WHERE origin.suite = %s AND
- origin.component = %s AND origin.type = %s""" % (osuite_id,
- originosuite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT origin.package, origin.priority, origin.section,
+ origin.maintainer, target.priority, target.section,
+ target.maintainer
+ FROM override origin LEFT JOIN override target
+ ON (origin.package = target.package
+ AND target.suite = :suite_id
+ AND origin.component = target.component
+ AND origin.type = target.type)
+ WHERE origin.suite = :originsuite_id
+ AND origin.component = :component_id
+ AND origin.type = :type_id""",
+ {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
+ 'component_id': component_id, 'type_id': type_id})
+ for i in q.fetchall():
package = i[0]
if not packages.has_key(package) or packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(["syncing override", osuite, component,
- type, package, priorities[i[4]], sections[i[5]],
+ otype, package, priorities[i[4]], sections[i[5]],
i[6], priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""UPDATE override SET priority=%s, section=%s,
- maintainer='%s' WHERE package='%s' AND
- suite=%s AND component=%s AND type=%s""" %
- (i[1], i[2], i[3], package, osuite_id,
- component_id, type_id))
+ session.execute("""UPDATE override
+ SET priority = :priority_id,
+ section = :section_id,
+ maintainer = :maintainer
+ WHERE package = :package
+ AND suite = :suite_id
+ AND component = :component_id
+ AND type = :type_id""",
+ {'priority_id': i[1], 'section_id': i[2],
+ 'maintainer': i[3], 'package': package,
+ 'suite_id': osuite_id, 'component_id': component_id,
+ 'type_id': type_id})
continue
# we can copy
packages[package] = 1
Logger.log(["copying missing override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id, :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': i[1],
+ 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
for package, hasoverride in packages.items():
if not hasoverride:
utils.warn("%s has no override!" % package)
- projectB.query("COMMIT WORK")
+ session.commit()
sys.stdout.flush()
################################################################################
def main ():
- global Logger, Options, projectB, sections, priorities
+ global Logger, Options, sections, priorities
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Check-Overrides::Options::Help"),
('n',"no-action", "Check-Overrides::Options::No-Action")]
for i in [ "help", "no-action" ]:
- if not Cnf.has_key("Check-Overrides::Options::%s" % (i)):
- Cnf["Check-Overrides::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Check-Overrides::Options")
+ if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
+ cnf["Check-Overrides::Options::%s" % (i)] = ""
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Check-Overrides::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# init sections, priorities:
- q = projectB.query("SELECT id, section FROM section")
- for i in q.getresult():
- sections[i[0]] = i[1]
- q = projectB.query("SELECT id, priority FROM priority")
- for i in q.getresult():
- priorities[i[0]] = i[1]
+
+ # We need forward and reverse
+ sections = get_sections(session)
+ for name, entry in sections.items():
+ sections[entry] = name
+
+ priorities = get_priorities(session)
+ for name, entry in priorities.items():
+ priorities[entry] = name
if not Options["No-Action"]:
- Logger = logging.Logger(Cnf, "check-overrides")
+ Logger = daklog.Logger(cnf, "check-overrides")
else:
- Logger = logging.Logger(Cnf, "check-overrides", 1)
+ Logger = daklog.Logger(cnf, "check-overrides", 1)
- gen_blacklist(Cnf["Dir::Queue::Accepted"])
+ gen_blacklist(cnf["Dir::Queue::Accepted"])
- for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
- if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
+ for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
+ if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
continue
osuite = osuite.lower()
originosuite = None
originremark = ""
try:
- originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
+ originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
originosuite = originosuite.lower()
originremark = " taking missing from %s" % originosuite
except KeyError:
print "Processing %s%s..." % (osuite, originremark)
# Get a list of all suites that use the override file of 'osuite'
- ocodename = Cnf["Suite::%s::codename" % osuite].lower()
+ ocodename = cnf["Suite::%s::codename" % osuite].lower()
suites = []
- for suite in Cnf.SubTree("Suite").List():
- if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite].lower():
- suites.append(suite)
-
- q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \
- % ", ".join([ repr(i) for i in suites ]).lower())
-
suiteids = []
- for i in q.getresult():
- suiteids.append(i[0])
+ for suite in cnf.SubTree("Suite").List():
+ if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower():
+ suites.append(suite)
+ s = get_suite(suite.lower(), session)
+ if s is not None:
+ suiteids.append(s.suite_id)
if len(suiteids) != len(suites) or len(suiteids) < 1:
utils.fubar("Couldn't find id's of all suites: %s" % suites)
- for component in Cnf.SubTree("Component").List():
+ for component in cnf.SubTree("Component").List():
# It is crucial for the dsc override creation based on binary
# overrides that 'dsc' goes first
- otypes = Cnf.ValueList("OverrideType")
+ otypes = cnf.ValueList("OverrideType")
otypes.remove("dsc")
otypes = ["dsc"] + otypes
for otype in otypes:
print "Processing %s [%s - %s] using %s..." \
% (osuite, component, otype, suites)
sys.stdout.flush()
- process(osuite, suiteids, originosuite, component, otype)
+ process(osuite, suiteids, originosuite, component, otype, session)
Logger.close()
################################################################################
-import pg, sys, os
+import sys, os
import apt_pkg, apt_inst
-from daklib import database
+
+from daklib.dbconn import *
+from daklib.config import Config
from daklib import utils
from daklib.regexes import re_no_epoch
################################################################################
-Cnf = None
-projectB = None
Options = None
stable = {}
stable_virtual = {}
################################################################################
def check_changes (filename):
+ cnf = Config()
+
try:
changes = utils.parse_changes(filename)
files = utils.build_file_list(changes)
# Move to the pool directory
cwd = os.getcwd()
f = files.keys()[0]
- pool_dir = Cnf["Dir::Pool"] + '/' + utils.poolify(changes["source"], files[f]["component"])
+ pool_dir = cnf["Dir::Pool"] + '/' + utils.poolify(changes["source"], files[f]["component"])
os.chdir(pool_dir)
changes_result = 0
################################################################################
def check_joey (filename):
+ cnf = Config()
+
f = utils.open_file(filename)
cwd = os.getcwd()
- os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"]))
+ os.chdir("%s/dists/proposed-updates" % (cnf["Dir::Root"]))
for line in f.readlines():
line = line.rstrip()
def parse_packages():
global stable, stable_virtual, architectures
+ cnf = Config()
+
# Parse the Packages files (since it's a sub-second operation on auric)
suite = "stable"
stable = {}
- components = Cnf.ValueList("Suite::%s::Components" % (suite))
- architectures = filter(utils.real_arch, database.get_suite_architectures(suite))
+ components = cnf.ValueList("Suite::%s::Components" % (suite))
+ architectures = [ a.arch_string for a in get_suite_architectures(suite, skipsrc=True, skipall=True) ]
for component in components:
for architecture in architectures:
- filename = "%s/dists/%s/%s/binary-%s/Packages" % (Cnf["Dir::Root"], suite, component, architecture)
+ filename = "%s/dists/%s/%s/binary-%s/Packages" % (cnf["Dir::Root"], suite, component, architecture)
packages = utils.open_file(filename, 'r')
Packages = apt_pkg.ParseTagFile(packages)
while Packages.Step():
################################################################################
def main ():
- global Cnf, projectB, Options
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('d', "debug", "Check-Proposed-Updates::Options::Debug"),
('q',"quiet","Check-Proposed-Updates::Options::Quiet"),
('v',"verbose","Check-Proposed-Updates::Options::Verbose"),
('h',"help","Check-Proposed-Updates::Options::Help")]
for i in [ "debug", "quiet", "verbose", "help" ]:
- if not Cnf.has_key("Check-Proposed-Updates::Options::%s" % (i)):
- Cnf["Check-Proposed-Updates::Options::%s" % (i)] = ""
+ if not cnf.has_key("Check-Proposed-Updates::Options::%s" % (i)):
+ cnf["Check-Proposed-Updates::Options::%s" % (i)] = ""
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Check-Proposed-Updates::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Check-Proposed-Updates::Options")
if Options["Help"]:
usage(0)
if not arguments:
utils.fubar("need at least one package name as an argument.")
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ DBConn()
print "Parsing packages files...",
parse_packages()
################################################################################
-import os, pg, sys
+import os, sys
import apt_pkg
-from daklib import database
+
+from daklib.dbconn import *
+from daklib.config import Config
from daklib import utils
from daklib.regexes import re_isdeb, re_isadeb, re_issource, re_no_epoch
################################################################################
-Cnf = None
-projectB = None
Options = None
pu = {}
################################################################################
def check_changes (filename):
+ cnf = Config()
+
try:
changes = utils.parse_changes(filename)
files = utils.build_file_list(changes)
new_num_files = len(files.keys())
if new_num_files == 0:
print "%s: no files left, superseded by %s" % (filename, pu_version)
- dest = Cnf["Dir::Morgue"] + "/misc/"
+ dest = cnf["Dir::Morgue"] + "/misc/"
if not Options["no-action"]:
utils.move(filename, dest)
elif new_num_files < num_files:
################################################################################
def check_joey (filename):
+ cnf = Config()
+
f = utils.open_file(filename)
cwd = os.getcwd()
- os.chdir("%s/dists/%s" % (Cnf["Dir::Root"]), Options["suite"])
+ os.chdir("%s/dists/%s" % (cnf["Dir::Root"]), Options["suite"])
for line in f.readlines():
line = line.rstrip()
def init_pu ():
global pu
- q = projectB.query("""
+ q = DBConn().session().execute("""
SELECT b.package, b.version, a.arch_string
FROM bin_associations ba, binaries b, suite su, architecture a
WHERE b.id = ba.bin AND ba.suite = su.id
- AND su.suite_name = '%s' AND a.id = b.architecture
+ AND su.suite_name = :suite_name AND a.id = b.architecture
UNION SELECT s.source, s.version, 'source'
FROM src_associations sa, source s, suite su
WHERE s.id = sa.source AND sa.suite = su.id
- AND su.suite_name = '%s'
+ AND su.suite_name = :suite_name
ORDER BY package, version, arch_string
-""" % (Options["suite"], Options["suite"]))
- ql = q.getresult()
- for i in ql:
+""" % {'suite_name': Options["suite"]})
+
+ for i in q.fetchall():
pkg = i[0]
version = i[1]
arch = i[2]
pu[pkg][arch] = version
def main ():
- global Cnf, projectB, Options
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('d', "debug", "Clean-Proposed-Updates::Options::Debug"),
('v', "verbose", "Clean-Proposed-Updates::Options::Verbose"),
('s', "suite", "Clean-Proposed-Updates::Options::Suite", "HasArg"),
('n', "no-action", "Clean-Proposed-Updates::Options::No-Action"),]
for i in [ "debug", "verbose", "help", "no-action" ]:
- if not Cnf.has_key("Clean-Proposed-Updates::Options::%s" % (i)):
- Cnf["Clean-Proposed-Updates::Options::%s" % (i)] = ""
+ if not cnf.has_key("Clean-Proposed-Updates::Options::%s" % (i)):
+ cnf["Clean-Proposed-Updates::Options::%s" % (i)] = ""
# suite defaults to proposed-updates
- if not Cnf.has_key("Clean-Proposed-Updates::Options::Suite"):
- Cnf["Clean-Proposed-Updates::Options::Suite"] = "proposed-updates"
+ if not cnf.has_key("Clean-Proposed-Updates::Options::Suite"):
+ cnf["Clean-Proposed-Updates::Options::Suite"] = "proposed-updates"
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Clean-Proposed-Updates::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Clean-Proposed-Updates::Options")
if Options["Help"]:
usage(0)
if not arguments:
utils.fubar("need at least one package name as an argument.")
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ DBConn()
init_pu()
################################################################################
-import os, stat, sys, time
+import os, os.path, stat, sys, time
import apt_pkg
from daklib import utils
+from daklib import daklog
+from daklib.config import Config
################################################################################
-Cnf = None
Options = None
+Logger = None
del_dir = None
delete_date = None
################################################################################
-def init ():
+def init (cnf):
global delete_date, del_dir
delete_date = int(time.time())-(int(Options["Days"])*84600)
+ date = time.strftime("%Y-%m-%d")
+ del_dir = os.path.join(cnf["Dir::Morgue"], cnf["Clean-Queues::MorgueSubDir"], date)
# Ensure a directory exists to remove files to
if not Options["No-Action"]:
- date = time.strftime("%Y-%m-%d")
- del_dir = Cnf["Dir::Morgue"] + '/' + Cnf["Clean-Queues::MorgueSubDir"] + '/' + date
if not os.path.exists(del_dir):
os.makedirs(del_dir, 02775)
if not os.path.isdir(del_dir):
# Move to the directory to clean
incoming = Options["Incoming"]
if incoming == "":
- incoming = Cnf["Dir::Queue::Unchecked"]
+ incoming = cnf["Dir::Queue::Unchecked"]
os.chdir(incoming)
# Remove a file to the morgue
def remove (f):
+ fname = os.path.basename(f)
if os.access(f, os.R_OK):
- dest_filename = del_dir + '/' + os.path.basename(f)
+ Logger.log(["move file to morgue", fname, del_dir])
+ if Options["Verbose"]:
+ print "Removing '%s' (to '%s')." % (fname, del_dir)
+ if Options["No-Action"]:
+ return
+
+ dest_filename = os.path.join(del_dir, fname)
# If the destination file exists; try to find another filename to use
if os.path.exists(dest_filename):
dest_filename = utils.find_next_free(dest_filename, 10)
+ Logger.log(["change destination file name", os.path.basename(dest_filename)])
utils.move(f, dest_filename, 0660)
else:
- utils.warn("skipping '%s', permission denied." % (os.path.basename(f)))
+ Logger.log(["skipping file because of permission problem", fname])
+ utils.warn("skipping '%s', permission denied." % fname)
# Removes any old files.
# [Used for Incoming/REJECT]
#
def flush_old ():
+ Logger.log(["check Incoming/REJECT for old files"])
for f in os.listdir('.'):
if os.path.isfile(f):
if os.stat(f)[stat.ST_MTIME] < delete_date:
- if Options["No-Action"]:
- print "I: Would delete '%s'." % (os.path.basename(f))
- else:
- if Options["Verbose"]:
- print "Removing '%s' (to '%s')." % (os.path.basename(f), del_dir)
- remove(f)
+ remove(f)
else:
if Options["Verbose"]:
print "Skipping, too new, '%s'." % (os.path.basename(f))
all_files = {}
changes_files = []
+ Logger.log(["check Incoming for old orphaned files"])
# Build up the list of all files in the directory
for i in os.listdir('.'):
if os.path.isfile(i):
# a .dsc) and should be deleted if old enough.
for f in all_files.keys():
if os.stat(f)[stat.ST_MTIME] < delete_date:
- if Options["No-Action"]:
- print "I: Would delete '%s'." % (os.path.basename(f))
- else:
- if Options["Verbose"]:
- print "Removing '%s' (to '%s')." % (os.path.basename(f), del_dir)
- remove(f)
+ remove(f)
else:
if Options["Verbose"]:
print "Skipping, too new, '%s'." % (os.path.basename(f))
################################################################################
def main ():
- global Cnf, Options
+ global Options, Logger
- Cnf = utils.get_conf()
+ cnf = Config()
for i in ["Help", "Incoming", "No-Action", "Verbose" ]:
- if not Cnf.has_key("Clean-Queues::Options::%s" % (i)):
- Cnf["Clean-Queues::Options::%s" % (i)] = ""
- if not Cnf.has_key("Clean-Queues::Options::Days"):
- Cnf["Clean-Queues::Options::Days"] = "14"
+ if not cnf.has_key("Clean-Queues::Options::%s" % (i)):
+ cnf["Clean-Queues::Options::%s" % (i)] = ""
+ if not cnf.has_key("Clean-Queues::Options::Days"):
+ cnf["Clean-Queues::Options::Days"] = "14"
Arguments = [('h',"help","Clean-Queues::Options::Help"),
('d',"days","Clean-Queues::Options::Days", "IntLevel"),
('n',"no-action","Clean-Queues::Options::No-Action"),
('v',"verbose","Clean-Queues::Options::Verbose")]
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Clean-Queues::Options")
+ apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
+ Options = cnf.SubTree("Clean-Queues::Options")
if Options["Help"]:
usage()
- init()
+ Logger = daklog.Logger(cnf, 'clean-queues', Options['No-Action'])
+
+ init(cnf)
if Options["Verbose"]:
print "Processing incoming..."
flush_orphans()
- reject = Cnf["Dir::Queue::Reject"]
+ reject = cnf["Dir::Queue::Reject"]
if os.path.exists(reject) and os.path.isdir(reject):
if Options["Verbose"]:
print "Processing incoming/REJECT..."
os.chdir(reject)
flush_old()
+ Logger.close()
+
#######################################################################################
if __name__ == '__main__':
################################################################################
-import os, pg, stat, sys, time
+import os, stat, sys, time
import apt_pkg
+from datetime import datetime, timedelta
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
+from daklib import daklog
################################################################################
-projectB = None
-Cnf = None
Options = None
-now_date = None; # mark newly "deleted" things as deleted "now"
-delete_date = None; # delete things marked "deleted" earler than this
-max_delete = None
+Logger = None
################################################################################
################################################################################
-def check_binaries():
- global delete_date, now_date
-
+def check_binaries(now_date, delete_date, max_delete, session):
print "Checking for orphaned binary packages..."
# Get the list of binary packages not in a suite and mark them for
# deletion.
- q = projectB.query("""
-SELECT b.file FROM binaries b, files f
+
+ q = session.execute("""
+SELECT b.file, f.filename FROM binaries b, files f
WHERE f.last_used IS NULL AND b.file = f.id
AND NOT EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
- ql = q.getresult()
- projectB.query("BEGIN WORK")
- for i in ql:
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id))
- projectB.query("COMMIT WORK")
+ for i in q.fetchall():
+ Logger.log(["set lastused", i[1]])
+ session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid AND last_used IS NULL",
+ {'lastused': now_date, 'fileid': i[0]})
+ session.commit()
# Check for any binaries which are marked for eventual deletion
# but are now used again.
- q = projectB.query("""
-SELECT b.file FROM binaries b, files f
+
+ q = session.execute("""
+SELECT b.file, f.filename FROM binaries b, files f
WHERE f.last_used IS NOT NULL AND f.id = b.file
AND EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
- ql = q.getresult()
- projectB.query("BEGIN WORK")
- for i in ql:
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id))
- projectB.query("COMMIT WORK")
+ for i in q.fetchall():
+ Logger.log(["unset lastused", i[1]])
+ session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid", {'fileid': i[0]})
+ session.commit()
########################################
-def check_sources():
- global delete_date, now_date
-
+def check_sources(now_date, delete_date, max_delete, session):
print "Checking for orphaned source packages..."
# Get the list of source packages not in a suite and not used by
# any binaries.
- q = projectB.query("""
-SELECT s.id, s.file FROM source s, files f
+ q = session.execute("""
+SELECT s.id, s.file, f.filename FROM source s, files f
WHERE f.last_used IS NULL AND s.file = f.id
AND NOT EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id)
AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)""")
#### have been marked for deletion (so the delay between bins go
#### byebye and sources go byebye is 0 instead of StayOfExecution)
- ql = q.getresult()
-
- projectB.query("BEGIN WORK")
- for i in ql:
+ for i in q.fetchall():
source_id = i[0]
dsc_file_id = i[1]
+ dsc_fname = i[2]
# Mark the .dsc file for deletion
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, dsc_file_id))
+ Logger.log(["set lastused", dsc_fname])
+ session.execute("""UPDATE files SET last_used = :last_used
+ WHERE id = :dscfileid AND last_used IS NULL""",
+ {'last_used': now_date, 'dscfileid': dsc_file_id})
+
# Mark all other files references by .dsc too if they're not used by anyone else
- x = projectB.query("SELECT f.id FROM files f, dsc_files d WHERE d.source = %s AND d.file = f.id" % (source_id))
- for j in x.getresult():
+ x = session.execute("""SELECT f.id, f.filename FROM files f, dsc_files d
+ WHERE d.source = :sourceid AND d.file = f.id""",
+ {'sourceid': source_id})
+ for j in x.fetchall():
file_id = j[0]
- y = projectB.query("SELECT id FROM dsc_files d WHERE d.file = %s" % (file_id))
- if len(y.getresult()) == 1:
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id))
- projectB.query("COMMIT WORK")
+ file_name = j[1]
+ y = session.execute("SELECT id FROM dsc_files d WHERE d.file = :fileid", {'fileid': file_id})
+ if len(y.fetchall()) == 1:
+ Logger.log(["set lastused", file_name])
+ session.execute("""UPDATE files SET last_used = :lastused
+ WHERE id = :fileid AND last_used IS NULL""",
+ {'lastused': now_date, 'fileid': file_id})
+
+ session.commit()
# Check for any sources which are marked for deletion but which
# are now used again.
- q = projectB.query("""
-SELECT f.id FROM source s, files f, dsc_files df
+ q = session.execute("""
+SELECT f.id, f.filename FROM source s, files f, dsc_files df
WHERE f.last_used IS NOT NULL AND s.id = df.source AND df.file = f.id
AND ((EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id))
OR (EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)))""")
#### XXX: this should also handle deleted binaries specially (ie, not
#### reinstate sources because of them
- ql = q.getresult()
- # Could be done in SQL; but left this way for hysterical raisins
- # [and freedom to innovate don'cha know?]
- projectB.query("BEGIN WORK")
- for i in ql:
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id))
- projectB.query("COMMIT WORK")
+ for i in q.fetchall():
+ Logger.log(["unset lastused", i[1]])
+ session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid",
+ {'fileid': i[0]})
-########################################
+ session.commit()
-def check_files():
- global delete_date, now_date
+########################################
+def check_files(now_date, delete_date, max_delete, session):
# FIXME: this is evil; nothing should ever be in this state. if
- # they are, it's a bug and the files should not be auto-deleted.
+ # they are, it's a bug.
- return
+ # However, we've discovered it happens sometimes so we print a huge warning
+ # and then mark the file for deletion. This probably masks a bug somwhere
+ # else but is better than collecting cruft forever
print "Checking for unused files..."
- q = projectB.query("""
-SELECT id FROM files f
+ q = session.execute("""
+SELECT id, filename FROM files f
WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = f.id)
- AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)""")
-
- projectB.query("BEGIN WORK")
- for i in q.getresult():
- file_id = i[0]
- projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s" % (now_date, file_id))
- projectB.query("COMMIT WORK")
-
-def clean_binaries():
- global delete_date, now_date
-
+ AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)
+ AND last_used IS NULL
+ ORDER BY filename""")
+
+ ql = q.fetchall()
+ if len(ql) > 0:
+ utils.warn("check_files found something it shouldn't")
+ for x in ql:
+ utils.warn("orphaned file: %s" % x)
+ Logger.log(["set lastused", x[1], "ORPHANED FILE"])
+ session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid",
+ {'lastused': now_date, 'fileid': x[0]})
+
+ session.commit()
+
+def clean_binaries(now_date, delete_date, max_delete, session):
# We do this here so that the binaries we remove will have their
# source also removed (if possible).
# XXX: why doesn't this remove the files here as well? I don't think it
# buys anything keeping this separate
print "Cleaning binaries from the DB..."
+ print "Deleting from binaries table... "
+ for bin in session.query(DBBinary).join(DBBinary.poolfile).filter(PoolFile.last_used <= delete_date):
+ Logger.log(["delete binary", bin.poolfile.filename])
+ if not Options["No-Action"]:
+ session.delete(bin)
if not Options["No-Action"]:
- before = time.time()
- sys.stdout.write("[Deleting from binaries table... ")
- projectB.query("DELETE FROM binaries WHERE EXISTS (SELECT 1 FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')" % (delete_date))
- sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+ session.commit()
########################################
-def clean():
- global delete_date, now_date, max_delete
+def clean(now_date, delete_date, max_delete, session):
+ cnf = Config()
+
count = 0
size = 0
print "Cleaning out packages..."
- date = time.strftime("%Y-%m-%d")
- dest = Cnf["Dir::Morgue"] + '/' + Cnf["Clean-Suites::MorgueSubDir"] + '/' + date
+ cur_date = now_date.strftime("%Y-%m-%d")
+ dest = os.path.join(cnf["Dir::Morgue"], cnf["Clean-Suites::MorgueSubDir"], cur_date)
if not os.path.exists(dest):
os.mkdir(dest)
# Delete from source
+ print "Deleting from source table... "
+ q = session.execute("""
+SELECT s.id, f.filename FROM source s, files f
+ WHERE f.last_used <= :deletedate
+ AND s.file = f.id""", {'deletedate': delete_date})
+ for s in q.fetchall():
+ Logger.log(["delete source", s[1], s[0]])
+ if not Options["No-Action"]:
+ session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]})
+ session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]})
+
if not Options["No-Action"]:
- before = time.time()
- sys.stdout.write("[Deleting from source table... ")
- projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date))
- projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date))
- sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+ session.commit()
# Delete files from the pool
- query = "SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)
+ old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date)
if max_delete is not None:
- query += " LIMIT %d" % max_delete
- sys.stdout.write("Limiting removals to %d\n" % max_delete)
+ old_files = old_files.limit(max_delete)
+ print "Limiting removals to %d" % max_delete
- q=projectB.query(query)
- for i in q.getresult():
- filename = i[0] + i[1]
+ for pf in old_files:
+ filename = os.path.join(pf.location.path, pf.filename)
if not os.path.exists(filename):
utils.warn("can not find '%s'." % (filename))
continue
+ Logger.log(["delete pool file", filename])
if os.path.isfile(filename):
if os.path.islink(filename):
count += 1
- if Options["No-Action"]:
- print "Removing symlink %s..." % (filename)
- else:
+ Logger.log(["delete symlink", filename])
+ if not Options["No-Action"]:
os.unlink(filename)
else:
size += os.stat(filename)[stat.ST_SIZE]
if os.path.exists(dest_filename):
dest_filename = utils.find_next_free(dest_filename)
- if Options["No-Action"]:
- print "Cleaning %s -> %s ..." % (filename, dest_filename)
- else:
+ Logger.log(["move to morgue", filename, dest_filename])
+ if not Options["No-Action"]:
utils.move(filename, dest_filename)
+
+ if not Options["No-Action"]:
+ session.delete(pf)
+
else:
utils.fubar("%s is neither symlink nor file?!" % (filename))
- # Delete from the 'files' table
if not Options["No-Action"]:
- before = time.time()
- sys.stdout.write("[Deleting from files table... ")
- projectB.query("DELETE FROM files WHERE last_used <= '%s'" % (delete_date))
- sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+ session.commit()
+
if count > 0:
- sys.stderr.write("Cleaned %d files, %s.\n" % (count, utils.size_type(size)))
+ Logger.log(["total", count, utils.size_type(size)])
+ print "Cleaned %d files, %s." % (count, utils.size_type(size))
################################################################################
-def clean_maintainers():
+def clean_maintainers(now_date, delete_date, max_delete, session):
print "Cleaning out unused Maintainer entries..."
- q = projectB.query("""
-SELECT m.id FROM maintainer m
+ # TODO Replace this whole thing with one SQL statement
+ q = session.execute("""
+SELECT m.id, m.name FROM maintainer m
WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id)
AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id OR s.changedby = m.id)
AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)""")
- ql = q.getresult()
count = 0
- projectB.query("BEGIN WORK")
- for i in ql:
+
+ for i in q.fetchall():
maintainer_id = i[0]
+ Logger.log(["delete maintainer", i[1]])
if not Options["No-Action"]:
- projectB.query("DELETE FROM maintainer WHERE id = %s" % (maintainer_id))
- count += 1
- projectB.query("COMMIT WORK")
+ session.execute("DELETE FROM maintainer WHERE id = :maint", {'maint': maintainer_id})
+ count += 1
+
+ if not Options["No-Action"]:
+ session.commit()
if count > 0:
- sys.stderr.write("Cleared out %d maintainer entries.\n" % (count))
+ Logger.log(["total", count])
+ print "Cleared out %d maintainer entries." % (count)
################################################################################
-def clean_fingerprints():
+def clean_fingerprints(now_date, delete_date, max_delete, session):
print "Cleaning out unused fingerprint entries..."
- q = projectB.query("""
-SELECT f.id FROM fingerprint f
+ # TODO Replace this whole thing with one SQL statement
+ q = session.execute("""
+SELECT f.id, f.fingerprint FROM fingerprint f
WHERE f.keyring IS NULL
AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""")
- ql = q.getresult()
count = 0
- projectB.query("BEGIN WORK")
- for i in ql:
+
+ for i in q.fetchall():
fingerprint_id = i[0]
+ Logger.log(["delete fingerprint", i[1]])
if not Options["No-Action"]:
- projectB.query("DELETE FROM fingerprint WHERE id = %s" % (fingerprint_id))
- count += 1
- projectB.query("COMMIT WORK")
+ session.execute("DELETE FROM fingerprint WHERE id = :fpr", {'fpr': fingerprint_id})
+ count += 1
+
+ if not Options["No-Action"]:
+ session.commit()
if count > 0:
- sys.stderr.write("Cleared out %d fingerprint entries.\n" % (count))
+ Logger.log(["total", count])
+ print "Cleared out %d fingerprint entries." % (count)
################################################################################
-def clean_queue_build():
- global now_date
+def clean_queue_build(now_date, delete_date, max_delete, session):
+
+ cnf = Config()
- if not Cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]:
+ if not cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]:
return
print "Cleaning out queue build symlinks..."
- our_delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Clean-Suites::QueueBuildStayOfExecution"])))
+ our_delete_date = now_date - timedelta(seconds = int(cnf["Clean-Suites::QueueBuildStayOfExecution"]))
count = 0
- q = projectB.query("SELECT filename FROM queue_build WHERE last_used <= '%s'" % (our_delete_date))
- for i in q.getresult():
- filename = i[0]
- if not os.path.exists(filename):
- utils.warn("%s (from queue_build) doesn't exist." % (filename))
+ for qf in session.query(QueueBuild).filter(QueueBuild.last_used <= our_delete_date):
+ if not os.path.exists(qf.filename):
+ utils.warn("%s (from queue_build) doesn't exist." % (qf.filename))
continue
- if not Cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(filename):
- utils.fubar("%s (from queue_build) should be a symlink but isn't." % (filename))
- os.unlink(filename)
+
+ if not cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(qf.filename):
+ utils.fubar("%s (from queue_build) should be a symlink but isn't." % (qf.filename))
+
+ Logger.log(["delete queue build", qf.filename])
+ if not Options["No-Action"]:
+ os.unlink(qf.filename)
+ session.delete(qf)
count += 1
- projectB.query("DELETE FROM queue_build WHERE last_used <= '%s'" % (our_delete_date))
+
+ if not Options["No-Action"]:
+ session.commit()
if count:
- sys.stderr.write("Cleaned %d queue_build files.\n" % (count))
+ Logger.log(["total", count])
+ print "Cleaned %d queue_build files." % (count)
+
+################################################################################
+
+def clean_empty_directories(session):
+ """
+ Removes empty directories from pool directories.
+ """
+
+ count = 0
+
+ cursor = session.execute(
+ "SELECT DISTINCT(path) FROM location WHERE type = :type",
+ {'type': 'pool'},
+ )
+ bases = [x[0] for x in cursor.fetchall()]
+
+ for base in bases:
+ for dirpath, dirnames, filenames in os.walk(base, topdown=False):
+ if not filenames and not dirnames:
+ to_remove = os.path.join(base, dirpath)
+ if not Options["No-Action"]:
+ Logger.log(["removing directory", to_remove])
+ os.removedirs(to_remove)
+ count += 1
+
+ if count:
+ Logger.log(["total removed directories", count])
################################################################################
def main():
- global Cnf, Options, projectB, delete_date, now_date, max_delete
+ global Options, Logger
+
+ cnf = Config()
- Cnf = utils.get_conf()
for i in ["Help", "No-Action", "Maximum" ]:
- if not Cnf.has_key("Clean-Suites::Options::%s" % (i)):
- Cnf["Clean-Suites::Options::%s" % (i)] = ""
+ if not cnf.has_key("Clean-Suites::Options::%s" % (i)):
+ cnf["Clean-Suites::Options::%s" % (i)] = ""
Arguments = [('h',"help","Clean-Suites::Options::Help"),
('n',"no-action","Clean-Suites::Options::No-Action"),
('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")]
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Clean-Suites::Options")
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Clean-Suites::Options")
- if Cnf["Clean-Suites::Options::Maximum"] != "":
+ if cnf["Clean-Suites::Options::Maximum"] != "":
try:
# Only use Maximum if it's an integer
- max_delete = int(Cnf["Clean-Suites::Options::Maximum"])
+ max_delete = int(cnf["Clean-Suites::Options::Maximum"])
if max_delete < 1:
utils.fubar("If given, Maximum must be at least 1")
except ValueError, e:
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ Logger = daklog.Logger(cnf, "clean-suites", debug=Options["No-Action"])
+
+ session = DBConn().session()
+
+ now_date = datetime.now()
+ delete_date = now_date - timedelta(seconds=int(cnf['Clean-Suites::StayOfExecution']))
- now_date = time.strftime("%Y-%m-%d %H:%M")
- delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Clean-Suites::StayOfExecution"])))
+ check_binaries(now_date, delete_date, max_delete, session)
+ clean_binaries(now_date, delete_date, max_delete, session)
+ check_sources(now_date, delete_date, max_delete, session)
+ check_files(now_date, delete_date, max_delete, session)
+ clean(now_date, delete_date, max_delete, session)
+ clean_maintainers(now_date, delete_date, max_delete, session)
+ clean_fingerprints(now_date, delete_date, max_delete, session)
+ clean_queue_build(now_date, delete_date, max_delete, session)
+ clean_empty_directories(session)
- check_binaries()
- clean_binaries()
- check_sources()
- check_files()
- clean()
- clean_maintainers()
- clean_fingerprints()
- clean_queue_build()
+ Logger.close()
################################################################################
+++ /dev/null
-#!/usr/bin/env python
-
-""" Check for fixable discrepancies between stable and unstable """
-# Copyright (C) 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-
-################################################################################
-
-import pg, sys
-import apt_pkg
-from daklib import database
-from daklib import utils
-
-################################################################################
-
-Cnf = None
-projectB = None
-
-################################################################################
-
-def usage(exit_code=0):
- print """Usage: dak compare-suites
-Looks for fixable descrepancies between stable and unstable.
-
- -h, --help show this help and exit."""
- sys.exit(exit_code)
-
-################################################################################
-
-def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
- Arguments = [('h',"help","Compare-Suites::Options::Help")]
- for i in [ "help" ]:
- if not Cnf.has_key("Compare-Suites::Options::%s" % (i)):
- Cnf["Compare-Suites::Options::%s" % (i)] = ""
-
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
-
- Options = Cnf.SubTree("Compare-Suites::Options")
- if Options["Help"]:
- usage()
-
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
- src_suite = "stable"
- dst_suite = "unstable"
-
- src_suite_id = database.get_suite_id(src_suite)
- dst_suite_id = database.get_suite_id(dst_suite)
- arch_all_id = database.get_architecture_id("all")
- dsc_type_id = database.get_override_type_id("dsc")
-
- for arch in database.get_suite_architectures(src_suite_id):
- if arch == "source":
- continue
-
- # Arch: all doesn't work; consider packages which go from
- # arch: all to arch: any, e.g. debconf... needs more checks
- # and thought later.
-
- if arch == "all":
- continue
- arch_id = database.get_architecture_id(arch)
- q = projectB.query("""
-SELECT b_src.package, b_src.version, a.arch_string
- FROM binaries b_src, bin_associations ba, override o, architecture a
- WHERE ba.bin = b_src.id AND ba.suite = %s AND b_src.architecture = %s
- AND a.id = b_src.architecture AND o.package = b_src.package
- AND o.suite = %s AND o.type != %s AND NOT EXISTS
- (SELECT 1 FROM bin_associations ba2, binaries b_dst
- WHERE ba2.bin = b_dst.id AND b_dst.package = b_src.package
- AND (b_dst.architecture = %s OR b_dst.architecture = %s)
- AND ba2.suite = %s AND EXISTS
- (SELECT 1 FROM bin_associations ba3, binaries b2
- WHERE ba3.bin = b2.id AND ba3.suite = %s AND b2.package = b_dst.package))
-ORDER BY b_src.package;"""
- % (src_suite_id, arch_id, dst_suite_id, dsc_type_id, arch_id, arch_all_id, dst_suite_id, dst_suite_id))
- for i in q.getresult():
- print " ".join(i)
-
-#######################################################################################
-
-if __name__ == '__main__':
- main()
import sys
import os
import logging
-import math
import gzip
import threading
-import traceback
import Queue
import apt_pkg
-import datetime #just for debugging, can be removed
from daklib import utils
from daklib.binary import Binary
from daklib.config import Config
-from daklib.dbconn import DBConn
-################################################################################
+from daklib.dbconn import *
-log=None
+################################################################################
def usage (exit_code=0):
print """Usage: dak contents [options] command [arguments]
options_prefix = "Contents"
options_prefix = "%s::Options" % options_prefix
-#log = logging.getLogger()
+log = logging.getLogger()
################################################################################
-# get all the arches delivered for a given suite
-# this should probably exist somehere common
-arches_q = """PREPARE arches_q(int) as
- SELECT s.architecture, a.arch_string
- FROM suite_architectures s
- JOIN architecture a ON (s.architecture=a.id)
- WHERE suite = $1"""
-
-# find me the .deb for a given binary id
-debs_q = """PREPARE debs_q(int, int) as
- SELECT b.id, f.filename FROM bin_assoc_by_arch baa
- JOIN binaries b ON baa.bin=b.id
- JOIN files f ON b.file=f.id
- WHERE suite = $1
- AND arch = $2"""
-
-# find me all of the contents for a given .deb
-contents_q = """PREPARE contents_q(int,int) as
- SELECT file, section, package
- FROM deb_contents
- WHERE suite = $1
- AND (arch = $2 or arch=2)"""
-# ORDER BY file"""
-
-# find me all of the contents for a given .udeb
-udeb_contents_q = """PREPARE udeb_contents_q(int,int,text, int) as
- SELECT file, section, package, arch
- FROM udeb_contents
- WHERE suite = $1
- AND otype = $2
- AND section = $3
- and arch = $4
- ORDER BY file"""
-
-
-# clear out all of the temporarily stored content associations
-# this should be run only after p-a has run. after a p-a
-# run we should have either accepted or rejected every package
-# so there should no longer be anything in the queue
-remove_pending_contents_cruft_q = """DELETE FROM pending_content_associations"""
-
class EndOfContents(object):
- pass
-
-class OneAtATime(object):
"""
+ A sentry object for the end of the filename stream
"""
- def __init__(self):
- self.next_in_line = None
- self.next_lock = threading.Condition()
-
- def enqueue(self, next):
- self.next_lock.acquire()
- while self.next_in_line:
- self.next_lock.wait()
-
- assert( not self.next_in_line )
- self.next_in_line = next
- self.next_lock.notify()
- self.next_lock.release()
-
- def dequeue(self):
- self.next_lock.acquire()
- while not self.next_in_line:
- self.next_lock.wait()
- result = self.next_in_line
- self.next_in_line = None
- self.next_lock.notify()
- self.next_lock.release()
- return result
-
-
-class ContentsWorkThread(threading.Thread):
+ pass
+
+class GzippedContentWriter(object):
"""
+ An object which will write contents out to a Contents-$arch.gz
+ file on a separate thread
"""
- def __init__(self, upstream, downstream):
- threading.Thread.__init__(self)
- self.upstream = upstream
- self.downstream = downstream
- def run(self):
- while True:
- try:
- contents_file = self.upstream.dequeue()
- if isinstance(contents_file,EndOfContents):
- if self.downstream:
- self.downstream.enqueue(contents_file)
- break
-
- s = datetime.datetime.now()
- print("%s start: %s" % (self,contents_file) )
- self._run(contents_file)
- print("%s finished: %s in %d seconds" % (self, contents_file, (datetime.datetime.now()-s).seconds ))
- if self.downstream:
- self.downstream.enqueue(contents_file)
- except:
- traceback.print_exc()
-
-class QueryThread(ContentsWorkThread):
- def __init__(self, upstream, downstream):
- ContentsWorkThread.__init__(self, upstream, downstream)
-
- def __str__(self):
- return "QueryThread"
- __repr__ = __str__
-
- def _run(self, contents_file):
- contents_file.query()
-
-class IngestThread(ContentsWorkThread):
- def __init__(self, upstream, downstream):
- ContentsWorkThread.__init__(self, upstream, downstream)
-
- def __str__(self):
- return "IngestThread"
- __repr__ = __str__
-
- def _run(self, contents_file):
- contents_file.ingest()
-
-class SortThread(ContentsWorkThread):
- def __init__(self, upstream, downstream):
- ContentsWorkThread.__init__(self, upstream, downstream)
-
- def __str__(self):
- return "SortThread"
- __repr__ = __str__
-
- def _run(self, contents_file):
- contents_file.sorted_keys = sorted(contents_file.filenames.keys())
-
-class OutputThread(ContentsWorkThread):
- def __init__(self, upstream, downstream):
- ContentsWorkThread.__init__(self, upstream, downstream)
-
- def __str__(self):
- return "OutputThread"
- __repr__ = __str__
-
- def _run(self, contents_file):
- contents_file.open_file()
- for fname in contents_file.sorted_keys:
- contents_file.filehandle.write("%s\t%s\n" % (fname,contents_file.filenames[fname]))
- contents_file.sorted_keys = None
- contents_file.filenames.clear()
-
-class GzipThread(ContentsWorkThread):
- def __init__(self, upstream, downstream):
- ContentsWorkThread.__init__(self, upstream, downstream)
-
- def __str__(self):
- return "GzipThread"
- __repr__ = __str__
-
- def _run(self, contents_file):
- os.system("gzip -f %s" % contents_file.filename)
-
-class ContentFile(object):
- def __init__(self,
- filename,
- suite_str,
- suite_id,
- arch_str,
- arch_id):
-
- self.filename = filename
- self.filenames = {}
- self.sorted_keys = None
- self.suite_str = suite_str
- self.suite_id = suite_id
- self.arch_str = arch_str
- self.arch_id = arch_id
- self.cursor = None
- self.filehandle = None
-
- def __str__(self):
- return self.filename
- __repr__ = __str__
-
-
- def cleanup(self):
- self.filenames = None
- self.sortedkeys = None
- self.filehandle.close()
- self.cursor.close()
-
- def query(self):
- self.cursor = DBConn().cursor();
-
- self.cursor.execute("""SELECT file, section || '/' || package
- FROM deb_contents
- WHERE ( arch=2 or arch = %d) AND suite = %d
- """ % (self.arch_id, self.suite_id))
-
- def ingest(self):
- while True:
- r = self.cursor.fetchone()
- if not r:
- break
- filename, package = r
- if self.filenames.has_key(filename):
- self.filenames[filename] += ",%s" % (package)
- else:
- self.filenames[filename] = "%s" % (package)
- self.cursor.close()
+ header = None # a class object holding the header section of contents file
- def open_file(self):
+ def __init__(self, filename):
+ """
+ @type filename: string
+ @param filename: the name of the file to write to
+ """
+ self.queue = Queue.Queue()
+ self.current_file = None
+ self.first_package = True
+ self.output = self.open_file(filename)
+ self.thread = threading.Thread(target=self.write_thread,
+ name='Contents writer')
+ self.thread.start()
+
+ def open_file(self, filename):
"""
opens a gzip stream to the contents file
"""
-# filepath = Config()["Contents::Root"] + self.filename
- self.filename = "/home/stew/contents/" + self.filename
- filedir = os.path.dirname(self.filename)
+ filepath = Config()["Contents::Root"] + filename
+ filedir = os.path.dirname(filepath)
if not os.path.isdir(filedir):
os.makedirs(filedir)
-# self.filehandle = gzip.open(self.filename, "w")
- self.filehandle = open(self.filename, "w")
- self._write_header()
+ return gzip.open(filepath, "w")
- def _write_header(self):
- self._get_header();
- self.filehandle.write(ContentFile.header)
+ def write(self, filename, section, package):
+ """
+ enqueue content to be written to the file on a separate thread
+ """
+ self.queue.put((filename,section,package))
- header=None
+ def write_thread(self):
+ """
+ the target of a Thread which will do the actual writing
+ """
+ while True:
+ next = self.queue.get()
+ if isinstance(next, EndOfContents):
+ self.output.write('\n')
+ self.output.close()
+ break
+
+ (filename,section,package)=next
+ if next != self.current_file:
+ # this is the first file, so write the header first
+ if not self.current_file:
+ self.output.write(self._getHeader())
+
+ self.output.write('\n%s\t' % filename)
+ self.first_package = True
+
+ self.current_file=filename
+
+ if not self.first_package:
+ self.output.write(',')
+ else:
+ self.first_package=False
+ self.output.write('%s/%s' % (section,package))
+
+ def finish(self):
+ """
+ enqueue the sentry object so that writers will know to terminate
+ """
+ self.queue.put(EndOfContents())
@classmethod
- def _get_header(self):
+ def _getHeader(self):
"""
Internal method to return the header for Contents.gz files
This is boilerplate which explains the contents of the file and how
it can be used.
"""
- if not ContentFile.header:
+ if not GzippedContentWriter.header:
if Config().has_key("Contents::Header"):
try:
h = open(os.path.join( Config()["Dir::Templates"],
Config()["Contents::Header"] ), "r")
- ContentFile.header = h.read()
+ GzippedContentWriter.header = h.read()
h.close()
except:
log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"],
traceback.format_exc() ))
- ContentFile.header = None
+ GzippedContentWriter.header = None
else:
- ContentFile.header = None
+ GzippedContentWriter.header = None
+
+ return GzippedContentWriter.header
- return ContentFile.header
class Contents(object):
"""
Class capable of generating Contents-$arch.gz files
-
- Usage GenerateContents().generateContents( ["main","contrib","non-free"] )
"""
def __init__(self):
def reject(self, message):
log.error("E: %s" % message)
- # goal column for section column
- _goal_column = 54
-
def cruft(self):
"""
remove files/paths from the DB which are no longer referenced
by binaries and clean the temporary table
"""
- cursor = DBConn().cursor();
- cursor.execute( "BEGIN WORK" )
- cursor.execute( remove_pending_contents_cruft_q )
- cursor.execute( remove_filename_cruft_q )
- cursor.execute( remove_filepath_cruft_q )
- cursor.execute( "COMMIT" )
+ s = DBConn().session()
+
+ # clear out all of the temporarily stored content associations
+ # this should be run only after p-a has run. after a p-a
+ # run we should have either accepted or rejected every package
+ # so there should no longer be anything in the queue
+ s.query(PendingContentAssociation).delete()
+
+ # delete any filenames we are storing which have no binary associated
+ # with them
+ cafq = s.query(ContentAssociation.filename_id).distinct()
+ cfq = s.query(ContentFilename)
+ cfq = cfq.filter(~ContentFilename.cafilename_id.in_(cafq))
+ cfq.delete()
+
+ # delete any paths we are storing which have no binary associated with
+ # them
+ capq = s.query(ContentAssociation.filepath_id).distinct()
+ cpq = s.query(ContentFilepath)
+ cpq = cpq.filter(~ContentFilepath.cafilepath_id.in_(capq))
+ cpq.delete()
+
+ s.commit()
def bootstrap(self):
"""
pooldir = Config()[ 'Dir::Pool' ]
- cursor = DBConn().cursor();
- DBConn().prepare("debs_q",debs_q)
- DBConn().prepare("arches_q",arches_q)
-
- suites = self._suites()
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
-
- arch_list = self._arches(cursor, suite_id)
- arch_all_id = DBConn().get_architecture_id("all")
- for arch_id in arch_list:
- cursor.execute( "EXECUTE debs_q(%d, %d)" % ( suite_id, arch_id[0] ) )
-
- count = 0
- while True:
- deb = cursor.fetchone()
- if not deb:
- break
- count += 1
- cursor1 = DBConn().cursor();
- cursor1.execute( "SELECT 1 FROM deb_contents WHERE binary_id = %d LIMIT 1" % (deb[0] ) )
- old = cursor1.fetchone()
- if old:
- log.log( "already imported: %s" % (deb[1]) )
+ s = DBConn().session()
+
+ for suite in s.query(Suite).all():
+ for arch in get_suite_architectures(suite.suite_name, skipsrc=True, skipall=True, session=s):
+ q = s.query(BinAssociation).join(Suite)
+ q = q.join(Suite).filter_by(suite_name=suite.suite_name)
+ q = q.join(DBBinary).join(Architecture).filter_by(arch.arch_string)
+ for ba in q:
+ filename = ba.binary.poolfile.filename
+ # Check for existing contents
+ existingq = s.query(ContentAssociations).filter_by(binary_pkg=ba.binary_id).limit(1)
+ if existingq.count() > 0:
+ log.debug( "already imported: %s" % (filename))
else:
-# log.debug( "scanning: %s" % (deb[1]) )
- log.log( "scanning: %s" % (deb[1]) )
- debfile = os.path.join( pooldir, deb[1] )
- if os.path.exists( debfile ):
- Binary(debfile, self.reject).scan_package(deb[0], True)
+ # We don't have existing contents so import them
+ log.debug( "scanning: %s" % (filename) )
+ debfile = os.path.join(pooldir, filename)
+ if os.path.exists(debfile):
+ Binary(debfile, self.reject).scan_package(ba.binary_id, True)
else:
- log.error("missing .deb: %s" % deb[1])
+ log.error("missing .deb: %s" % filename)
def generate(self):
- """
- Generate contents files for both deb and udeb
- """
- DBConn().prepare("arches_q", arches_q)
- self.deb_generate()
-# self.udeb_generate()
-
- def deb_generate(self):
"""
Generate Contents-$arch.gz files for every available arch in each given suite.
"""
- cursor = DBConn().cursor()
- debtype_id = DBConn().get_override_type_id("deb")
- suites = self._suites()
-
- inputtoquery = OneAtATime()
- querytoingest = OneAtATime()
- ingesttosort = OneAtATime()
- sorttooutput = OneAtATime()
- outputtogzip = OneAtATime()
-
- qt = QueryThread(inputtoquery,querytoingest)
- it = IngestThread(querytoingest,ingesttosort)
-# these actually make things worse
-# it2 = IngestThread(querytoingest,ingesttosort)
-# it3 = IngestThread(querytoingest,ingesttosort)
-# it4 = IngestThread(querytoingest,ingesttosort)
- st = SortThread(ingesttosort,sorttooutput)
- ot = OutputThread(sorttooutput,outputtogzip)
- gt = GzipThread(outputtogzip, None)
-
- qt.start()
- it.start()
-# it2.start()
-# it3.start()
-# it2.start()
- st.start()
- ot.start()
- gt.start()
-
- # Get our suites, and the architectures
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
- arch_list = self._arches(cursor, suite_id)
-
- for (arch_id,arch_str) in arch_list:
- print( "suite: %s, arch: %s time: %s" %(suite_id, arch_id, datetime.datetime.now().isoformat()) )
-
-# filename = "dists/%s/Contents-%s.gz" % (suite, arch_str)
- filename = "dists/%s/Contents-%s" % (suite, arch_str)
- cf = ContentFile(filename, suite, suite_id, arch_str, arch_id)
- inputtoquery.enqueue( cf )
-
- inputtoquery.enqueue( EndOfContents() )
- gt.join()
-
- def udeb_generate(self):
- """
- Generate Contents-$arch.gz files for every available arch in each given suite.
- """
- cursor = DBConn().cursor()
-
- DBConn().prepare("udeb_contents_q", udeb_contents_q)
- udebtype_id=DBConn().get_override_type_id("udeb")
- suites = self._suites()
+ session = DBConn().session()
-# for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"),
-# ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
+ arch_all_id = get_architecture("all", session).arch_id
- for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s"),
- ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s")]:
+ # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
+ # This is HORRIBLY debian specific :-/
+ for dtype, section, fn_pattern in \
+ [('deb', None, "dists/%s/Contents-%s.gz"),
+ ('udeb', "debian-installer", "dists/%s/Contents-udeb-%s.gz"),
+ ('udeb', "non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
- section_id = DBConn().get_section_id(section) # all udebs should be here)
- if section_id != -1:
+ overridetype = get_override_type(dtype, session)
- # Get our suites, and the architectures
- for suite in [i.lower() for i in suites]:
- suite_id = DBConn().get_suite_id(suite)
- arch_list = self._arches(cursor, suite_id)
+ # For udebs, we only look in certain sections (see the for loop above)
+ if section is not None:
+ section = get_section(section, session)
- for arch_id in arch_list:
+ # Get our suites
+ for suite in which_suites():
+ # Which architectures do we need to work on
+ arch_list = get_suite_architectures(suite.suite_name, skipsrc=True, skipall=True, session=session)
- writer = GzippedContentWriter(fn_pattern % (suite, arch_id[1]))
- try:
-
- cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id, arch_id))
+ # Set up our file writer dictionary
+ file_writers = {}
+ try:
+ # One file writer per arch
+ for arch in arch_list:
+ file_writers[arch.arch_id] = GzippedContentWriter(fn_pattern % (suite, arch.arch_string))
- while True:
- r = cursor.fetchone()
- if not r:
- break
+ for r in get_suite_contents(suite, overridetype, section, session=session).fetchall():
+ filename, section, package, arch_id = r
- filename, section, package, arch = r
+ if arch_id == arch_all_id:
+ # It's arch all, so all contents files get it
+ for writer in file_writers.values():
writer.write(filename, section, package)
- finally:
- writer.close()
-
-
-
-################################################################################
-
- def _suites(self):
- """
- return a list of suites to operate on
- """
- if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
- suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
- else:
- suites = [ 'unstable', 'testing' ]
-# suites = Config().SubTree("Suite").List()
-
- return suites
-
- def _arches(self, cursor, suite):
- """
- return a list of archs to operate on
- """
- arch_list = []
- cursor.execute("EXECUTE arches_q(%d)" % (suite))
- while True:
- r = cursor.fetchone()
- if not r:
- break
-
- if r[1] != "source" and r[1] != "all":
- arch_list.append((r[0], r[1]))
+ else:
+ if file_writers.has_key(arch_id):
+ file_writers[arch_id].write(filename, section, package)
- return arch_list
+ finally:
+ # close all the files
+ for writer in file_writers.values():
+ writer.finish()
################################################################################
-
def main():
cnf = Config()
-# log = logging.Logger(cnf, "contents")
-
+
arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"),
('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
if cnf.has_key("%s::%s" % (options_prefix,"Help")):
usage()
-# level=logging.INFO
-# if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
-# level=logging.ERROR
+ level=logging.INFO
+ if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
+ level=logging.ERROR
-# elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
-# level=logging.DEBUG
+ elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
+ level=logging.DEBUG
-# logging.basicConfig( level=level,
-# format='%(asctime)s %(levelname)s %(message)s',
-# stream = sys.stderr )
+ logging.basicConfig( level=level,
+ format='%(asctime)s %(levelname)s %(message)s',
+ stream = sys.stderr )
commands[args[0]](Contents())
+def which_suites(session):
+ """
+ return a list of suites to operate on
+ """
+ if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
+ suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
+ else:
+ suites = Config().SubTree("Suite").List()
+
+ return [get_suite(s.lower(), session) for s in suites]
+
+
if __name__ == '__main__':
main()
################################################################################
-import pg, sys, time
+import sys, time
import apt_pkg
+
+from daklib.dbconn import *
+from daklib.config import Config
from daklib import utils
-from daklib import database
-from daklib import logging
+from daklib import daklog
from daklib.regexes import re_comments
################################################################################
-Cnf = None
-projectB = None
Logger = None
################################################################################
################################################################################
-def process_file (file, suite, component, type, action, noaction=0):
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
+def process_file(file, suite, component, otype, mode, action, session):
+ cnf = Config()
+
+ s = get_suite(suite, session=session)
+ if s is None:
utils.fubar("Suite '%s' not recognised." % (suite))
+ suite_id = s.suite_id
- component_id = database.get_component_id(component)
- if component_id == -1:
+ c = get_component(component, session=session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
+ component_id = c.component_id
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (type))
+ o = get_override_type(otype)
+ if o is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (otype))
+ type_id = o.overridetype_id
# --set is done mostly internal for performance reasons; most
# invocations of --set will be updates and making people wait 2-3
c_removed = 0
c_error = 0
- q = projectB.query("SELECT o.package, o.priority, o.section, o.maintainer, p.priority, s.section FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s and o.priority = p.id and o.section = s.id"
- % (suite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT o.package, o.priority, o.section, o.maintainer, p.priority, s.section
+ FROM override o, priority p, section s
+ WHERE o.suite = :suiteid AND o.component = :componentid AND o.type = :typeid
+ and o.priority = p.id and o.section = s.id""",
+ {'suiteid': suite_id, 'componentid': component_id, 'typeid': type_id})
+ for i in q.fetchall():
original[i[0]] = i[1:]
start_time = time.time()
- if not noaction:
- projectB.query("BEGIN WORK")
+
+ section_cache = get_sections(session)
+ priority_cache = get_priorities(session)
+
+ # Our session is already in a transaction
+
for line in file.readlines():
line = re_comments.sub('', line).strip()
if line == "":
continue
- maintainer_override = ""
- if type == "dsc":
+ maintainer_override = None
+ if otype == "dsc":
split_line = line.split(None, 2)
if len(split_line) == 2:
(package, section) = split_line
c_error += 1
continue
- section_id = database.get_section_id(section)
- if section_id == -1:
+ if not section_cache.has_key(section):
utils.warn("'%s' is not a valid section. ['%s' in suite %s, component %s]." % (section, package, suite, component))
c_error += 1
continue
- priority_id = database.get_priority_id(priority)
- if priority_id == -1:
+
+ section_id = section_cache[section]
+
+ if not priority_cache.has_key(priority):
utils.warn("'%s' is not a valid priority. ['%s' in suite %s, component %s]." % (priority, package, suite, component))
c_error += 1
continue
+ priority_id = priority_cache[priority]
+
if new.has_key(package):
utils.warn("Can't insert duplicate entry for '%s'; ignoring all but the first. [suite %s, component %s]" % (package, suite, component))
c_error += 1
continue
new[package] = ""
+
if original.has_key(package):
(old_priority_id, old_section_id, old_maintainer_override, old_priority, old_section) = original[package]
- if action == "add" or old_priority_id == priority_id and \
+ if mode == "add" or old_priority_id == priority_id and \
old_section_id == section_id and \
old_maintainer_override == maintainer_override:
# If it's unchanged or we're in 'add only' mode, ignore it
# If it's changed, delete the old one so we can
# reinsert it with the new information
c_updated += 1
- if not noaction:
- projectB.query("DELETE FROM override WHERE suite = %s AND component = %s AND package = '%s' AND type = %s"
- % (suite_id, component_id, package, type_id))
+ if action:
+ session.execute("""DELETE FROM override WHERE suite = :suite AND component = :component
+ AND package = :package AND type = :typeid""",
+ {'suite': suite_id, 'component': component_id,
+ 'package': package, 'typeid': type_id})
+
# Log changes
if old_priority_id != priority_id:
- Logger.log(["changed priority",package,old_priority,priority])
+ Logger.log(["changed priority", package, old_priority, priority])
if old_section_id != section_id:
- Logger.log(["changed section",package,old_section,section])
+ Logger.log(["changed section", package, old_section, section])
if old_maintainer_override != maintainer_override:
- Logger.log(["changed maintainer override",package,old_maintainer_override,maintainer_override])
+ Logger.log(["changed maintainer override", package, old_maintainer_override, maintainer_override])
update_p = 1
- elif action == "change":
+ elif mode == "change":
# Ignore additions in 'change only' mode
c_skipped += 1
continue
c_added += 1
update_p = 0
- if not noaction:
- if maintainer_override:
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '%s')"
- % (suite_id, component_id, type_id, package, priority_id, section_id, maintainer_override))
+ if action:
+ if not maintainer_override:
+ m_o = None
else:
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section,maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')"
- % (suite_id, component_id, type_id, package, priority_id, section_id))
+ m_o = maintainer_override
+ session.execute("""INSERT INTO override (suite, component, type, package,
+ priority, section, maintainer)
+ VALUES (:suiteid, :componentid, :typeid,
+ :package, :priorityid, :sectionid,
+ :maintainer)""",
+ {'suiteid': suite_id, 'componentid': component_id,
+ 'typeid': type_id, 'package': package,
+ 'priorityid': priority_id, 'sectionid': section_id,
+ 'maintainer': m_o})
if not update_p:
- Logger.log(["new override",suite,component,type,package,priority,section,maintainer_override])
+ Logger.log(["new override", suite, component, otype, package,priority,section,maintainer_override])
- if action == "set":
+ if mode == "set":
# Delete any packages which were removed
for package in original.keys():
if not new.has_key(package):
- if not noaction:
- projectB.query("DELETE FROM override WHERE suite = %s AND component = %s AND package = '%s' AND type = %s"
- % (suite_id, component_id, package, type_id))
+ if action:
+ session.execute("""DELETE FROM override
+ WHERE suite = :suiteid AND component = :componentid
+ AND package = :package AND type = :typeid""",
+ {'suiteid': suite_id, 'componentid': component_id,
+ 'package': package, 'typeid': type_id})
c_removed += 1
- Logger.log(["removed override",suite,component,type,package])
+ Logger.log(["removed override", suite, component, otype, package])
- if not noaction:
- projectB.query("COMMIT WORK")
- if not Cnf["Control-Overrides::Options::Quiet"]:
+ if action:
+ session.commit()
+
+ if not cnf["Control-Overrides::Options::Quiet"]:
print "Done in %d seconds. [Updated = %d, Added = %d, Removed = %d, Skipped = %d, Errors = %d]" % (int(time.time()-start_time), c_updated, c_added, c_removed, c_skipped, c_error)
- Logger.log(["set complete",c_updated, c_added, c_removed, c_skipped, c_error])
+
+ Logger.log(["set complete", c_updated, c_added, c_removed, c_skipped, c_error])
################################################################################
-def list_overrides(suite, component, type):
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
+def list_overrides(suite, component, otype, session):
+ dat = {}
+ s = get_suite(suite, session)
+ if s is None:
utils.fubar("Suite '%s' not recognised." % (suite))
- component_id = database.get_component_id(component)
- if component_id == -1:
+ dat['suiteid'] = s.suite_id
+
+ c = get_component(component, session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
+ dat['componentid'] = c.component_id
+
+ o = get_override_type(otype)
+ if o is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
- if type == "dsc":
- q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, type_id))
- for i in q.getresult():
+ dat['typeid'] = o.overridetype_id
+
+ if otype == "dsc":
+ q = session.execute("""SELECT o.package, s.section, o.maintainer FROM override o, section s
+ WHERE o.suite = :suiteid AND o.component = :componentid
+ AND o.type = :typeid AND o.section = s.id
+ ORDER BY s.section, o.package""", dat)
+ for i in q.fetchall():
print utils.result_join(i)
else:
- q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT o.package, p.priority, s.section, o.maintainer, p.level
+ FROM override o, priority p, section s
+ WHERE o.suite = :suiteid AND o.component = :componentid
+ AND o.type = :typeid AND o.priority = p.id AND o.section = s.id
+ ORDER BY s.section, p.level, o.package""", dat)
+ for i in q.fetchall():
print utils.result_join(i[:-1])
################################################################################
def main ():
- global Cnf, projectB, Logger
+ global Logger
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a', "add", "Control-Overrides::Options::Add"),
('c', "component", "Control-Overrides::Options::Component", "HasArg"),
('h', "help", "Control-Overrides::Options::Help"),
# Default arguments
for i in [ "add", "help", "list", "quiet", "set", "change", "no-action" ]:
- if not Cnf.has_key("Control-Overrides::Options::%s" % (i)):
- Cnf["Control-Overrides::Options::%s" % (i)] = ""
- if not Cnf.has_key("Control-Overrides::Options::Component"):
- Cnf["Control-Overrides::Options::Component"] = "main"
- if not Cnf.has_key("Control-Overrides::Options::Suite"):
- Cnf["Control-Overrides::Options::Suite"] = "unstable"
- if not Cnf.has_key("Control-Overrides::Options::Type"):
- Cnf["Control-Overrides::Options::Type"] = "deb"
-
- file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
-
- if Cnf["Control-Overrides::Options::Help"]:
+ if not cnf.has_key("Control-Overrides::Options::%s" % (i)):
+ cnf["Control-Overrides::Options::%s" % (i)] = ""
+ if not cnf.has_key("Control-Overrides::Options::Component"):
+ cnf["Control-Overrides::Options::Component"] = "main"
+ if not cnf.has_key("Control-Overrides::Options::Suite"):
+ cnf["Control-Overrides::Options::Suite"] = "unstable"
+ if not cnf.has_key("Control-Overrides::Options::Type"):
+ cnf["Control-Overrides::Options::Type"] = "deb"
+
+ file_list = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+
+ if cnf["Control-Overrides::Options::Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
- action = None
+ mode = None
for i in [ "add", "list", "set", "change" ]:
- if Cnf["Control-Overrides::Options::%s" % (i)]:
- if action:
+ if cnf["Control-Overrides::Options::%s" % (i)]:
+ if mode:
utils.fubar("Can not perform more than one action at once.")
- action = i
+ mode = i
+
+ # Need an action...
+ if mode is None:
+ utils.fubar("No action specified.")
- (suite, component, otype) = (Cnf["Control-Overrides::Options::Suite"],
- Cnf["Control-Overrides::Options::Component"],
- Cnf["Control-Overrides::Options::Type"])
+ (suite, component, otype) = (cnf["Control-Overrides::Options::Suite"],
+ cnf["Control-Overrides::Options::Component"],
+ cnf["Control-Overrides::Options::Type"])
- if action == "list":
- list_overrides(suite, component, otype)
+ if mode == "list":
+ list_overrides(suite, component, otype, session)
else:
- if database.get_suite_untouchable(suite):
+ if get_suite(suite).untouchable:
utils.fubar("%s: suite is untouchable" % suite)
- noaction = 0
- if Cnf["Control-Overrides::Options::No-Action"]:
+ action = True
+ if cnf["Control-Overrides::Options::No-Action"]:
utils.warn("In No-Action Mode")
- noaction = 1
+ action = False
- Logger = logging.Logger(Cnf, "control-overrides", noaction)
+ Logger = daklog.Logger(cnf.Cnf, "control-overrides", mode)
if file_list:
for f in file_list:
- process_file(utils.open_file(f), suite, component, otype, action, noaction)
+ process_file(utils.open_file(f), suite, component, otype, mode, action, session)
else:
- process_file(sys.stdin, suite, component, otype, action, noaction)
+ process_file(sys.stdin, suite, component, otype, mode, action, session)
Logger.close()
#######################################################################################
#######################################################################################
-import pg, sys
+import sys
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
from daklib import utils
#######################################################################################
-Cnf = None
-projectB = None
Logger = None
################################################################################
#######################################################################################
-def get_id (package, version, architecture):
+def get_id(package, version, architecture, session):
if architecture == "source":
- q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
+ q = session.execute("SELECT id FROM source WHERE source = :package AND version = :version",
+ {'package': package, 'version': version})
else:
- q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
-
- ql = q.getresult()
- if not ql:
+ q = session.execute("""SELECT b.id FROM binaries b, architecture a
+ WHERE b.package = :package AND b.version = :version
+ AND (a.arch_string = :arch OR a.arch_string = 'all')
+ AND b.architecture = a.id""",
+ {'package': package, 'version': version, 'arch': architecture})
+
+ ql = q.fetchall()
+ if len(ql) < 1:
utils.warn("Couldn't find '%s_%s_%s'." % (package, version, architecture))
return None
+
if len(ql) > 1:
utils.warn("Found more than one match for '%s_%s_%s'." % (package, version, architecture))
return None
+
return ql[0][0]
#######################################################################################
-def set_suite (file, suite_id):
+def set_suite(file, suite, session):
+ suite_id = suite.suite_id
lines = file.readlines()
- projectB.query("BEGIN WORK")
+ # Our session is already in a transaction
# Build up a dictionary of what is currently in the suite
current = {}
- q = projectB.query("SELECT b.package, b.version, a.arch_string, ba.id FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+ q = session.execute("""SELECT b.package, b.version, a.arch_string, ba.id
+ FROM binaries b, bin_associations ba, architecture a
+ WHERE ba.suite = :suiteid
+ AND ba.bin = b.id AND b.architecture = a.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
key = " ".join(i[:3])
current[key] = i[3]
- q = projectB.query("SELECT s.source, s.version, sa.id FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+
+ q = session.execute("""SELECT s.source, s.version, sa.id
+ FROM source s, src_associations sa
+ WHERE sa.suite = :suiteid
+ AND sa.source = s.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
key = " ".join(i[:2]) + " source"
current[key] = i[2]
(package, version, architecture) = key.split()
pkid = current[key]
if architecture == "source":
- q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (pkid))
+ session.execute("""DELETE FROM src_associations WHERE id = :pkid""", {'pkid': pkid})
else:
- q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (pkid))
+ session.execute("""DELETE FROM bin_associations WHERE id = :pkid""", {'pkid': pkid})
Logger.log(["removed", key, pkid])
# Check to see which packages need added and add them
for key in desired.keys():
if not current.has_key(key):
(package, version, architecture) = key.split()
- pkid = get_id (package, version, architecture)
+ pkid = get_id (package, version, architecture, session)
if not pkid:
continue
if architecture == "source":
- q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO src_associations (suite, source)
+ VALUES (:suiteid, :pkid)""", {'suiteid': suite_id, 'pkid': pkid})
else:
- q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO bin_associations (suite, bin)
+ VALUES (:suiteid, :pkid)""", {'suiteid': suite_id, 'pkid': pkid})
Logger.log(["added", key, pkid])
- projectB.query("COMMIT WORK")
+ session.commit()
#######################################################################################
-def process_file (file, suite, action):
-
- suite_id = database.get_suite_id(suite)
-
+def process_file(file, suite, action, session):
if action == "set":
- set_suite (file, suite_id)
+ set_suite(file, suite, session)
return
- lines = file.readlines()
+ suite_id = suite.suite_id
- projectB.query("BEGIN WORK")
+ lines = file.readlines()
+ # Our session is already in a transaction
for line in lines:
split_line = line.strip().split()
if len(split_line) != 3:
(package, version, architecture) = split_line
- pkid = get_id(package, version, architecture)
+ pkid = get_id(package, version, architecture, session)
if not pkid:
continue
if architecture == "source":
- # Find the existing assoications ID, if any
- q = projectB.query("SELECT id FROM src_associations WHERE suite = %s and source = %s" % (suite_id, pkid))
- ql = q.getresult()
- if not ql:
- assoication_id = None
+ # Find the existing association ID, if any
+ q = session.execute("""SELECT id FROM src_associations
+ WHERE suite = :suiteid and source = :pkid""",
+ {'suiteid': suite_id, 'pkid': pkid})
+ ql = q.fetchall()
+ if len(ql) < 1:
+ association_id = None
else:
- assoication_id = ql[0][0]
+ association_id = ql[0][0]
+
# Take action
if action == "add":
- if assoication_id:
+ if association_id:
utils.warn("'%s_%s_%s' already exists in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO src_associations (suite, source)
+ VALUES (:suiteid, :pkid)""",
+ {'suiteid': suite_id, 'pkid': pkid})
elif action == "remove":
- if assoication_id == None:
+ if association_id == None:
utils.warn("'%s_%s_%s' doesn't exist in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (assoication_id))
+ session.execute("""DELETE FROM src_associations WHERE id = :pkid""", {'pkid': association_id})
else:
- # Find the existing assoications ID, if any
- q = projectB.query("SELECT id FROM bin_associations WHERE suite = %s and bin = %s" % (suite_id, pkid))
- ql = q.getresult()
- if not ql:
- assoication_id = None
+ # Find the existing associations ID, if any
+ q = session.execute("""SELECT id FROM bin_associations
+ WHERE suite = :suiteid and bin = :pkid""",
+ {'suiteid': suite_id, 'pkid': pkid})
+ ql = q.fetchall()
+ if len(ql) < 1:
+ association_id = None
else:
- assoication_id = ql[0][0]
+ association_id = ql[0][0]
+
# Take action
if action == "add":
- if assoication_id:
+ if association_id:
utils.warn("'%s_%s_%s' already exists in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, pkid))
+ session.execute("""INSERT INTO bin_associations (suite, bin)
+ VALUES (%s, %s)""",
+ {'suiteid': suite_id, 'pkid': pkid})
elif action == "remove":
- if assoication_id == None:
+ if association_id == None:
utils.warn("'%s_%s_%s' doesn't exist in suite %s." % (package, version, architecture, suite))
continue
else:
- q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (assoication_id))
+ session.execute("""DELETE FROM bin_associations WHERE id = :pkid""", {'pkid': association_id})
- projectB.query("COMMIT WORK")
+ session.commit()
#######################################################################################
-def get_list (suite):
- suite_id = database.get_suite_id(suite)
+def get_list(suite, session):
+ suite_id = suite.suite_id
# List binaries
- q = projectB.query("SELECT b.package, b.version, a.arch_string FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+ q = session.execute("""SELECT b.package, b.version, a.arch_string
+ FROM binaries b, bin_associations ba, architecture a
+ WHERE ba.suite = :suiteid
+ AND ba.bin = b.id AND b.architecture = a.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
print " ".join(i)
# List source
- q = projectB.query("SELECT s.source, s.version FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id))
- ql = q.getresult()
- for i in ql:
+ q = session.execute("""SELECT s.source, s.version
+ FROM source s, src_associations sa
+ WHERE sa.suite = :suiteid
+ AND sa.source = s.id""", {'suiteid': suite_id})
+ for i in q.fetchall():
print " ".join(i) + " source"
#######################################################################################
def main ():
- global Cnf, projectB, Logger
+ global Logger
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a',"add","Control-Suite::Options::Add", "HasArg"),
('h',"help","Control-Suite::Options::Help"),
('s',"set", "Control-Suite::Options::Set", "HasArg")]
for i in ["add", "help", "list", "remove", "set", "version" ]:
- if not Cnf.has_key("Control-Suite::Options::%s" % (i)):
- Cnf["Control-Suite::Options::%s" % (i)] = ""
+ if not cnf.has_key("Control-Suite::Options::%s" % (i)):
+ cnf["Control-Suite::Options::%s" % (i)] = ""
try:
- file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
+ file_list = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv);
except SystemError, e:
print "%s\n" % e
usage(1)
- Options = Cnf.SubTree("Control-Suite::Options")
+ Options = cnf.SubTree("Control-Suite::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"],int(Cnf["DB::Port"]))
-
- database.init(Cnf, projectB)
+ session = DBConn().session()
action = None
for i in ("add", "list", "remove", "set"):
- if Cnf["Control-Suite::Options::%s" % (i)] != "":
- suite = Cnf["Control-Suite::Options::%s" % (i)]
- if database.get_suite_id(suite) == -1:
- utils.fubar("Unknown suite '%s'." %(suite))
+ if cnf["Control-Suite::Options::%s" % (i)] != "":
+ suite_name = cnf["Control-Suite::Options::%s" % (i)]
+ suite = get_suite(suite_name, session=session)
+ if suite is None:
+ utils.fubar("Unknown suite '%s'." % (suite_name))
else:
if action:
utils.fubar("Can only perform one action at a time.")
utils.fubar("No action specified.")
# Safety/Sanity check
- if action == "set" and suite not in ["testing", "etch-m68k"]:
- utils.fubar("Will not reset suite %s" % (suite))
+ # XXX: This should be stored in the database
+ if action == "set" and suite_name not in ["testing", "etch-m68k"]:
+ utils.fubar("Will not reset suite %s" % (suite_name))
if action == "list":
- get_list(suite)
+ get_list(suite, session)
else:
- Logger = logging.Logger(Cnf, "control-suite")
+ Logger = daklog.Logger(cnf.Cnf, "control-suite")
if file_list:
for f in file_list:
- process_file(utils.open_file(f), suite, action)
+ process_file(utils.open_file(f), suite, action, session)
else:
- process_file(sys.stdin, suite, action)
+ process_file(sys.stdin, suite, action, session)
Logger.close()
#######################################################################################
# you might as well write some letters to God about how unfair entropy
# is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au
-## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add overrides, avoid ANAIS for duplicated packages
+## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of
+## duplicate source package stuff, improve experimental ?, add overrides,
+## avoid ANAIS for duplicated packages
################################################################################
-import commands, pg, os, sys, time, re
+import commands, os, sys, re
import apt_pkg
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_extract_src_version
################################################################################
-Cnf = None
-projectB = None
-suite = "unstable" # Default
-suite_id = None
no_longer_in_suite = {}; # Really should be static to add_nbs, but I'm lazy
source_binaries = {}
################################################################################
-def add_nbs(nbs_d, source, version, package):
+def add_nbs(nbs_d, source, version, package, suite_id, session):
# Ensure the package is still in the suite (someone may have already removed it)
if no_longer_in_suite.has_key(package):
return
else:
- q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package))
- if not q.getresult():
+ q = session.execute("""SELECT b.id FROM binaries b, bin_associations ba
+ WHERE ba.bin = b.id AND ba.suite = :suite_id
+ AND b.package = :package LIMIT 1""", {'suite_id': suite_id,
+ 'package': package})
+ if not q.fetchall():
no_longer_in_suite[package] = ""
return
################################################################################
# Check for packages built on architectures they shouldn't be.
-def do_anais(architecture, binaries_list, source):
+def do_anais(architecture, binaries_list, source, session):
if architecture == "any" or architecture == "all":
return ""
for arch in architecture.split():
architectures[arch.strip()] = ""
for binary in binaries_list:
- q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary))
- ql = q.getresult()
+ q = session.execute("""SELECT a.arch_string, b.version
+ FROM binaries b, bin_associations ba, architecture a
+ WHERE ba.suite = :suiteid AND ba.bin = b.id
+ AND b.architecture = a.id AND b.package = :package""",
+ {'suiteid': suite_id, 'package': binary})
versions = []
- for i in ql:
+ for i in q.fetchall():
arch = i[0]
version = i[1]
if architectures.has_key(arch):
print
def parse_nfu(architecture):
+ cnf = Config()
# utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
r = re.compile("^\w+/([^_]+)_.*: Not-For-Us")
ret = set()
- filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
+ filename = "%s/%s-all.txt" % (cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
# Not all architectures may have a wanna-build dump, so we want to ignore missin
# files
################################################################################
-def do_nviu():
- experimental_id = database.get_suite_id("experimental")
- if experimental_id == -1:
+def do_newer_version(lowersuite_name, highersuite_name, code, session):
+ lowersuite = get_suite(lowersuite_name, session)
+ if not lowersuite:
+ return
+
+ highersuite = get_suite(highersuite_name, session)
+ if not highersuite:
return
- # Check for packages in experimental obsoleted by versions in unstable
- q = projectB.query("""
-SELECT s.source, s.version AS experimental, s2.version AS unstable
+
+ # Check for packages in $highersuite obsoleted by versions in $lowersuite
+ q = session.execute("""
+SELECT s.source, s.version AS lower, s2.version AS higher
FROM src_associations sa, source s, source s2, src_associations sa2
- WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id
+ WHERE sa.suite = :highersuite_id AND sa2.suite = :lowersuite_id AND sa.source = s.id
AND sa2.source = s2.id AND s.source = s2.source
- AND s.version < s2.version""" % (experimental_id,
- database.get_suite_id("unstable")))
- ql = q.getresult()
+ AND s.version < s2.version""", {'lowersuite_id': lowersuite.suite_id,
+ 'highersuite_id': highersuite.suite_id})
+ ql = q.fetchall()
if ql:
- nviu_to_remove = []
- print "Newer version in unstable"
- print "-------------------------"
+ nv_to_remove = []
+ print "Newer version in %s" % lowersuite.suite_name
+ print "-----------------" + "-" * len(lowersuite.suite_name)
print
for i in ql:
- (source, experimental_version, unstable_version) = i
- print " o %s (%s, %s)" % (source, experimental_version, unstable_version)
- nviu_to_remove.append(source)
+ (source, higher_version, lower_version) = i
+ print " o %s (%s, %s)" % (source, higher_version, lower_version)
+ nv_to_remove.append(source)
print
print "Suggested command:"
- print " dak rm -m \"[auto-cruft] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove))
+ print " dak rm -m \"[auto-cruft] %s\" -s %s %s" % (code, highersuite.suite_name,
+ " ".join(nv_to_remove))
print
################################################################################
output += " o %s: %s\n" % (version, ", ".join(packages))
if all_packages:
all_packages.sort()
- cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite, " ".join(all_packages))
+ cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite.suite_name, " ".join(all_packages))
output += "\n"
print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
print
-def get_suite_binaries():
+def get_suite_binaries(suite, session):
# Initalize a large hash table of all binary packages
binaries = {}
- before = time.time()
- sys.stderr.write("[Getting a list of binary packages in %s..." % (suite))
- q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id))
- ql = q.getresult()
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
- for i in ql:
+ print "Getting a list of binary packages in %s..." % suite.suite_name
+ q = session.execute("""SELECT distinct b.package
+ FROM binaries b, bin_associations ba
+ WHERE ba.suite = :suiteid AND ba.bin = b.id""",
+ {'suiteid': suite.suite_id})
+ for i in q.fetchall():
binaries[i[0]] = ""
return binaries
################################################################################
def main ():
- global Cnf, projectB, suite, suite_id, source_binaries, source_versions
+ global suite, suite_id, source_binaries, source_versions
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Cruft-Report::Options::Help"),
('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
('s',"suite","Cruft-Report::Options::Suite","HasArg"),
('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
for i in [ "help" ]:
- if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
- Cnf["Cruft-Report::Options::%s" % (i)] = ""
- Cnf["Cruft-Report::Options::Suite"] = Cnf["Dinstall::DefaultSuite"]
+ if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
+ cnf["Cruft-Report::Options::%s" % (i)] = ""
+ cnf["Cruft-Report::Options::Suite"] = cnf["Dinstall::DefaultSuite"]
- if not Cnf.has_key("Cruft-Report::Options::Mode"):
- Cnf["Cruft-Report::Options::Mode"] = "daily"
+ if not cnf.has_key("Cruft-Report::Options::Mode"):
+ cnf["Cruft-Report::Options::Mode"] = "daily"
- if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
- Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
+ if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
+ cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Cruft-Report::Options")
+ Options = cnf.SubTree("Cruft-Report::Options")
if Options["Help"]:
usage()
utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
usage(1)
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
nfu_packages = {}
- suite = Options["Suite"]
- suite_id = database.get_suite_id(suite)
+ suite = get_suite(Options["Suite"].lower(), session)
+ suite_id = suite.suite_id
+ suite_name = suite.suite_name.lower()
bin_not_built = {}
if "bnb" in checks:
- bins_in_suite = get_suite_binaries()
+ bins_in_suite = get_suite_binaries(suite_name, session)
# Checks based on the Sources files
- components = Cnf.ValueList("Suite::%s::Components" % (suite))
+ components = cnf.ValueList("Suite::%s::Components" % (suite_name))
for component in components:
- filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component)
+ filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite_name, component)
# apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
bin_not_built[source][binary] = ""
if "anais" in checks:
- anais_output += do_anais(architecture, binaries_list, source)
+ anais_output += do_anais(architecture, binaries_list, source, session)
# Check for duplicated packages and build indices for checking "no source" later
source_index = component + '/' + source
# Checks based on the Packages files
check_components = components[:]
- if suite != "experimental":
+ if suite_name != "experimental":
check_components.append('main/debian-installer');
+
for component in check_components:
- architectures = filter(utils.real_arch, database.get_suite_architectures(suite))
+ architectures = [ a.arch_string for a in get_suite_architectures(suite_name,
+ skipsrc=True, skipall=True,
+ session=session) ]
for architecture in architectures:
- if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
- continue
- filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture)
+ if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
+ continue
+ filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suite_name, component, architecture)
# apt_pkg.ParseTagFile needs a real file handle
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
latest_version = versions.pop()
source_version = source_versions.get(source,"0")
if apt_pkg.VersionCompare(latest_version, source_version) == 0:
- add_nbs(dubious_nbs, source, latest_version, package)
+ add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
else:
- add_nbs(real_nbs, source, latest_version, package)
+ add_nbs(real_nbs, source, latest_version, package, suite_id, session)
if "nviu" in checks:
- do_nviu()
+ do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nbs" in checks:
do_nbs(real_nbs)
################################################################################
import sys
-import imp
import daklib.utils
-import daklib.extensions
-
-################################################################################
-
-class UserExtension:
- def __init__(self, user_extension = None):
- if user_extension:
- m = imp.load_source("dak_userext", user_extension)
- d = m.__dict__
- else:
- m, d = None, {}
- self.__dict__["_module"] = m
- self.__dict__["_d"] = d
-
- def __getattr__(self, a):
- if a in self.__dict__: return self.__dict__[a]
- if a[0] == "_": raise AttributeError, a
- return self._d.get(a, None)
-
- def __setattr__(self, a, v):
- self._d[a] = v
-
-################################################################################
-
-class UserExtension:
- def __init__(self, user_extension = None):
- if user_extension:
- m = imp.load_source("dak_userext", user_extension)
- d = m.__dict__
- else:
- m, d = None, {}
- self.__dict__["_module"] = m
- self.__dict__["_d"] = d
-
- def __getattr__(self, a):
- if a in self.__dict__: return self.__dict__[a]
- if a[0] == "_": raise AttributeError, a
- return self._d.get(a, None)
-
- def __setattr__(self, a, v):
- self._d[a] = v
################################################################################
"Override cruft checks"),
("check-proposed-updates",
"Dependency checking for proposed-updates"),
- ("compare-suites",
- "Show fixable discrepencies between suites"),
("control-overrides",
"Manipulate/list override entries in bulk"),
("control-suite",
"Show information useful for NEW processing"),
("find-null-maintainers",
"Check for users with no packages in the archive"),
- ("import-archive",
- "Populate SQL database based from an archive tree"),
("import-keyring",
"Populate fingerprint/uid table based on a new/updated keyring"),
("import-ldap-fingerprints",
"Syncs fingerprint and uid tables with Debian LDAP db"),
("import-users-from-passwd",
"Sync PostgreSQL users with passwd file"),
+ ("admin",
+ "Perform administration on the dak database"),
("init-db",
"Update the database to match the conf file"),
("update-db",
"Generates override files"),
("poolize",
"Move packages from dists/ to pool/"),
- ("reject-proposed-updates",
- "Manually reject from proposed-updates"),
("new-security-install",
"New way to install a security upload into the archive"),
("split-done",
def main():
"""Launch dak functionality."""
- Cnf = daklib.utils.get_conf()
-
- if Cnf.has_key("Dinstall::UserExtensions"):
- userext = UserExtension(Cnf["Dinstall::UserExtensions"])
- else:
- userext = UserExtension()
-
functionality = init()
modules = [ command for (command, _) in functionality ]
# Invoke the module
module = __import__(cmdname.replace("-","_"))
- module.dak_userext = userext
- userext.dak_module = module
-
- daklib.extensions.init(cmdname, module, userext)
- if userext.init is not None: userext.init(cmdname)
-
module.main()
################################################################################
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Make sure we always have primary keys
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+
+################################################################################
+
+def do_update(self):
+ print "Adding primary keys to various tables"
+
+ try:
+ c = self.db.cursor()
+ c.execute("ALTER TABLE content_associations ADD PRIMARY KEY (id)")
+ c.execute("ALTER TABLE override ADD PRIMARY KEY (suite, component, package, type)")
+ c.execute("ALTER TABLE pending_content_associations ADD PRIMARY KEY (id)")
+ c.execute("ALTER TABLE queue_build ADD PRIMARY KEY (suite, queue, filename)")
+ c.execute("ALTER TABLE suite_architectures ADD PRIMARY KEY (suite, architecture)")
+
+ c.execute("UPDATE config SET value = '14' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply process-new update 14, rollback issued. Error message : %s" % (str(msg))
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Adding table for allowed source formats
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Raphael Hertzog <hertzog@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+
+################################################################################
+
+def do_update(self):
+ print "Adding tables listing allowed source formats"
+
+ try:
+ c = self.db.cursor()
+ c.execute("""
+ CREATE TABLE src_format (
+ id SERIAL PRIMARY KEY,
+ format_name TEXT NOT NULL,
+ UNIQUE (format_name)
+ )
+ """)
+ c.execute("INSERT INTO src_format (format_name) VALUES('1.0')")
+ c.execute("INSERT INTO src_format (format_name) VALUES('3.0 (quilt)')")
+ c.execute("INSERT INTO src_format (format_name) VALUES('3.0 (native)')")
+
+ c.execute("""
+ CREATE TABLE suite_src_formats (
+ suite INT4 NOT NULL REFERENCES suite(id),
+ src_format INT4 NOT NULL REFERENCES src_format(id),
+ PRIMARY KEY (suite, src_format)
+ )
+ """)
+
+ print "Authorize format 1.0 on all suites by default"
+ c.execute("SELECT id FROM suite")
+ suites = c.fetchall()
+ c.execute("SELECT id FROM src_format WHERE format_name = '1.0'")
+ formats = c.fetchall()
+ for s in suites:
+ for f in formats:
+ c.execute("INSERT INTO suite_src_formats (suite, src_format) VALUES(%s, %s)", (s[0], f[0]))
+
+ print "Authorize all other formats on tpu, unstable & experimental by default"
+ c.execute("SELECT id FROM suite WHERE suite_name IN ('testing-proposed-updates', 'unstable', 'experimental')")
+ suites = c.fetchall()
+ c.execute("SELECT id FROM src_format WHERE format_name != '1.0'")
+ formats = c.fetchall()
+ for s in suites:
+ for f in formats:
+ c.execute("INSERT INTO suite_src_formats (suite, src_format) VALUES(%s, %s)", (s[0], f[0]))
+
+ c.execute("UPDATE config SET value = '15' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply source format update 15, rollback issued. Error message : %s" % (str(msg))
import sys
import apt_pkg
-from daklib import queue
+from daklib.changes import Changes
from daklib import utils
################################################################################
if Options["Help"]:
usage()
- k = queue.Upload(Cnf)
+
for arg in sys.argv[1:]:
arg = utils.validate_changes_file_arg(arg,require_changes=-1)
- k.pkg.changes_file = arg
- print "%s:" % (arg)
- k.init_vars()
- k.update_vars()
-
- changes = k.pkg.changes
- print " Changes:"
- # Mandatory changes fields
- for i in [ "source", "version", "maintainer", "urgency", "changedby822",
- "changedby2047", "changedbyname", "maintainer822",
- "maintainer2047", "maintainername", "maintaineremail",
- "fingerprint", "changes" ]:
- print " %s: %s" % (i.capitalize(), changes[i])
- del changes[i]
- # Mandatory changes lists
- for i in [ "distribution", "architecture", "closes" ]:
- print " %s: %s" % (i.capitalize(), " ".join(changes[i].keys()))
- del changes[i]
- # Optional changes fields
- for i in [ "changed-by", "filecontents", "format", "adv id" ]:
- if changes.has_key(i):
- print " %s: %s" % (i.capitalize(), changes[i])
- del changes[i]
- print
- if changes:
- utils.warn("changes still has following unrecognised keys: %s" % (changes.keys()))
-
- dsc = k.pkg.dsc
- print " Dsc:"
- for i in [ "source", "version", "maintainer", "fingerprint", "uploaders",
- "bts changelog" ]:
- if dsc.has_key(i):
- print " %s: %s" % (i.capitalize(), dsc[i])
- del dsc[i]
- print
- if dsc:
- utils.warn("dsc still has following unrecognised keys: %s" % (dsc.keys()))
-
- files = k.pkg.files
- print " Files:"
- for f in files.keys():
- print " %s:" % (f)
- for i in [ "package", "version", "architecture", "type", "size",
- "md5sum", "sha1sum", "sha256sum", "component", "location id",
- "source package", "source version", "maintainer", "dbtype",
- "files id", "new", "section", "priority", "pool name" ]:
- if files[f].has_key(i):
- print " %s: %s" % (i.capitalize(), files[f][i])
- del files[f][i]
- if files[f]:
- utils.warn("files[%s] still has following unrecognised keys: %s" % (f, files[f].keys()))
- print
-
- dsc_files = k.pkg.dsc_files
- print " Dsc Files:"
- for f in dsc_files.keys():
- print " %s:" % (f)
- # Mandatory fields
- for i in [ "size", "md5sum" ]:
- print " %s: %s" % (i.capitalize(), dsc_files[f][i])
- del dsc_files[f][i]
- # Optional fields
- for i in [ "files id" ]:
- if dsc_files[f].has_key(i):
- print " %s: %s" % (i.capitalize(), dsc_files[f][i])
- del dsc_files[f][i]
- if dsc_files[f]:
- utils.warn("dsc_files[%s] still has following unrecognised keys: %s" % (f, dsc_files[f].keys()))
+ k = Changes()
+ k.load_dot_dak(arg)
+ print arg
+ print k
################################################################################
import errno
import os
-import pg
import re
import sys
import md5
import apt_inst
import shutil
import commands
-from daklib import database
+
from daklib import utils
+from daklib.dbconn import DBConn, get_binary_from_name_suite
from daklib.regexes import html_escaping, re_html_escaping, re_version, re_spacestrip, \
re_contrib, re_nonfree, re_localhost, re_newlinespace, \
re_package, re_doc_directory
################################################################################
Cnf = None
-projectB = None
-
Cnf = utils.get_conf()
-projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
-database.init(Cnf, projectB)
printed_copyrights = {}
package_relations = {} #: Store relations of packages for later output
suite_where = " ='%s'" % suite
comma_count = 1
+ session = DBConn().session()
for l in depends_tree:
if (comma_count >= 2):
result += ", "
result += " | "
# doesn't do version lookup yet.
- q = projectB.query("SELECT DISTINCT(b.package), b.version, c.name, su.suite_name FROM binaries b, files fi, location l, component c, bin_associations ba, suite su WHERE b.package='%s' AND b.file = fi.id AND fi.location = l.id AND l.component = c.id AND ba.bin=b.id AND ba.suite = su.id AND su.suite_name %s ORDER BY b.version desc" % (d['name'], suite_where))
- ql = q.getresult()
- if ql:
- i = ql[0]
+ res = get_binary_from_name_suite(d['name'], suite_where)
+ if res.rowcount > 0:
+ i = res.fetchone()
adepends = d['name']
if d['version'] != '' :
# else: => byhand
def main ():
- global Cnf, projectB, db_files, waste, excluded
+ global Cnf, db_files, waste, excluded
# Cnf = utils.get_conf()
################################################################################
-import ldap, pg, sys, time
+import ldap, sys, time
import apt_pkg
-from daklib import utils
-################################################################################
-
-Cnf = None
-projectB = None
+from daklib.dbconn import *
+from daklib.config import Config
################################################################################
return ret[0]
def main():
- global Cnf, projectB
+ cnf = Config()
- Cnf = utils.get_conf()
Arguments = [('h',"help","Find-Null-Maintainers::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Find-Null-Maintainers::Options::%s" % (i)):
- Cnf["Find-Null-Maintainers::Options::%s" % (i)] = ""
+ if not cnf.has_key("Find-Null-Maintainers::Options::%s" % (i)):
+ cnf["Find-Null-Maintainers::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Find-Null-Maintainers::Options")
+ Options = cnf.SubTree("Find-Null-Maintainers::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ session = DBConn().session()
- before = time.time()
- sys.stderr.write("[Getting info from the LDAP server...")
- LDAPDn = Cnf["Import-LDAP-Fingerprints::LDAPDn"]
- LDAPServer = Cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ print "Getting info from the LDAP server..."
+ LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
+ LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
- "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Import-Users-From-Passwd::ValidGID"]),
+ "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "cn", "mn", "sn", "createTimestamp"])
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
db_uid = {}
db_unstable_uid = {}
- before = time.time()
- sys.stderr.write("[Getting UID info for entire archive...")
- q = projectB.query("SELECT DISTINCT u.uid FROM uid u, fingerprint f WHERE f.uid = u.id;")
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
- for i in q.getresult():
+ print "Getting UID info for entire archive..."
+ q = session.execute("SELECT DISTINCT u.uid FROM uid u, fingerprint f WHERE f.uid = u.id")
+ for i in q.fetchall():
db_uid[i[0]] = ""
- before = time.time()
- sys.stderr.write("[Getting UID info for unstable...")
- q = projectB.query("""
+ print "Getting UID info for unstable..."
+ q = session.execute("""
SELECT DISTINCT u.uid FROM suite su, src_associations sa, source s, fingerprint f, uid u
WHERE f.uid = u.id AND sa.source = s.id AND sa.suite = su.id
AND su.suite_name = 'unstable' AND s.sig_fpr = f.id
SELECT DISTINCT u.uid FROM suite su, bin_associations ba, binaries b, fingerprint f, uid u
WHERE f.uid = u.id AND ba.bin = b.id AND ba.suite = su.id
AND su.suite_name = 'unstable' AND b.sig_fpr = f.id""")
- sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
- for i in q.getresult():
+ for i in q.fetchall():
db_unstable_uid[i[0]] = ""
now = time.time()
import sys
import os
import tempfile
-import subprocess
import time
import apt_pkg
-import pg
+
from daklib import utils
-from daklib import database
+from daklib.dbconn import get_suite, get_suite_architectures
################################################################################
-projectB = None
Cnf = None
Logger = None
Options = None
def main():
- global Cnf, Options, Logger, projectB
+ global Cnf, Options, Logger
os.umask(0002)
if Options.has_key("RootDir"): Cnf["Dir::Root"] = Options["RootDir"]
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
if not suites:
suites = Cnf.SubTree("Suite").List()
- for suite in suites:
- print "Processing: " + suite
- SuiteBlock = Cnf.SubTree("Suite::" + suite)
+ for suitename in suites:
+ print "Processing: " + suitename
+ SuiteBlock = Cnf.SubTree("Suite::" + suitename)
+
+ suiteobj = get_suite(suitename.lower())
- if database.get_suite_untouchable(suite):
+ # Use the canonical version of the suite name
+ suite = suiteobj.suite_name
+
+ if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
- suite = suite.lower()
-
- architectures = database.get_suite_architectures(suite)
- if architectures == None:
- architectures = []
+ architectures = get_suite_architectures(suite, skipall=True)
if SuiteBlock.has_key("Components"):
components = SuiteBlock.ValueList("Components")
print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename)
continue
- for architecture in architectures:
- if architecture == "all":
- continue
+ for archobj in architectures:
+ architecture = archobj.arch_string
if architecture != "source":
# Process Contents
################################################################################
-import sys, os, stat, time, pg
+import sys, os, stat, time
import gzip, bz2
import apt_pkg
+
from daklib import utils
-from daklib import database
from daklib.dak_exceptions import *
+from daklib.dbconn import *
################################################################################
Cnf = None
-projectB = None
out = None
AptCnf = None
################################################################################
def main ():
- global Cnf, AptCnf, projectB, out
+ global Cnf, AptCnf, out
out = sys.stdout
Cnf = utils.get_conf()
AptCnf = apt_pkg.newConfiguration()
apt_pkg.ReadConfigFileISC(AptCnf, Options["Apt-Conf"])
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
if not suites:
suites = Cnf.SubTree("Suite").List()
- for suite in suites:
- print "Processing: " + suite
- SuiteBlock = Cnf.SubTree("Suite::" + suite)
-
- if database.get_suite_untouchable(suite) and not Options["Force-Touch"]:
- print "Skipping: " + suite + " (untouchable)"
+ for suitename in suites:
+ print "Processing: " + suitename
+ SuiteBlock = Cnf.SubTree("Suite::" + suitename)
+ suiteobj = get_suite(suitename.lower())
+ if not suiteobj:
+ print "ALERT: Cannot find suite %s!" % (suitename.lower())
continue
- suite = suite.lower()
+ # Use the canonical name
+ suite = suiteobj.suite_name.lower()
- origin = SuiteBlock["Origin"]
- label = SuiteBlock.get("Label", origin)
- codename = SuiteBlock.get("CodeName", "")
+ if suiteobj.untouchable and not Options["Force-Touch"]:
+ print "Skipping: " + suite + " (untouchable)"
+ continue
+ origin = suiteobj.origin
+ label = suiteobj.label or suiteobj.origin
+ codename = suiteobj.codename or ""
version = ""
- description = ""
-
- q = projectB.query("SELECT version, description FROM suite WHERE suite_name = '%s'" % (suite))
- qs = q.getresult()
- if len(qs) == 1:
- if qs[0][0] != "-": version = qs[0][0]
- if qs[0][1]: description = qs[0][1]
+ if suiteobj.version and suiteobj.version != '-':
+ version = suiteobj.version
+ description = suiteobj.description or ""
- architectures = database.get_suite_architectures(suite)
- if architectures == None:
- architectures = []
+ architectures = get_suite_architectures(suite, skipall=True, skipsrc=True)
if SuiteBlock.has_key("NotAutomatic"):
notautomatic = "yes"
print Cnf["Dir::Root"] + tree + "/Release"
out = open(Cnf["Dir::Root"] + tree + "/Release", "w")
- out.write("Origin: %s\n" % (origin))
+ out.write("Origin: %s\n" % (suiteobj.origin))
out.write("Label: %s\n" % (label))
out.write("Suite: %s\n" % (suite))
if version != "":
if notautomatic != "":
out.write("NotAutomatic: %s\n" % (notautomatic))
- out.write("Architectures: %s\n" % (" ".join(filter(utils.real_arch, architectures))))
+ out.write("Architectures: %s\n" % (" ".join([a.arch_string for a in architectures])))
if components:
out.write("Components: %s\n" % (" ".join(components)))
+++ /dev/null
-#!/usr/bin/env python
-
-""" Populate the DB """
-# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-###############################################################################
-
-# 04:36|<aj> elmo: you're making me waste 5 seconds per architecture!!!!!! YOU BASTARD!!!!!
-
-###############################################################################
-
-# This code is a horrible mess for two reasons:
-
-# (o) For Debian's usage, it's doing something like 160k INSERTs,
-# even on auric, that makes the program unusable unless we get
-# involed in sorts of silly optimization games (local dicts to avoid
-# redundant SELECTS, using COPY FROM rather than INSERTS etc.)
-
-# (o) It's very site specific, because I don't expect to use this
-# script again in a hurry, and I don't want to spend any more time
-# on it than absolutely necessary.
-
-###############################################################################
-
-import commands, os, pg, sys, time
-import apt_pkg
-from daklib import database
-from daklib import utils
-from daklib.dak_exceptions import *
-from daklib.regexes import re_arch_from_filename, re_taint_free, re_no_epoch, \
- re_extract_src_version
-
-###############################################################################
-
-Cnf = None
-projectB = None
-files_id_cache = {}
-source_cache = {}
-arch_all_cache = {}
-binary_cache = {}
-location_path_cache = {}
-#
-files_id_serial = 0
-source_id_serial = 0
-src_associations_id_serial = 0
-dsc_files_id_serial = 0
-files_query_cache = None
-source_query_cache = None
-src_associations_query_cache = None
-dsc_files_query_cache = None
-orig_tar_gz_cache = {}
-#
-binaries_id_serial = 0
-binaries_query_cache = None
-bin_associations_id_serial = 0
-bin_associations_query_cache = None
-#
-source_cache_for_binaries = {}
-reject_message = ""
-
-################################################################################
-
-def usage(exit_code=0):
- print """Usage: dak import-archive
-Initializes a projectB database from an existing archive
-
- -a, --action actually perform the initalization
- -h, --help show this help and exit."""
- sys.exit(exit_code)
-
-###############################################################################
-
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-###############################################################################
-
-def check_signature (filename):
- if not re_taint_free.match(os.path.basename(filename)):
- reject("!!WARNING!! tainted filename: '%s'." % (filename))
- return None
-
- status_read, status_write = os.pipe()
- cmd = "gpgv --status-fd %s %s %s" \
- % (status_write, utils.gpg_keyring_args(), filename)
- (output, status, exit_status) = utils.gpgv_get_status_output(cmd, status_read, status_write)
-
- # Process the status-fd output
- keywords = {}
- bad = internal_error = ""
- for line in status.split('\n'):
- line = line.strip()
- if line == "":
- continue
- split = line.split()
- if len(split) < 2:
- internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
- continue
- (gnupg, keyword) = split[:2]
- if gnupg != "[GNUPG:]":
- internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
- continue
- args = split[2:]
- if keywords.has_key(keyword) and keyword != "NODATA" and keyword != "SIGEXPIRED":
- internal_error += "found duplicate status token ('%s').\n" % (keyword)
- continue
- else:
- keywords[keyword] = args
-
- # If we failed to parse the status-fd output, let's just whine and bail now
- if internal_error:
- reject("internal error while performing signature check on %s." % (filename))
- reject(internal_error, "")
- reject("Please report the above errors to the Archive maintainers by replying to this mail.", "")
- return None
-
- # Now check for obviously bad things in the processed output
- if keywords.has_key("SIGEXPIRED"):
- utils.warn("%s: signing key has expired." % (filename))
- if keywords.has_key("KEYREVOKED"):
- reject("key used to sign %s has been revoked." % (filename))
- bad = 1
- if keywords.has_key("BADSIG"):
- reject("bad signature on %s." % (filename))
- bad = 1
- if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
- reject("failed to check signature on %s." % (filename))
- bad = 1
- if keywords.has_key("NO_PUBKEY"):
- args = keywords["NO_PUBKEY"]
- if len(args) < 1:
- reject("internal error while checking signature on %s." % (filename))
- bad = 1
- else:
- fingerprint = args[0]
- if keywords.has_key("BADARMOR"):
- reject("ascii armour of signature was corrupt in %s." % (filename))
- bad = 1
- if keywords.has_key("NODATA"):
- utils.warn("no signature found for %s." % (filename))
- return "NOSIG"
- #reject("no signature found in %s." % (filename))
- #bad = 1
-
- if bad:
- return None
-
- # Next check gpgv exited with a zero return code
- if exit_status and not keywords.has_key("NO_PUBKEY"):
- reject("gpgv failed while checking %s." % (filename))
- if status.strip():
- reject(utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
- else:
- reject(utils.prefix_multi_line_string(output, " [GPG output:] "), "")
- return None
-
- # Sanity check the good stuff we expect
- if not keywords.has_key("VALIDSIG"):
- if not keywords.has_key("NO_PUBKEY"):
- reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename))
- bad = 1
- else:
- args = keywords["VALIDSIG"]
- if len(args) < 1:
- reject("internal error while checking signature on %s." % (filename))
- bad = 1
- else:
- fingerprint = args[0]
- if not keywords.has_key("GOODSIG") and not keywords.has_key("NO_PUBKEY"):
- reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename))
- bad = 1
- if not keywords.has_key("SIG_ID") and not keywords.has_key("NO_PUBKEY"):
- reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename))
- bad = 1
-
- # Finally ensure there's not something we don't recognise
- known_keywords = utils.Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
- SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
- NODATA="")
-
- for keyword in keywords.keys():
- if not known_keywords.has_key(keyword):
- reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename))
- bad = 1
-
- if bad:
- return None
- else:
- return fingerprint
-
-################################################################################
-
-# Prepares a filename or directory (s) to be file.filename by stripping any part of the location (sub) from it.
-def poolify (s, sub):
- for i in xrange(len(sub)):
- if sub[i:] == s[0:len(sub)-i]:
- return s[len(sub)-i:]
- return s
-
-def update_archives ():
- projectB.query("DELETE FROM archive")
- for archive in Cnf.SubTree("Archive").List():
- SubSec = Cnf.SubTree("Archive::%s" % (archive))
- projectB.query("INSERT INTO archive (name, origin_server, description) VALUES ('%s', '%s', '%s')"
- % (archive, SubSec["OriginServer"], SubSec["Description"]))
-
-def update_components ():
- projectB.query("DELETE FROM component")
- for component in Cnf.SubTree("Component").List():
- SubSec = Cnf.SubTree("Component::%s" % (component))
- projectB.query("INSERT INTO component (name, description, meets_dfsg) VALUES ('%s', '%s', '%s')" %
- (component, SubSec["Description"], SubSec["MeetsDFSG"]))
-
-def update_locations ():
- projectB.query("DELETE FROM location")
- for location in Cnf.SubTree("Location").List():
- SubSec = Cnf.SubTree("Location::%s" % (location))
- archive_id = database.get_archive_id(SubSec["archive"])
- type = SubSec.Find("type")
- for component in Cnf.SubTree("Component").List():
- component_id = database.get_component_id(component)
- projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" %
- (location, component_id, archive_id, SubSec["type"]))
-
-def update_architectures ():
- projectB.query("DELETE FROM architecture")
- for arch in Cnf.SubTree("Architectures").List():
- projectB.query("INSERT INTO architecture (arch_string, description) VALUES ('%s', '%s')" % (arch, Cnf["Architectures::%s" % (arch)]))
-
-def update_suites ():
- projectB.query("DELETE FROM suite")
- for suite in Cnf.SubTree("Suite").List():
- SubSec = Cnf.SubTree("Suite::%s" %(suite))
- projectB.query("INSERT INTO suite (suite_name) VALUES ('%s')" % suite.lower())
- for i in ("Version", "Origin", "Description"):
- if SubSec.has_key(i):
- projectB.query("UPDATE suite SET %s = '%s' WHERE suite_name = '%s'" % (i.lower(), SubSec[i], suite.lower()))
- for architecture in database.get_suite_architectures(suite):
- architecture_id = database.get_architecture_id (architecture)
- projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id))
-
-def update_override_type():
- projectB.query("DELETE FROM override_type")
- for type in Cnf.ValueList("OverrideType"):
- projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type))
-
-def update_priority():
- projectB.query("DELETE FROM priority")
- for priority in Cnf.SubTree("Priority").List():
- projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)]))
-
-def update_section():
- projectB.query("DELETE FROM section")
- for component in Cnf.SubTree("Component").List():
- if Cnf["Control-Overrides::ComponentPosition"] == "prefix":
- suffix = ""
- if component != 'main':
- prefix = component + '/'
- else:
- prefix = ""
- else:
- prefix = ""
- if component != 'main':
- suffix = '/' + component
- else:
- suffix = ""
- for section in Cnf.ValueList("Section"):
- projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix))
-
-def get_location_path(directory):
- global location_path_cache
-
- if location_path_cache.has_key(directory):
- return location_path_cache[directory]
-
- q = projectB.query("SELECT DISTINCT path FROM location WHERE path ~ '%s'" % (directory))
- try:
- path = q.getresult()[0][0]
- except:
- utils.fubar("[import-archive] get_location_path(): Couldn't get path for %s" % (directory))
- location_path_cache[directory] = path
- return path
-
-################################################################################
-
-def get_or_set_files_id (filename, size, md5sum, location_id):
- global files_id_cache, files_id_serial, files_query_cache
-
- cache_key = "_".join((filename, size, md5sum, repr(location_id)))
- if not files_id_cache.has_key(cache_key):
- files_id_serial += 1
- files_query_cache.write("%d\t%s\t%s\t%s\t%d\t\\N\n" % (files_id_serial, filename, size, md5sum, location_id))
- files_id_cache[cache_key] = files_id_serial
-
- return files_id_cache[cache_key]
-
-###############################################################################
-
-def process_sources (filename, suite, component, archive):
- global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message
-
- suite = suite.lower()
- suite_id = database.get_suite_id(suite)
- try:
- file = utils.open_file (filename)
- except CantOpenError:
- utils.warn("can't open '%s'" % (filename))
- return
- Scanner = apt_pkg.ParseTagFile(file)
- while Scanner.Step() != 0:
- package = Scanner.Section["package"]
- version = Scanner.Section["version"]
- directory = Scanner.Section["directory"]
- dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, re_no_epoch.sub('', version)))
- # Sometimes the Directory path is a lie; check in the pool
- if not os.path.exists(dsc_file):
- if directory.split('/')[0] == "dists":
- directory = Cnf["Dir::PoolRoot"] + utils.poolify(package, component)
- dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, re_no_epoch.sub('', version)))
- if not os.path.exists(dsc_file):
- utils.fubar("%s not found." % (dsc_file))
- install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file)))
- fingerprint = check_signature(dsc_file)
- fingerprint_id = database.get_or_set_fingerprint_id(fingerprint)
- if reject_message:
- utils.fubar("%s: %s" % (dsc_file, reject_message))
- maintainer = Scanner.Section["maintainer"]
- maintainer = maintainer.replace("'", "\\'")
- maintainer_id = database.get_or_set_maintainer_id(maintainer)
- location = get_location_path(directory.split('/')[0])
- location_id = database.get_location_id (location, component, archive)
- if not directory.endswith("/"):
- directory += '/'
- directory = poolify (directory, location)
- if directory != "" and not directory.endswith("/"):
- directory += '/'
- no_epoch_version = re_no_epoch.sub('', version)
- # Add all files referenced by the .dsc to the files table
- ids = []
- for line in Scanner.Section["files"].split('\n'):
- id = None
- (md5sum, size, filename) = line.strip().split()
- # Don't duplicate .orig.tar.gz's
- if filename.endswith(".orig.tar.gz"):
- cache_key = "%s_%s_%s" % (filename, size, md5sum)
- if orig_tar_gz_cache.has_key(cache_key):
- id = orig_tar_gz_cache[cache_key]
- else:
- id = get_or_set_files_id (directory + filename, size, md5sum, location_id)
- orig_tar_gz_cache[cache_key] = id
- else:
- id = get_or_set_files_id (directory + filename, size, md5sum, location_id)
- ids.append(id)
- # If this is the .dsc itself; save the ID for later.
- if filename.endswith(".dsc"):
- files_id = id
- filename = directory + package + '_' + no_epoch_version + '.dsc'
- cache_key = "%s_%s" % (package, version)
- if not source_cache.has_key(cache_key):
- nasty_key = "%s_%s" % (package, version)
- source_id_serial += 1
- if not source_cache_for_binaries.has_key(nasty_key):
- source_cache_for_binaries[nasty_key] = source_id_serial
- tmp_source_id = source_id_serial
- source_cache[cache_key] = source_id_serial
- source_query_cache.write("%d\t%s\t%s\t%d\t%d\t%s\t%s\n" % (source_id_serial, package, version, maintainer_id, files_id, install_date, fingerprint_id))
- for id in ids:
- dsc_files_id_serial += 1
- dsc_files_query_cache.write("%d\t%d\t%d\n" % (dsc_files_id_serial, tmp_source_id,id))
- else:
- tmp_source_id = source_cache[cache_key]
-
- src_associations_id_serial += 1
- src_associations_query_cache.write("%d\t%d\t%d\n" % (src_associations_id_serial, suite_id, tmp_source_id))
-
- file.close()
-
-###############################################################################
-
-def process_packages (filename, suite, component, archive):
- global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message
-
- count_total = 0
- count_bad = 0
- suite = suite.lower()
- suite_id = database.get_suite_id(suite)
- try:
- file = utils.open_file (filename)
- except CantOpenError:
- utils.warn("can't open '%s'" % (filename))
- return
- Scanner = apt_pkg.ParseTagFile(file)
- while Scanner.Step() != 0:
- package = Scanner.Section["package"]
- version = Scanner.Section["version"]
- maintainer = Scanner.Section["maintainer"]
- maintainer = maintainer.replace("'", "\\'")
- maintainer_id = database.get_or_set_maintainer_id(maintainer)
- architecture = Scanner.Section["architecture"]
- architecture_id = database.get_architecture_id (architecture)
- fingerprint = "NOSIG"
- fingerprint_id = database.get_or_set_fingerprint_id(fingerprint)
- if not Scanner.Section.has_key("source"):
- source = package
- else:
- source = Scanner.Section["source"]
- source_version = ""
- if source.find("(") != -1:
- m = re_extract_src_version.match(source)
- source = m.group(1)
- source_version = m.group(2)
- if not source_version:
- source_version = version
- filename = Scanner.Section["filename"]
- if filename.endswith(".deb"):
- type = "deb"
- else:
- type = "udeb"
- location = get_location_path(filename.split('/')[0])
- location_id = database.get_location_id (location, component.replace("/debian-installer", ""), archive)
- filename = poolify (filename, location)
- if architecture == "all":
- filename = re_arch_from_filename.sub("binary-all", filename)
- cache_key = "%s_%s" % (source, source_version)
- source_id = source_cache_for_binaries.get(cache_key, None)
- size = Scanner.Section["size"]
- md5sum = Scanner.Section["md5sum"]
- files_id = get_or_set_files_id (filename, size, md5sum, location_id)
- cache_key = "%s_%s_%s_%d_%d_%d_%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id)
- if not arch_all_cache.has_key(cache_key):
- arch_all_cache[cache_key] = 1
- cache_key = "%s_%s_%s_%d" % (package, version, repr(source_id), architecture_id)
- if not binary_cache.has_key(cache_key):
- if not source_id:
- source_id = "\N"
- count_bad += 1
- else:
- source_id = repr(source_id)
- binaries_id_serial += 1
- binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type, fingerprint_id))
- binary_cache[cache_key] = binaries_id_serial
- tmp_binaries_id = binaries_id_serial
- else:
- tmp_binaries_id = binary_cache[cache_key]
-
- bin_associations_id_serial += 1
- bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, suite_id, tmp_binaries_id))
- count_total += 1
-
- file.close()
- if count_bad != 0:
- print "%d binary packages processed; %d with no source match which is %.2f%%" % (count_total, count_bad, (float(count_bad)/count_total)*100)
- else:
- print "%d binary packages processed; 0 with no source match which is 0%%" % (count_total)
-
-###############################################################################
-
-def do_sources(sources, suite, component, server):
- (fd, temp_filename) = utils.temp_filename()
- (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename))
- if (result != 0):
- utils.fubar("Gunzip invocation failed!\n%s" % (output), result)
- print 'Processing '+sources+'...'
- process_sources (temp_filename, suite, component, server)
- os.unlink(temp_filename)
-
-###############################################################################
-
-def do_da_do_da ():
- global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache
-
- Cnf = utils.get_conf()
- Arguments = [('a', "action", "Import-Archive::Options::Action"),
- ('h', "help", "Import-Archive::Options::Help")]
- for i in [ "action", "help" ]:
- if not Cnf.has_key("Import-Archive::Options::%s" % (i)):
- Cnf["Import-Archive::Options::%s" % (i)] = ""
-
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
-
- Options = Cnf.SubTree("Import-Archive::Options")
- if Options["Help"]:
- usage()
-
- if not Options["Action"]:
- utils.warn("""no -a/--action given; not doing anything.
-Please read the documentation before running this script.
-""")
- usage(1)
-
- print "Re-Creating DB..."
- (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1")
- if (result != 0):
- utils.fubar("psql invocation failed!\n", result)
- print output
-
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
-
- database.init (Cnf, projectB)
-
- print "Adding static tables from conf file..."
- projectB.query("BEGIN WORK")
- update_architectures()
- update_components()
- update_archives()
- update_locations()
- update_suites()
- update_override_type()
- update_priority()
- update_section()
- projectB.query("COMMIT WORK")
-
- files_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"files","w")
- source_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"source","w")
- src_associations_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"src_associations","w")
- dsc_files_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"dsc_files","w")
- binaries_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"binaries","w")
- bin_associations_query_cache = utils.open_file(Cnf["Import-Archive::ExportDir"]+"bin_associations","w")
-
- projectB.query("BEGIN WORK")
- # Process Sources files to popoulate `source' and friends
- for location in Cnf.SubTree("Location").List():
- SubSec = Cnf.SubTree("Location::%s" % (location))
- server = SubSec["Archive"]
- type = Cnf.Find("Location::%s::Type" % (location))
- if type == "pool":
- for suite in Cnf.ValueList("Location::%s::Suites" % (location)):
- for component in Cnf.SubTree("Component").List():
- sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz'
- do_sources(sources, suite, component, server)
- else:
- utils.fubar("Unknown location type ('%s')." % (type))
-
- # Process Packages files to populate `binaries' and friends
-
- for location in Cnf.SubTree("Location").List():
- SubSec = Cnf.SubTree("Location::%s" % (location))
- server = SubSec["Archive"]
- type = Cnf.Find("Location::%s::Type" % (location))
- if type == "pool":
- for suite in Cnf.ValueList("Location::%s::Suites" % (location)):
- udeb_components = map(lambda x: x+"/debian-installer",
- Cnf.ValueList("Suite::%s::UdebComponents" % suite))
- for component in Cnf.SubTree("Component").List() + udeb_components:
- architectures = filter(utils.real_arch, database.get_suite_architectures(suite))
- for architecture in architectures:
- packages = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages'
- print 'Processing '+packages+'...'
- process_packages (packages, suite, component, server)
-
- files_query_cache.close()
- source_query_cache.close()
- src_associations_query_cache.close()
- dsc_files_query_cache.close()
- binaries_query_cache.close()
- bin_associations_query_cache.close()
- print "Writing data to `files' table..."
- projectB.query("COPY files FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"files"))
- print "Writing data to `source' table..."
- projectB.query("COPY source FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"source"))
- print "Writing data to `src_associations' table..."
- projectB.query("COPY src_associations FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"src_associations"))
- print "Writing data to `dsc_files' table..."
- projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"dsc_files"))
- print "Writing data to `binaries' table..."
- projectB.query("COPY binaries FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"binaries"))
- print "Writing data to `bin_associations' table..."
- projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Import-Archive::ExportDir"]+"bin_associations"))
- print "Committing..."
- projectB.query("COMMIT WORK")
-
- # Add the constraints and otherwise generally clean up the database.
- # See add_constraints.sql for more details...
-
- print "Running add_constraints.sql..."
- (result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"]))
- print output
- if (result != 0):
- utils.fubar("psql invocation failed!\n%s" % (output), result)
-
- return
-
-################################################################################
-
-def main():
- utils.try_with_debug(do_da_do_da)
-
-################################################################################
-
-if __name__ == '__main__':
- main()
################################################################################
-from daklib import database
-from daklib import utils
import sys, os, re
-import apt_pkg, pg, ldap, email.Utils
+import apt_pkg, ldap, email.Utils
+
+from daklib.config import Config
+from daklib.dbconn import *
+
# Globals
-Cnf = None
Options = None
-projectB = None
################################################################################
-def get_uid_info():
+def get_uid_info(session):
byname = {}
byid = {}
- q = projectB.query("SELECT id, uid, name FROM uid")
- for (keyid, uid, name) in q.getresult():
+ q = session.execute("SELECT id, uid, name FROM uid")
+ for (keyid, uid, name) in q.fetchall():
byname[uid] = (keyid, name)
byid[keyid] = (uid, name)
return (byname, byid)
-def get_fingerprint_info():
+def get_fingerprint_info(session):
fins = {}
- q = projectB.query("SELECT f.fingerprint, f.id, f.uid, f.keyring FROM fingerprint f")
- for (fingerprint, fingerprint_id, uid, keyring) in q.getresult():
+ q = session.execute("SELECT f.fingerprint, f.id, f.uid, f.keyring FROM fingerprint f")
+ for (fingerprint, fingerprint_id, uid, keyring) in q.fetchall():
fins[fingerprint] = (uid, fingerprint_id, keyring)
return fins
################################################################################
-class Keyring:
+class Keyring(object):
gpg_invocation = "gpg --no-default-keyring --keyring %s" +\
" --with-colons --fingerprint --fingerprint"
keys = {}
return "".join(esclist)
def __init__(self, keyring):
+ self.cnf = Config()
k = os.popen(self.gpg_invocation % keyring, "r")
keys = self.keys
key = None
return ({}, {})
def import_users_from_ldap(self):
- LDAPDn = Cnf["Import-LDAP-Fingerprints::LDAPDn"]
- LDAPServer = Cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ LDAPDn = self.cnf["Import-LDAP-Fingerprints::LDAPDn"]
+ LDAPServer = self.cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
- "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Import-Users-From-Passwd::ValidGID"]),
+ "(&(keyfingerprint=*)(gidnumber=%s))" % (self.cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "keyfingerprint", "cn", "mn", "sn"])
ldap_fin_uid_id = {}
keys[key]["uid"] = uid
if keyid != None: continue
- keyid = database.get_or_set_uid_id(uid)
+ keyid = get_or_set_uid(uid).uid
byuid[keyid] = (uid, name)
byname[uid] = (keyid, name)
keys[x]["uid"] = format % "invalid-uid"
else:
uid = format % keys[x]["email"]
- keyid = database.get_or_set_uid_id(uid)
+ keyid = get_or_set_uid(uid).uid
byuid[keyid] = (uid, keys[x]["name"])
byname[uid] = (keyid, keys[x]["name"])
keys[x]["uid"] = uid
if any_invalid:
uid = format % "invalid-uid"
- keyid = database.get_or_set_uid_id(uid)
+ keyid = get_or_set_uid(uid).uid
byuid[keyid] = (uid, "ungeneratable user id")
byname[uid] = (keyid, "ungeneratable user id")
return (byname, byuid)
################################################################################
def main():
- global Cnf, projectB, Options
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Import-Keyring::Options::Help"),
('L',"import-ldap-users","Import-Keyring::Options::Import-Ldap-Users"),
('U',"generate-users","Import-Keyring::Options::Generate-Users", "HasArg"),
]
for i in [ "help", "report-changes", "generate-users", "import-ldap-users" ]:
- if not Cnf.has_key("Import-Keyring::Options::%s" % (i)):
- Cnf["Import-Keyring::Options::%s" % (i)] = ""
+ if not cnf.has_key("Import-Keyring::Options::%s" % (i)):
+ cnf["Import-Keyring::Options::%s" % (i)] = ""
- keyring_names = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ keyring_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
### Parse options
- Options = Cnf.SubTree("Import-Keyring::Options")
+ Options = cnf.SubTree("Import-Keyring::Options")
if Options["Help"]:
usage()
changes = [] # (uid, changes strings)
### Initialise
-
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
- projectB.query("BEGIN WORK")
+ session = DBConn().session()
### Cache all the existing fingerprint entries
-
- db_fin_info = get_fingerprint_info()
+ db_fin_info = get_fingerprint_info(session)
### Parse the keyring
keyring = Keyring(keyringname)
is_dm = "false"
- if Cnf.has_key("Import-Keyring::"+keyringname+"::Debian-Maintainer"):
- projectB.query("UPDATE keyrings SET debian_maintainer = '%s' WHERE name = '%s'" % (Cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"], keyringname.split("/")[-1]))
- is_dm = Cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"]
+ if cnf.has_key("Import-Keyring::"+keyringname+"::Debian-Maintainer"):
+ session.execute("UPDATE keyrings SET debian_maintainer = :dm WHERE name = :name",
+ {'dm': cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"],
+ 'name': keyringname.split("/")[-1]})
+
+ is_dm = cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"]
- keyring_id = database.get_or_set_keyring_id(
- keyringname.split("/")[-1])
+ keyring_id = get_or_set_keyring(
+ keyringname.split("/")[-1], session,
+ ).keyring_id
### Generate new uid entries if they're needed (from LDAP or the keyring)
(desuid_byname, desuid_byid) = keyring.generate_desired_users()
### Cache all the existing uid entries
- (db_uid_byname, db_uid_byid) = get_uid_info()
+ (db_uid_byname, db_uid_byid) = get_uid_info(session)
### Update full names of applicable users
for keyid in desuid_byid.keys():
uid = (keyid, desuid_byid[keyid][0])
name = desuid_byid[keyid][1]
- oname = db_uid_byid[keyid][1]
+ oname = db_uid_byname[keyid][1]
if name and oname != name:
changes.append((uid[1], "Full name: %s" % (name)))
- projectB.query("UPDATE uid SET name = '%s' WHERE id = %s" %
- (pg.escape_string(name), keyid))
+ session.execute("UPDATE uid SET name = :name WHERE id = :keyid",
+ {'name': name, 'keyid': keyid})
# The fingerprint table (fpr) points to a uid and a keyring.
# If the uid is being decided here (ldap/generate) we set it to it.
if kr != keyring_id: continue
if f in fpr: continue
changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s" % (f)))
- projectB.query("UPDATE fingerprint SET keyring = NULL WHERE id = %d" % (fid))
+ session.execute("UPDATE fingerprint SET keyring = NULL WHERE id = :fprid", {'fprid': fid})
# For the keys in this keyring, add/update any fingerprints that've
# changed.
if oldfid == -1:
changes.append((newuiduid, "Added key: %s" % (f)))
if newuid:
- projectB.query("INSERT INTO fingerprint (fingerprint, uid, keyring) VALUES ('%s', %d, %d)" % (f, newuid, keyring_id))
+ session.execute("""INSERT INTO fingerprint (fingerprint, uid, keyring)
+ VALUES (:fpr, :uid, :keyring)""",
+ {'fpr': f, 'uid': uid, 'keyring': keyring_id})
else:
- projectB.query("INSERT INTO fingerprint (fingerprint, keyring) VALUES ('%s', %d)" % (f, keyring_id))
+ session.execute("""INSERT INTO fingerprint (fingerprint, keyring)
+ VALUES (:fpr, :keyring)""",
+ {'fpr': f, 'keyring': keyring_id})
else:
if newuid and olduid != newuid:
if olduid != -1:
else:
changes.append((newuiduid, "Linked key: %s" % f))
changes.append((newuiduid, " (formerly unowned)"))
- projectB.query("UPDATE fingerprint SET uid = %d WHERE id = %d" % (newuid, oldfid))
+ session.execute("UPDATE fingerprint SET uid = :uid WHERE id = :fpr",
+ {'uid': newuid, 'fpr': oldfid})
if oldkid != keyring_id:
# Only change the keyring if it won't result in a loss of permissions
- q = projectB.query("SELECT debian_maintainer FROM keyrings WHERE id = '%d'" % (keyring_id))
- if is_dm == "false" and q.getresult()[0][0] == 'f':
- projectB.query("UPDATE fingerprint SET keyring = %d WHERE id = %d" % (keyring_id, oldfid))
+ q = session.execute("SELECT debian_maintainer FROM keyrings WHERE id = :keyring",
+ {'keyring': keyring_id})
+ if is_dm == "false" and not q.fetchall()[0][0]:
+ session.execute("UPDATE fingerprint SET keyring = :keyring WHERE id = :fpr",
+ {'keyring': keyring_id, 'fpr': oldfid})
else:
print "Key %s exists in both DM and DD keyrings. Not demoting." % (f)
# All done!
-
- projectB.query("COMMIT WORK")
+ session.commit()
changesd = {}
for (k, v) in changes:
################################################################################
-import commands, ldap, pg, re, sys
+import commands, ldap, sys
import apt_pkg
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_gpg_fingerprint, re_debian_address
################################################################################
-Cnf = None
-projectB = None
-
-################################################################################
-
def usage(exit_code=0):
print """Usage: dak import-ldap-fingerprints
Syncs fingerprint and uid tables with a debian.org LDAP DB
name += get_ldap_value(entry, "sn")
return name.rstrip()
-def escape_string(str):
- return str.replace("'", "\\'")
-
def main():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Import-LDAP-Fingerprints::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Import-LDAP-Fingerprints::Options::%s" % (i)):
- Cnf["Import-LDAP-Fingerprints::Options::%s" % (i)] = ""
+ if not cnf.has_key("Import-LDAP-Fingerprints::Options::%s" % (i)):
+ cnf["Import-LDAP-Fingerprints::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Import-LDAP-Fingerprints::Options")
+ Options = cnf.SubTree("Import-LDAP-Fingerprints::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
- LDAPDn = Cnf["Import-LDAP-Fingerprints::LDAPDn"]
- LDAPServer = Cnf["Import-LDAP-Fingerprints::LDAPServer"]
+ LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
+ LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
- "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Import-Users-From-Passwd::ValidGID"]),
+ "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "keyfingerprint", "cn", "mn", "sn"])
- projectB.query("BEGIN WORK")
-
+ # Our database session is already in a transaction
# Sync LDAP with DB
db_fin_uid = {}
db_uid_name = {}
ldap_fin_uid_id = {}
- q = projectB.query("""
+ q = session.execute("""
SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
UNION SELECT f.fingerprint, f.id, null FROM fingerprint f where f.uid is null""")
- for i in q.getresult():
+ for i in q.fetchall():
(fingerprint, fingerprint_id, uid) = i
db_fin_uid[fingerprint] = (uid, fingerprint_id)
- q = projectB.query("SELECT id, name FROM uid")
- for i in q.getresult():
+ q = session.execute("SELECT id, name FROM uid")
+ for i in q.fetchall():
(uid, name) = i
db_uid_name[uid] = name
for i in Attrs:
entry = i[1]
fingerprints = entry["keyFingerPrint"]
- uid = entry["uid"][0]
+ uid_name = entry["uid"][0]
name = get_ldap_name(entry)
- uid_id = database.get_or_set_uid_id(uid)
+ uid = get_or_set_uid(uid_name, session)
+ uid_id = uid.uid_id
if not db_uid_name.has_key(uid_id) or db_uid_name[uid_id] != name:
- q = projectB.query("UPDATE uid SET name = '%s' WHERE id = %d" % (escape_string(name), uid_id))
- print "Assigning name of %s as %s" % (uid, name)
+ session.execute("UPDATE uid SET name = :name WHERE id = :uidid", {'name': name, 'uidid': uid_id})
+ print "Assigning name of %s as %s" % (uid_name, name)
for fingerprint in fingerprints:
- ldap_fin_uid_id[fingerprint] = (uid, uid_id)
+ ldap_fin_uid_id[fingerprint] = (uid_name, uid_id)
if db_fin_uid.has_key(fingerprint):
(existing_uid, fingerprint_id) = db_fin_uid[fingerprint]
if not existing_uid:
- q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
- print "Assigning %s to 0x%s." % (uid, fingerprint)
- elif existing_uid == uid:
+ session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
+ {'uidid': uid_id, 'fprid': fingerprint_id})
+ print "Assigning %s to 0x%s." % (uid_name, fingerprint)
+ elif existing_uid == uid_name:
pass
elif '@' not in existing_uid:
- q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
- print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid, fingerprint)
+ session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
+ {'uidid': uid_id, 'fprid': fingerprint_id})
+ print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid_name, fingerprint)
else:
- utils.warn("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid))
+ utils.warn("%s has %s in LDAP, but database says it should be %s." % \
+ (uid_name, fingerprint, existing_uid))
# Try to update people who sign with non-primary key
- q = projectB.query("SELECT fingerprint, id FROM fingerprint WHERE uid is null")
- for i in q.getresult():
+ q = session.execute("SELECT fingerprint, id FROM fingerprint WHERE uid is null")
+ for i in q.fetchall():
(fingerprint, fingerprint_id) = i
cmd = "gpg --no-default-keyring %s --fingerprint %s" \
% (utils.gpg_keyring_args(), fingerprint)
m = re_gpg_fingerprint.search(output)
if not m:
print output
- utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % (fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] ")))
+ utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % \
+ (fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ","")
if not ldap_fin_uid_id.has_key(primary_key):
utils.warn("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint))
else:
(uid, uid_id) = ldap_fin_uid_id[primary_key]
- q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
+ session.execute("UPDATE fingerprint SET uid = :uid WHERE id = :fprid",
+ {'uid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid, fingerprint)
else:
extra_keyrings = ""
- for keyring in Cnf.ValueList("Import-LDAP-Fingerprints::ExtraKeyrings"):
+ for keyring in cnf.ValueList("Import-LDAP-Fingerprints::ExtraKeyrings"):
extra_keyrings += " --keyring=%s" % (keyring)
cmd = "gpg %s %s --list-key %s" \
% (utils.gpg_keyring_args(), extra_keyrings, fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
- cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (Cnf["Import-LDAP-Fingerprints::KeyServer"], fingerprint)
+ cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (cnf["Import-LDAP-Fingerprints::KeyServer"], fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
print "0x%s: NOT found on keyserver." % (fingerprint)
prompt = "Map to %s - %s (y/N) ? " % (uid, name.replace(" "," "))
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
- uid_id = database.get_or_set_uid_id(uid)
- projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
+ uid_o = get_or_set_uid(uid, session=session)
+ uid_id = uid_o.uid_id
+ session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
+ {'uidid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid, fingerprint)
else:
uid = None
- projectB.query("COMMIT WORK")
+
+ # Commit it all
+ session.commit()
############################################################
################################################################################
-import pg, pwd, sys
+import pwd
+import sys
+import re
import apt_pkg
-from daklib import utils
-################################################################################
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import utils
-Cnf = None
-projectB = None
################################################################################
def usage (exit_code=0):
################################################################################
def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('n', "no-action", "Import-Users-From-Passwd::Options::No-Action"),
('q', "quiet", "Import-Users-From-Passwd::Options::Quiet"),
('v', "verbose", "Import-Users-From-Passwd::Options::Verbose"),
('h', "help", "Import-Users-From-Passwd::Options::Help")]
for i in [ "no-action", "quiet", "verbose", "help" ]:
- if not Cnf.has_key("Import-Users-From-Passwd::Options::%s" % (i)):
- Cnf["Import-Users-From-Passwd::Options::%s" % (i)] = ""
+ if not cnf.has_key("Import-Users-From-Passwd::Options::%s" % (i)):
+ cnf["Import-Users-From-Passwd::Options::%s" % (i)] = ""
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Import-Users-From-Passwd::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Import-Users-From-Passwd::Options")
if Options["Help"]:
usage()
utils.warn("dak import-users-from-passwd takes no non-option arguments.")
usage(1)
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- valid_gid = int(Cnf.get("Import-Users-From-Passwd::ValidGID",""))
+ session = DBConn().session()
+ valid_gid = int(cnf.get("Import-Users-From-Passwd::ValidGID",""))
passwd_unames = {}
for entry in pwd.getpwall():
passwd_unames[uname] = ""
postgres_unames = {}
- q = projectB.query("SELECT usename FROM pg_user")
- ql = q.getresult()
- for i in ql:
+ q = session.execute("SELECT usename FROM pg_user")
+ for i in q.fetchall():
uname = i[0]
postgres_unames[uname] = ""
known_postgres_unames = {}
- for i in Cnf.get("Import-Users-From-Passwd::KnownPostgres","").split(","):
+ for i in cnf.get("Import-Users-From-Passwd::KnownPostgres","").split(","):
uname = i.strip()
known_postgres_unames[uname] = ""
keys = passwd_unames.keys()
keys.sort()
+ safe_name = re.compile('^[A-Za-z0-9]+$')
for uname in keys:
if not postgres_unames.has_key(uname):
if not Options["Quiet"]:
print "Creating %s user in Postgres." % (uname)
if not Options["No-Action"]:
- q = projectB.query('CREATE USER "%s"' % (uname))
+ if safe_name.match(uname):
+ # NB: I never figured out how to use a bind parameter for this query
+ # XXX: Fix this as it looks like a potential SQL injection attack to me
+ # (hence the safe_name match we do)
+ q = session.execute('CREATE USER "%s"' % (uname))
+ else:
+ print "NOT CREATING USER %s. Doesn't match safety regex" % uname
+
+ session.commit()
#######################################################################################
################################################################################
-import psycopg2, sys
+import sys
import apt_pkg
from daklib import utils
-from daklib.dbconn import DBConn
+from daklib.dbconn import *
from daklib.config import Config
################################################################################
################################################################################
-def sql_get (config, key):
- """Return the value of config[key] or None if it doesn't exist."""
-
- try:
- return config[key]
- except KeyError:
- return None
-
-################################################################################
-
class InitDB(object):
def __init__(self, Cnf, projectB):
self.Cnf = Cnf
def do_archive(self):
"""initalize the archive table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM archive")
- archive_add = "INSERT INTO archive (name, origin_server, description) VALUES (%s, %s, %s)"
+ # Remove existing archives
+ s = self.projectB.session()
+ s.query(Archive).delete()
+
for name in self.Cnf.SubTree("Archive").List():
- archive_config = self.Cnf.SubTree("Archive::%s" % (name))
- origin_server = sql_get(archive_config, "OriginServer")
- description = sql_get(archive_config, "Description")
- c.execute(archive_add, [name, origin_server, description])
- self.projectB.commit()
+ a = Archive()
+ a.archive_name = name
+ a.origin_server = self.Cnf.get("Archive::%s::OriginServer" % name, "")
+ a.description = self.Cnf.get("Archive::%s::Description" % name, "")
+ s.add(a)
+
+ s.commit()
def do_architecture(self):
"""Initalize the architecture table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM architecture")
- arch_add = "INSERT INTO architecture (arch_string, description) VALUES (%s, %s)"
+ # Remove existing architectures
+ s = self.projectB.session()
+ s.query(Architecture).delete()
+
for arch in self.Cnf.SubTree("Architectures").List():
- description = self.Cnf["Architectures::%s" % (arch)]
- c.execute(arch_add, [arch, description])
- self.projectB.commit()
+ a = Architecture()
+ a.arch_string = arch
+ a.description = self.Cnf.get("Architecture::%s" % arch, "")
+ s.add(a)
+
+ s.commit()
def do_component(self):
"""Initalize the component table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM component")
-
- comp_add = "INSERT INTO component (name, description, meets_dfsg) " + \
- "VALUES (%s, %s, %s)"
+ # Remove existing components
+ s = self.projectB.session()
+ s.query(Component).delete()
for name in self.Cnf.SubTree("Component").List():
- component_config = self.Cnf.SubTree("Component::%s" % (name))
- description = sql_get(component_config, "Description")
- meets_dfsg = (component_config.get("MeetsDFSG").lower() == "true")
- c.execute(comp_add, [name, description, meets_dfsg])
+ c = Component()
+ c.component_name = name
+ c.description = self.Cnf.get("Component::%s::Description" % name, "")
+ c.meets_dfsg = False
+ if self.Cnf.get("Component::%s::MeetsDFSG" % name, "false").lower() == 'true':
+ c.meets_dfsg = True
+ s.add(c)
- self.projectB.commit()
+ s.commit()
def do_location(self):
"""Initalize the location table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM location")
-
- loc_add = "INSERT INTO location (path, component, archive, type) " + \
- "VALUES (%s, %s, %s, %s)"
+ # Remove existing locations
+ s = self.projectB.session()
+ s.query(Location).delete()
for location in self.Cnf.SubTree("Location").List():
- location_config = self.Cnf.SubTree("Location::%s" % (location))
- archive_id = self.projectB.get_archive_id(location_config["Archive"])
- if archive_id == -1:
- utils.fubar("Archive '%s' for location '%s' not found."
- % (location_config["Archive"], location))
- location_type = location_config.get("type")
- if location_type == "pool":
- for component in self.Cnf.SubTree("Component").List():
- component_id = self.projectB.get_component_id(component)
- c.execute(loc_add, [location, component_id, archive_id, location_type])
- else:
- utils.fubar("E: type '%s' not recognised in location %s."
- % (location_type, location))
-
- self.projectB.commit()
+ archive_name = self.Cnf.get("Location::%s::Archive" % location, "")
+ a = s.query(Archive).filter_by(archive_name=archive_name)
+ if a.count() < 1:
+ utils.fubar("E: Archive '%s' for location '%s' not found" % (archive_name, location))
+ archive_id = a.one().archive_id
+
+ location_type = self.Cnf.get("Location::%s::Type" % location, "")
+ if location_type != 'pool':
+ utils.fubar("E: type %s not recognised for location %s" % (location_type, location))
+
+ for component in self.Cnf.SubTree("Component").List():
+ c = s.query(Component).filter_by(component_name=component)
+ if c.count() < 1:
+ utils.fubar("E: Can't find component %s for location %s" % (component, location))
+ component_id = c.one().component_id
+
+ l = Location()
+ l.path = location
+ l.archive_id = archive_id
+ l.component_id = component_id
+ l.archive_type = location_type
+ s.add(l)
+
+ s.commit()
def do_suite(self):
"""Initalize the suite table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM suite")
-
- suite_add = "INSERT INTO suite (suite_name, version, origin, description) " + \
- "VALUES (%s, %s, %s, %s)"
-
- sa_add = "INSERT INTO suite_architectures (suite, architecture) " + \
- "VALUES (currval('suite_id_seq'), %s)"
+ s = self.projectB.session()
+ s.query(Suite).delete()
for suite in self.Cnf.SubTree("Suite").List():
- suite_config = self.Cnf.SubTree("Suite::%s" %(suite))
- version = sql_get(suite_config, "Version")
- origin = sql_get(suite_config, "Origin")
- description = sql_get(suite_config, "Description")
- c.execute(suite_add, [suite.lower(), version, origin, description])
- for architecture in self.Cnf.SubTree("Architectures").List():
- architecture_id = self.projectB.get_architecture_id (architecture)
- if architecture_id < 0:
- utils.fubar("architecture '%s' not found in architecture"
- " table for suite %s."
- % (architecture, suite))
- c.execute(sa_add, [architecture_id])
-
- self.projectB.commit()
+ su = Suite()
+ su.version = self.Cnf.get("Suite::%s::Version" % suite, "-")
+ su.origin = self.Cnf.get("Suite::%s::Origin" % suite, "")
+ su.description = self.Cnf.get("Suite::%s::Description" % suite, "")
+ s.add(su)
+
+ for architecture in self.Cnf.ValueList("Suite::%s::Architectures" % (suite)):
+ sa = SuiteArchitecture()
+ sa.suite_id = su.suite_id
+ a = s.query(Architecture).filter_by(arch_string=architecture)
+ if a.count() < 1:
+ utils.fubar("E: Architecture %s not found for suite %s" % (architecture, suite))
+ sa.arch_id = a.one().arch_id
+ s.add(sa)
+
+ s.commit()
def do_override_type(self):
"""Initalize the override_type table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM override_type")
-
- over_add = "INSERT INTO override_type (type) VALUES (%s)"
+ s = self.projectB.session()
+ s.query(OverrideType).delete()
for override_type in self.Cnf.ValueList("OverrideType"):
- c.execute(over_add, [override_type])
+ ot = OverrideType()
+ ot.overridetype = override_type
+ s.add(ot)
- self.projectB.commit()
+ s.commit()
def do_priority(self):
"""Initialize the priority table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM priority")
-
- prio_add = "INSERT INTO priority (priority, level) VALUES (%s, %s)"
+ s = self.projectB.session()
+ s.query(Priority).delete()
for priority in self.Cnf.SubTree("Priority").List():
- c.execute(prio_add, [priority, self.Cnf["Priority::%s" % (priority)]])
+ p = Priority()
+ p.priority = priority
+ p.level = self.Cnf.get("Priority::%s", 0)
+ s.add(p)
- self.projectB.commit()
+ s.commit()
def do_section(self):
"""Initalize the section table."""
- c = self.projectB.cursor()
- c.execute("DELETE FROM section")
-
- sect_add = "INSERT INTO section (section) VALUES (%s)"
+ s = self.projectB.session()
+ s.query(Section).delete()
for component in self.Cnf.SubTree("Component").List():
if self.Cnf["Control-Overrides::ComponentPosition"] == "prefix":
suffix = '/' + component
else:
suffix = ""
+
for section in self.Cnf.ValueList("Section"):
- c.execute(sect_add, [prefix + section + suffix])
+ sec = Section()
+ sec.section = prefix + section + suffix
+ s.add(sec)
- self.projectB.commit()
+ s.commit()
def do_all(self):
self.do_archive()
################################################################################
import os
-import pg
import sys
import apt_pkg
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import utils
################################################################################
################################################################################
def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a', "architecture", "Ls::Options::Architecture", "HasArg"),
('b', "binarytype", "Ls::Options::BinaryType", "HasArg"),
for i in [ "architecture", "binarytype", "component", "format",
"greaterorequal", "greaterthan", "regex", "suite",
"source-and-binary", "help" ]:
- if not Cnf.has_key("Ls::Options::%s" % (i)):
- Cnf["Ls::Options::%s" % (i)] = ""
+ if not cnf.has_key("Ls::Options::%s" % (i)):
+ cnf["Ls::Options::%s" % (i)] = ""
- packages = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Ls::Options")
+ packages = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Ls::Options")
if Options["Help"]:
usage()
if not packages:
utils.fubar("need at least one package name as an argument.")
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# If cron.daily is running; warn the user that our output might seem strange
- if os.path.exists(os.path.join(Cnf["Dir::Root"], "Archive_Maintenance_In_Progress")):
+ if os.path.exists(os.path.join(cnf["Dir::Root"], "Archive_Maintenance_In_Progress")):
utils.warn("Archive maintenance is in progress; database inconsistencies are possible.")
# Handle buildd maintenance helper options
if Options["Source-And-Binary"]:
new_packages = []
for package in packages:
- q = projectB.query("SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s '%s' %s" % (comparison_operator, package, con_suites))
- new_packages.extend([ i[0] for i in q.getresult() ])
+ q = session.execute("SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s :package %s" % (comparison_operator, con_suites),
+ {'package': package})
+ new_packages.extend([ i[0] for i in q.fetchall() ])
if package not in new_packages:
new_packages.append(package)
packages = new_packages
results = 0
for package in packages:
- q = projectB.query("""
+ q = session.execute("""
SELECT b.package, b.version, a.arch_string, su.suite_name, c.name, m.name
FROM binaries b, architecture a, suite su, bin_associations ba,
files f, location l, component c, maintainer m
- WHERE b.package %s '%s' AND a.id = b.architecture AND su.id = ba.suite
+ WHERE b.package %s :package AND a.id = b.architecture AND su.id = ba.suite
AND b.id = ba.bin AND b.file = f.id AND f.location = l.id
AND l.component = c.id AND b.maintainer = m.id %s %s %s
-""" % (comparison_operator, package, con_suites, con_architectures, con_bintype))
- ql = q.getresult()
+""" % (comparison_operator, con_suites, con_architectures, con_bintype), {'package': package})
+ ql = q.fetchall()
if check_source:
- q = projectB.query("""
+ q = session.execute("""
SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name
FROM source s, suite su, src_associations sa, files f, location l,
component c, maintainer m
- WHERE s.source %s '%s' AND su.id = sa.suite AND s.id = sa.source
+ WHERE s.source %s :package AND su.id = sa.suite AND s.id = sa.source
AND s.file = f.id AND f.location = l.id AND l.component = c.id
AND s.maintainer = m.id %s
-""" % (comparison_operator, package, con_suites))
- ql.extend(q.getresult())
+""" % (comparison_operator, con_suites), {'package': package})
+ ql.extend(q.fetchall())
d = {}
highver = {}
for i in ql:
################################################################################
-import pg
import sys
import apt_pkg
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
+from daklib import textutils
from daklib.regexes import re_comments
################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
maintainer_from_source_cache = {} #: caches the maintainer name <email> per source_id
packages = {} #: packages data to write out
-fixed_maintainer_cache = {} #: caches fixed ( L{daklib.utils.fix_maintainer} ) maintainer data
+fixed_maintainer_cache = {} #: caches fixed ( L{daklib.textutils.fix_maintainer} ) maintainer data
################################################################################
Fixup maintainer entry, cache the result.
@type maintainer: string
- @param maintainer: A maintainer entry as passed to L{daklib.utils.fix_maintainer}
+ @param maintainer: A maintainer entry as passed to L{daklib.textutils.fix_maintainer}
@rtype: tuple
@returns: fixed maintainer tuple
global fixed_maintainer_cache
if not fixed_maintainer_cache.has_key(maintainer):
- fixed_maintainer_cache[maintainer] = utils.fix_maintainer(maintainer)[0]
+ fixed_maintainer_cache[maintainer] = textutils.fix_maintainer(maintainer)[0]
return fixed_maintainer_cache[maintainer]
-def get_maintainer (maintainer):
+def get_maintainer(maintainer, session):
"""
Retrieves maintainer name from database, passes it through fix_maintainer and
passes on whatever that returns.
@type maintainer: int
@param maintainer: maintainer_id
"""
- return fix_maintainer(database.get_maintainer(maintainer))
+ q = session.execute("SELECT name FROM maintainer WHERE id = :id", {'id': maintainer}).fetchall()
+ return fix_maintainer(q[0][0])
-def get_maintainer_from_source (source_id):
+def get_maintainer_from_source(source_id, session):
"""
Returns maintainer name for given source_id.
global maintainer_from_source_cache
if not maintainer_from_source_cache.has_key(source_id):
- q = projectB.query("SELECT m.name FROM maintainer m, source s WHERE s.id = %s and s.maintainer = m.id" % (source_id))
- maintainer = q.getresult()[0][0]
+ q = session.execute("""SELECT m.name FROM maintainer m, source s
+ WHERE s.id = :sourceid AND s.maintainer = m.id""",
+ {'sourceid': source_id})
+ maintainer = q.fetchall()[0][0]
maintainer_from_source_cache[source_id] = fix_maintainer(maintainer)
return maintainer_from_source_cache[source_id]
################################################################################
def main():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Make-Maintainers::Options::Help")]
- if not Cnf.has_key("Make-Maintainers::Options::Help"):
- Cnf["Make-Maintainers::Options::Help"] = ""
+ if not cnf.has_key("Make-Maintainers::Options::Help"):
+ cnf["Make-Maintainers::Options::Help"] = ""
- extra_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Make-Maintainers::Options")
+ extra_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Make-Maintainers::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
- for suite in Cnf.SubTree("Suite").List():
+ for suite in cnf.SubTree("Suite").List():
suite = suite.lower()
- suite_priority = int(Cnf["Suite::%s::Priority" % (suite)])
+ suite_priority = int(cnf["Suite::%s::Priority" % (suite)])
# Source packages
- q = projectB.query("SELECT s.source, s.version, m.name FROM src_associations sa, source s, suite su, maintainer m WHERE su.suite_name = '%s' AND sa.suite = su.id AND sa.source = s.id AND m.id = s.maintainer" % (suite))
- sources = q.getresult()
- for source in sources:
+ q = session.execute("""SELECT s.source, s.version, m.name
+ FROM src_associations sa, source s, suite su, maintainer m
+ WHERE su.suite_name = :suite_name
+ AND sa.suite = su.id AND sa.source = s.id
+ AND m.id = s.maintainer""",
+ {'suite_name': suite})
+ for source in q.fetchall():
package = source[0]
version = source[1]
maintainer = fix_maintainer(source[2])
packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version }
# Binary packages
- q = projectB.query("SELECT b.package, b.source, b.maintainer, b.version FROM bin_associations ba, binaries b, suite s WHERE s.suite_name = '%s' AND ba.suite = s.id AND ba.bin = b.id" % (suite))
- binaries = q.getresult()
- for binary in binaries:
+ q = session.execute("""SELECT b.package, b.source, b.maintainer, b.version
+ FROM bin_associations ba, binaries b, suite s
+ WHERE s.suite_name = :suite_name
+ AND ba.suite = s.id AND ba.bin = b.id""",
+ {'suite_name': suite})
+ for binary in q.fetchall():
package = binary[0]
source_id = binary[1]
version = binary[3]
# Use the source maintainer first; falling back on the binary maintainer as a last resort only
if source_id:
- maintainer = get_maintainer_from_source(source_id)
+ maintainer = get_maintainer_from_source(source_id, session)
else:
- maintainer = get_maintainer(binary[2])
+ maintainer = get_maintainer(binary[2], session)
if packages.has_key(package):
if packages[package]["priority"] <= suite_priority:
if apt_pkg.VersionCompare(packages[package]["version"], version) < 0:
################################################################################
-import pg
+import os
import sys
import apt_pkg
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
-override = {} #: override data to write out
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib import utils
################################################################################
################################################################################
-def do_list(output_file, suite, component, otype):
+def do_list(output_file, suite, component, otype, session):
"""
Fetch override data for suite from the database and dump it.
@type output_file: fileobject
@param output_file: where to write the overrides to
- @type suite: string
- @param suite: The name of the suite
+ @type suite: Suite object
+ @param suite: A suite object describing the Suite
- @type component: string
+ @type component: Component object
@param component: The name of the component
- @type otype: string
- @param otype: type of override. deb/udeb/dsc
-
- """
- global override
-
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
- utils.fubar("Suite '%s' not recognised." % (suite))
+ @type otype: OverrideType object
+ @param otype: object of type of override. deb/udeb/dsc
- component_id = database.get_component_id(component)
- if component_id == -1:
- utils.fubar("Component '%s' not recognised." % (component))
+ @type session: SQLA Session
+ @param session: the database session in use
- otype_id = database.get_override_type_id(otype)
- if otype_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
-
- override.setdefault(suite, {})
- override[suite].setdefault(component, {})
- override[suite][component].setdefault(otype, {})
+ """
+ # Here's a nice example of why the object API isn't always the
+ # right answer. On my laptop, the object version of the code
+ # takes 1:45, the 'dumb' tuple-based one takes 0:16 - mhy
+
+ if otype.overridetype == "dsc":
+ #q = session.query(Override).filter_by(suite_id = suite.suite_id)
+ #q = q.filter_by(component_id = component.component_id)
+ #q = q.filter_by(overridetype_id = otype.overridetype_id)
+ #q = q.join(Section).order_by(Section.section, Override.package)
+ #for o in q.all():
+ # dat = (o.package, o.section.section, o.maintainer)
+ # output_file.write(utils.result_join(dat) + '\n')
+ q = session.execute("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite.suite_id, component.component_id, otype.overridetype_id))
+ for i in q.fetchall():
+ output_file.write(utils.result_join(i) + '\n')
- if otype == "dsc":
- q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, otype_id))
- for i in q.getresult():
- override[suite][component][otype][i[0]] = i
- output_file.write(utils.result_join(i)+'\n')
else:
- q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, otype_id))
- for i in q.getresult():
- i = i[:-1]; # Strip the priority level
- override[suite][component][otype][i[0]] = i
- output_file.write(utils.result_join(i)+'\n')
+ #q = session.query(Override).filter_by(suite_id = suite.suite_id)
+ #q = q.filter_by(component_id = component.component_id)
+ #q = q.filter_by(overridetype_id = otype.overridetype_id)
+ #q = q.join(Priority).join(Section).order_by(Section.section, Priority.level, Override.package)
+ #for o in q.all():
+ # dat = (o.package, o.priority.priority, o.section.section, o.maintainer)
+ # output_file.write(utils.result_join(dat) + '\n')
+ q = session.execute("SELECT o.package, p.priority, s.section, o.maintainer FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite.suite_id, component.component_id, otype.overridetype_id))
+ for i in q.fetchall():
+ output_file.write(utils.result_join(i) + '\n')
################################################################################
def main ():
- global Cnf, projectB, override
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Make-Overrides::Options::Help")]
for i in [ "help" ]:
- if not Cnf.has_key("Make-Overrides::Options::%s" % (i)):
- Cnf["Make-Overrides::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Make-Overrides::Options")
+ if not cnf.has_key("Make-Overrides::Options::%s" % (i)):
+ cnf["Make-Overrides::Options::%s" % (i)] = ""
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Make-Overrides::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ d = DBConn()
+ session = d.session()
- for suite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
- if database.get_suite_untouchable(suite):
+ for suite_name in cnf.SubTree("Check-Overrides::OverrideSuites").List():
+ suite = get_suite(suite_name.lower(), session)
+ if not suite:
+ utils.fubar('Suite %s not found' % suite_name)
+ if suite.untouchable:
continue
- suite = suite.lower()
- sys.stderr.write("Processing %s...\n" % (suite))
- override_suite = Cnf["Suite::%s::OverrideCodeName" % (suite)]
- for component in Cnf.SubTree("Component").List():
- for otype in Cnf.ValueList("OverrideType"):
- if otype == "deb":
+ sys.stderr.write("Processing %s...\n" % (suite.suite_name))
+ override_suite = cnf["Suite::%s::OverrideCodeName" % (suite_name)]
+
+ for component_name in cnf.SubTree("Component").List():
+ component = get_component(component_name, session)
+ if not component:
+ utils.fubar('Component %s not found' % component_name)
+
+ for otype_name in cnf.ValueList("OverrideType"):
+ otype = get_override_type(otype_name, session)
+ if not otype:
+ utils.fubar('OverrideType %s not found' % otype_name)
+
+ if otype_name == "deb":
suffix = ""
- elif otype == "udeb":
+ elif otype_name == "udeb":
if component == "contrib":
continue # Ick2
suffix = ".debian-installer"
- elif otype == "dsc":
+ elif otype_name == "dsc":
suffix = ".src"
- filename = "%s/override.%s.%s%s" % (Cnf["Dir::Override"], override_suite, component, suffix)
+
+ filename = os.path.join(cnf["Dir::Override"], "override.%s.%s%s" % (override_suite, component.component_name, suffix))
output_file = utils.open_file(filename, 'w')
- do_list(output_file, suite, component, otype)
+ do_list(output_file, suite, component, otype, session)
output_file.close()
################################################################################
################################################################################
-import os
-import pg
-import sys
-from daklib import database
-from daklib import utils
-
-################################################################################
-
-projectB = None #: database connection, pgobject
+from daklib.dbconn import *
################################################################################
ORDER BY source, version, package, bin_version
"""
- for row in projectB.query(query_sources).getresult():
+ session = DBConn().session()
+
+ for row in session.execute(query_sources).fetchall():
(source, version, path) = row
print "Path: %s"%path
print "Source: %s"%source
print "Source-Version: %s"%version
print
- for row in projectB.query(query_binaries).getresult():
+ for row in session.execute(query_binaries).fetchall():
(source, version, arch, path, bin, binv) = row
print "Path: %s"%path
print "Source: %s"%source
################################################################################
def main():
- global projectB
-
- Cnf = utils.get_conf()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ DBConn()
build_mapping()
#########################################################################################
import copy
import os
-import pg
import sys
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib import daklog
from daklib import utils
################################################################################
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
Logger = None #: Logger object
Options = None #: Parsed CommandLine arguments
#####################################################
def delete_packages(delete_versions, pkg, dominant_arch, suite,
- dominant_version, delete_table, delete_col, packages):
- suite_id = database.get_suite_id(suite)
+ dominant_version, delete_table, delete_col, packages, session):
+ suite_o = get_suite(suite.lower(), session)
+ suite_id = suite_o.suite_id
for version in delete_versions:
delete_unique_id = version[1]
if not packages.has_key(delete_unique_id):
delete_version = version[0]
delete_id = packages[delete_unique_id]["sourceid"]
delete_arch = packages[delete_unique_id]["arch"]
- if not database.get_suite_untouchable(suite) or Options["Force"]:
+ if Options["Force"] or not suite_o.untouchable:
if Options["No-Delete"]:
print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch)
else:
Logger.log(["dominated", pkg, delete_arch, delete_version, dominant_version, dominant_arch])
- projectB.query("DELETE FROM %s WHERE suite = %s AND %s = %s" % (delete_table, suite_id, delete_col, delete_id))
+ # TODO: Fix properly
+ session.execute("DELETE FROM %s WHERE suite = :suiteid AND %s = :delid" % (delete_table, delete_col), {'suiteid': suite_id, 'delid': delete_id})
del packages[delete_unique_id]
else:
if Options["No-Delete"]:
#####################################################
-def resolve_arch_all_vs_any(versions, packages):
+def resolve_arch_all_vs_any(versions, packages, session):
""" Per-suite&pkg: resolve arch-all, vs. arch-any, assumes only one arch-all """
arch_all_version = None
arch_any_versions = copy.copy(versions)
if apt_pkg.VersionCompare(highest_arch_any_version, arch_all_version) < 1:
# arch: all dominates
delete_packages(arch_any_versions, pkg, "all", suite,
- arch_all_version, delete_table, delete_col, packages)
+ arch_all_version, delete_table, delete_col, packages, session)
else:
# arch: any dominates
delete_packages(arch_all_versions, pkg, "any", suite,
highest_arch_any_version, delete_table, delete_col,
- packages)
+ packages, session)
#####################################################
-def remove_duplicate_versions(versions, packages):
+def remove_duplicate_versions(versions, packages, session):
""" Per-suite&pkg&arch: resolve duplicate versions """
# Sort versions into descending order
versions.sort(version_cmp)
delete_col = "bin"
# Remove all but the highest
delete_packages(dominated_versions, pkg, arch, suite,
- dominant_version, delete_table, delete_col, packages)
+ dominant_version, delete_table, delete_col, packages, session)
return [dominant_versions]
################################################################################
-def cleanup(packages):
+def cleanup(packages, session):
# Build up the index used by the clean up functions
d = {}
for unique_id in packages.keys():
for arch in d[suite][pkg].keys():
versions = d[suite][pkg][arch]
if len(versions) > 1:
- d[suite][pkg][arch] = remove_duplicate_versions(versions, packages)
+ d[suite][pkg][arch] = remove_duplicate_versions(versions, packages, session)
# Arch: all -> any and vice versa
for suite in d.keys():
for arch in arches.keys():
if arch != "source":
versions.extend(d[suite][pkg][arch])
- resolve_arch_all_vs_any(versions, packages)
+ resolve_arch_all_vs_any(versions, packages, session)
################################################################################
def write_filelist(suite, component, arch, type, list, packages, dislocated_files):
+ cnf = Config()
+
# Work out the filename
if arch != "source":
if type == "udeb":
arch = "debian-installer_binary-%s" % (arch)
elif type == "deb":
arch = "binary-%s" % (arch)
- filename = os.path.join(Cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch))
+ filename = os.path.join(cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch))
output = utils.open_file(filename, "w")
# Generate the final list of files
files = {}
################################################################################
-def write_filelists(packages, dislocated_files):
+def write_filelists(packages, dislocated_files, session):
# Build up the index to iterate over
+ cnf = Config()
+
d = {}
for unique_id in packages.keys():
suite = packages[unique_id]["suite"]
d[suite][component][arch][packagetype].append(unique_id)
# Flesh out the index
if not Options["Suite"]:
- suites = Cnf.SubTree("Suite").List()
+ suites = cnf.SubTree("Suite").List()
else:
suites = utils.split_args(Options["Suite"])
for suite in [ i.lower() for i in suites ]:
d.setdefault(suite, {})
if not Options["Component"]:
- components = Cnf.ValueList("Suite::%s::Components" % (suite))
+ components = cnf.ValueList("Suite::%s::Components" % (suite))
else:
components = utils.split_args(Options["Component"])
- udeb_components = Cnf.ValueList("Suite::%s::UdebComponents" % (suite))
+ udeb_components = cnf.ValueList("Suite::%s::UdebComponents" % (suite))
for component in components:
d[suite].setdefault(component, {})
if component in udeb_components:
else:
binary_types = [ "deb" ]
if not Options["Architecture"]:
- architectures = database.get_suite_architectures(suite)
+ architectures = [ a.arch_string for a in get_suite_architectures(suite, session=session) ]
else:
architectures = utils.split_args(Options["Architecture"])
for arch in [ i.lower() for i in architectures ]:
d[suite][component][arch].setdefault(packagetype, [])
# Then walk it
for suite in d.keys():
- if Cnf.has_key("Suite::%s::Components" % (suite)):
+ if cnf.has_key("Suite::%s::Components" % (suite)):
for component in d[suite].keys():
for arch in d[suite][component].keys():
if arch == "all":
filelist = d[suite][component][arch][packagetype]
# If it's a binary, we need to add in the arch: all debs too
if arch != "source":
- archall_suite = Cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
+ archall_suite = cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
if archall_suite:
filelist.extend(d[archall_suite][component]["all"][packagetype])
elif d[suite][component].has_key("all") and \
if Options["Suite"]:
suites = utils.split_args(Options["Suite"])
for suite in suites:
- archall_suite = Cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
+ archall_suite = cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite))
if archall_suite and archall_suite not in suites:
utils.warn("Adding %s as %s maps Arch: all from it." % (archall_suite, suite))
suites.append(archall_suite)
(con_suites, con_architectures, con_components, check_source) = \
utils.parse_args(Options)
+ session = DBConn().session()
+
dislocated_files = {}
+ # TODO: Fix this properly
+ query_args = {'con_suites': con_suites, 'con_architectures': con_architectures, 'con_components': con_components}
+
query = """
SELECT b.id, b.package, a.arch_string, b.version, l.path, f.filename, c.name,
f.id, su.suite_name, b.type
component c, suite su
WHERE b.id = ba.bin AND b.file = f.id AND b.architecture = a.id
AND f.location = l.id AND l.component = c.id AND ba.suite = su.id
- %s %s %s""" % (con_suites, con_architectures, con_components)
+ %(con_suites)s %(con_architectures)s %(con_components)s""" % query_args
if check_source:
query += """
UNION
su.suite_name, 'dsc'
FROM source s, src_associations sa, files f, location l, component c, suite su
WHERE s.id = sa.source AND s.file = f.id AND f.location = l.id
- AND l.component = c.id AND sa.suite = su.id %s %s""" % (con_suites, con_components)
- q = projectB.query(query)
- ql = q.getresult()
+ AND l.component = c.id AND sa.suite = su.id %(con_suites)s %(con_components)s""" % query_args
+
# Build up the main index of packages
packages = {}
unique_id = 0
- for i in ql:
+
+ q = session.execute(query)
+ for i in q.fetchall():
(sourceid, pkg, arch, version, path, filename, component, file_id, suite, filetype) = i
+
# 'id' comes from either 'binaries' or 'source', so it's not unique
unique_id += 1
packages[unique_id] = Dict(sourceid=sourceid, pkg=pkg, arch=arch, version=version,
path=path, filename=filename,
component=component, file_id=file_id,
suite=suite, filetype = filetype)
- cleanup(packages)
- write_filelists(packages, dislocated_files)
+ cleanup(packages, session)
+ session.commit()
+ write_filelists(packages, dislocated_files, session)
################################################################################
def main():
- global Cnf, projectB, Options, Logger
+ global Options, Logger
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('a', "architecture", "Make-Suite-File-List::Options::Architecture", "HasArg"),
('c', "component", "Make-Suite-File-List::Options::Component", "HasArg"),
('h', "help", "Make-Suite-File-List::Options::Help"),
('f', "force", "Make-Suite-File-List::Options::Force"),
('s', "suite", "Make-Suite-File-List::Options::Suite", "HasArg")]
for i in ["architecture", "component", "help", "no-delete", "suite", "force" ]:
- if not Cnf.has_key("Make-Suite-File-List::Options::%s" % (i)):
- Cnf["Make-Suite-File-List::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Make-Suite-File-List::Options")
+ if not cnf.has_key("Make-Suite-File-List::Options::%s" % (i)):
+ cnf["Make-Suite-File-List::Options::%s" % (i)] = ""
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Make-Suite-File-List::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
- Logger = logging.Logger(Cnf, "make-suite-file-list")
+ DBConn()
+
+ Logger = daklog.Logger(cnf.Cnf, "make-suite-file-list")
do_da_do_da()
Logger.close()
import apt_pkg, os, sys, pwd, time, commands
from daklib import queue
-from daklib import logging
+from daklib import daklog
from daklib import utils
-from daklib import database
+from daklib.dbconn import DBConn, get_or_set_queue, get_suite_architectures
from daklib.regexes import re_taint_free
Cnf = None
if Options["No-Action"]:
Options["Sudo"] = ""
if not Options["Sudo"] and not Options["No-Action"]:
- Logger = Upload.Logger = logging.Logger(Cnf, "new-security-install")
+ Logger = Upload.Logger = daklog.Logger(Cnf, "new-security-install")
return arguments
ver, suite)
adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header))
- arches = database.get_suite_architectures(suite)
+ arches = [x.arch_name for x in get_suite_architectures(suite)]
if "source" in arches:
arches.remove("source")
if "all" in arches:
if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"):
utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"])
+ session = DBConn().session()
+
dest = Cnf["Dir::Queue::Unembargoed"]
- emb_q = database.get_or_set_queue_id("embargoed")
- une_q = database.get_or_set_queue_id("unembargoed")
+ emb_q = get_or_set_queue("embargoed", session)
+ une_q = get_or_set_queue("unembargoed", session)
for c in changes:
print "Disembargoing %s" % (c)
if "source" in Upload.pkg.changes["architecture"].keys():
print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])
- Upload.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]))
+ session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)",
+ {'package': Upload.pkg.changes["source"], 'version': Upload.pkg.changes["version"]})
files = {}
for suite in Upload.pkg.changes["distribution"].keys():
files[os.path.join(dest_dir, file)] = 1
files = files.keys()
- Upload.projectB.query("BEGIN WORK")
for f in files:
- Upload.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q))
- Upload.projectB.query("COMMIT WORK")
+ session.execute("UPDATE queue_build SET queue = :unembargoed WHERE filename = :filename AND queue = :embargoed",
+ {'unembargoed': une_q.queue_id, 'filename': f, 'embargoed': emb_q.queue_id})
+ session.commit()
for file in Upload.pkg.files.keys():
utils.copy(file, os.path.join(dest, file))
utils.copy(k, os.path.join(dest, k))
os.unlink(k)
+ session.commit()
+
def do_Reject(): sudo("R", _do_Reject, True)
def _do_Reject():
global changes
+
+ session = DBConn().session()
+
for c in changes:
print "Rejecting %s..." % (c)
Upload.init_vars()
if not aborted:
os.unlink(c[:-8]+".dak")
for f in files:
- Upload.projectB.query(
- "DELETE FROM queue_build WHERE filename = '%s'" % (f))
+ session.execute("DELETE FROM queue_build WHERE filename = :filename",
+ {'filename': f})
os.unlink(f)
print "Updating buildd information..."
if os.path.exists(adv_file):
os.unlink(adv_file)
+ session.commit()
+
def do_DropAdvisory():
for c in changes:
Upload.init_vars()
## That Alisha Rules The World
################################################################################
-import pg, sys
+import os
+import sys
import apt_pkg
-from daklib import logging
-from daklib import database
-from daklib import utils
-
-################################################################################
-Cnf = None
-projectB = None
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
+from daklib import utils
################################################################################
sys.exit(exit_code)
def main ():
- global Cnf, projectB
-
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Override::Options::Help"),
('d',"done","Override::Options::Done", "HasArg"),
('s',"suite","Override::Options::Suite", "HasArg"),
]
for i in ["help", "no-action"]:
- if not Cnf.has_key("Override::Options::%s" % (i)):
- Cnf["Override::Options::%s" % (i)] = ""
- if not Cnf.has_key("Override::Options::Suite"):
- Cnf["Override::Options::Suite"] = "unstable"
+ if not cnf.has_key("Override::Options::%s" % (i)):
+ cnf["Override::Options::%s" % (i)] = ""
+ if not cnf.has_key("Override::Options::Suite"):
+ cnf["Override::Options::Suite"] = "unstable"
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Override::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Override::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
if not arguments:
utils.fubar("package name is a required argument.")
if arguments and len(arguments) == 1:
# Determine if the argument is a priority or a section...
arg = arguments.pop()
- q = projectB.query("""
- SELECT ( SELECT COUNT(*) FROM section WHERE section=%s ) AS secs,
- ( SELECT COUNT(*) FROM priority WHERE priority=%s ) AS prios
- """ % ( pg._quote(arg,"str"), pg._quote(arg,"str")))
- r = q.getresult()
+ q = session.execute("""
+ SELECT ( SELECT COUNT(*) FROM section WHERE section = :arg ) AS secs,
+ ( SELECT COUNT(*) FROM priority WHERE priority = :arg ) AS prios
+ """, {'arg': arg})
+ r = q.fetchall()
if r[0][0] == 1:
- arguments = (arg,".")
+ arguments = (arg, ".")
elif r[0][1] == 1:
- arguments = (".",arg)
+ arguments = (".", arg)
else:
utils.fubar("%s is not a valid section or priority" % (arg))
eqdsc = '!='
if packagetype == 'source':
eqdsc = '='
- q = projectB.query("""
+ q = session.execute("""
SELECT priority.priority AS prio, section.section AS sect, override_type.type AS type
FROM override, priority, section, suite, override_type
WHERE override.priority = priority.id
AND override.type = override_type.id
AND override_type.type %s 'dsc'
AND override.section = section.id
- AND override.package = %s
+ AND override.package = :package
AND override.suite = suite.id
- AND suite.suite_name = %s
- """ % (eqdsc, pg._quote(package,"str"), pg._quote(suite,"str")))
+ AND suite.suite_name = :suite
+ """ % (eqdsc), {'package': package, 'suite': suite})
- if q.ntuples() == 0:
+ if q.rowcount == 0:
continue
- if q.ntuples() > 1:
+ if q.rowcount > 1:
utils.fubar("%s is ambiguous. Matches %d packages" % (package,q.ntuples()))
- r = q.getresult()
+ r = q.fetchone()
if packagetype == 'binary':
- oldsection = r[0][1]
- oldpriority = r[0][0]
+ oldsection = r[1]
+ oldpriority = r[0]
else:
- oldsourcesection = r[0][1]
+ oldsourcesection = r[1]
oldpriority = 'source'
if not oldpriority and not oldsourcesection:
utils.fubar("Unable to find package %s" % (package))
+
if oldsection and oldsourcesection and oldsection != oldsourcesection:
# When setting overrides, both source & binary will become the same section
utils.warn("Source is in section '%s' instead of '%s'" % (oldsourcesection, oldsection))
+
if not oldsection:
oldsection = oldsourcesection
if not arguments:
print "%s is in section '%s' at priority '%s'" % (
- package,oldsection,oldpriority)
+ package, oldsection, oldpriority)
sys.exit(0)
# At this point, we have a new section and priority... check they're valid...
if newpriority == ".":
newpriority = oldpriority
- q = projectB.query("SELECT id FROM section WHERE section=%s" % (
- pg._quote(newsection,"str")))
-
- if q.ntuples() == 0:
+ s = get_section(newsection, session)
+ if s is None:
utils.fubar("Supplied section %s is invalid" % (newsection))
- newsecid = q.getresult()[0][0]
+ newsecid = s.section_id
- q = projectB.query("SELECT id FROM priority WHERE priority=%s" % (
- pg._quote(newpriority,"str")))
-
- if q.ntuples() == 0:
+ p = get_priority(newpriority, session)
+ if p is None:
utils.fubar("Supplied priority %s is invalid" % (newpriority))
- newprioid = q.getresult()[0][0]
+ newprioid = p.priority_id
if newpriority == oldpriority and newsection == oldsection:
print "I: Doing nothing"
if newpriority != oldpriority:
print "I: Will change priority from %s to %s" % (oldpriority,newpriority)
+
if newsection != oldsection:
print "I: Will change section from %s to %s" % (oldsection,newsection)
game_over()
- Logger = logging.Logger(Cnf, "override")
+ Logger = daklog.Logger(cnf.Cnf, "override")
- projectB.query("BEGIN WORK")
+ dsc_otype_id = get_override_type('dsc').overridetype_id
+
+ # We're already in a transaction
# We're in "do it" mode, we have something to do... do it
if newpriority != oldpriority:
- q = projectB.query("""
+ session.execute("""
UPDATE override
- SET priority=%d
- WHERE package=%s
- AND override.type != %d
- AND suite = (SELECT id FROM suite WHERE suite_name=%s)""" % (
- newprioid,
- pg._quote(package,"str"), database.get_override_type_id("dsc"),
- pg._quote(suite,"str") ))
- Logger.log(["changed priority",package,oldpriority,newpriority])
+ SET priority = :newprioid
+ WHERE package = :package
+ AND override.type != :otypedsc
+ AND suite = (SELECT id FROM suite WHERE suite_name = :suite)""",
+ {'newprioid': newprioid, 'package': package,
+ 'otypedsc': dsc_otype_id, 'suite': suite})
+
+ Logger.log(["changed priority", package, oldpriority, newpriority])
if newsection != oldsection:
- q = projectB.query("""
+ q = session.execute("""
UPDATE override
- SET section=%d
- WHERE package=%s
- AND suite = (SELECT id FROM suite WHERE suite_name=%s)""" % (
- newsecid,
- pg._quote(package,"str"),
- pg._quote(suite,"str") ))
- Logger.log(["changed section",package,oldsection,newsection])
- projectB.query("COMMIT WORK")
+ SET section = :newsecid
+ WHERE package = :package
+ AND suite = (SELECT id FROM suite WHERE suite_name = :suite)""",
+ {'newsecid': newsecid, 'package': package,
+ 'suite': suite})
+
+ Logger.log(["changed section", package, oldsection, newsection])
+
+ session.commit()
if Options.has_key("Done"):
Subst = {}
- Subst["__OVERRIDE_ADDRESS__"] = Cnf["Override::MyEmailAddress"]
- Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
+ Subst["__OVERRIDE_ADDRESS__"] = cnf["Override::MyEmailAddress"]
+ Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
bcc = []
- if Cnf.Find("Dinstall::Bcc") != "":
- bcc.append(Cnf["Dinstall::Bcc"])
- if Cnf.Find("Override::Bcc") != "":
- bcc.append(Cnf["Override::Bcc"])
+ if cnf.Find("Dinstall::Bcc") != "":
+ bcc.append(cnf["Dinstall::Bcc"])
+ if cnf.Find("Override::Bcc") != "":
+ bcc.append(cnf["Override::Bcc"])
if bcc:
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst["__BCC__"] = "X-Filler: 42"
- Subst["__CC__"] = "Cc: " + package + "@" + Cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override\nX-Katie: alicia"
+ Subst["__CC__"] = "Cc: " + package + "@" + Cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override"
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
Subst["__WHOAMI__"] = utils.whoami()
summary += "Changed section from %s to %s\n" % (oldsection,newsection)
Subst["__SUMMARY__"] = summary
+ template = os.path.join(cnf["Dir::Templates"], "override.bug-close")
for bug in utils.split_args(Options["Done"]):
Subst["__BUG_NUMBER__"] = bug
- mail_message = utils.TemplateSubst(
- Subst,Cnf["Dir::Templates"]+"/override.bug-close")
+ mail_message = utils.TemplateSubst(Subst, template)
utils.send_mail(mail_message)
- Logger.log(["closed bug",bug])
+ Logger.log(["closed bug", bug])
Logger.close()
- print "Done"
-
#################################################################################
if __name__ == '__main__':
import fcntl
import os
import sys
-import time
-import re
-import apt_pkg, commands
-from daklib import database
-from daklib import logging
-from daklib import queue
+from datetime import datetime
+import apt_pkg
+
+from daklib import daklog
+from daklib.queue import *
from daklib import utils
+from daklib.dbconn import *
from daklib.dak_exceptions import *
from daklib.regexes import re_default_answer, re_issource, re_fdnic
+from daklib.urgencylog import UrgencyLog
+from daklib.summarystats import SummaryStats
+from daklib.config import Config
###############################################################################
-Cnf = None
Options = None
Logger = None
-Urgency_Logger = None
-projectB = None
-Upload = None
-pkg = None
-
-reject_message = ""
-changes = None
-dsc = None
-dsc_files = None
-files = None
-Subst = None
-
-install_count = 0
-install_bytes = 0.0
-
-installing_to_stable = 0
-
-###############################################################################
-
-# FIXME: this should go away to some Debian specific file
-# FIXME: should die if file already exists
-
-class Urgency_Log:
- "Urgency Logger object"
- def __init__ (self, Cnf):
- "Initialize a new Urgency Logger object"
- self.Cnf = Cnf
- self.timestamp = time.strftime("%Y%m%d%H%M%S")
- # Create the log directory if it doesn't exist
- self.log_dir = Cnf["Dir::UrgencyLog"]
- if not os.path.exists(self.log_dir) or not os.access(self.log_dir, os.W_OK):
- utils.warn("UrgencyLog directory %s does not exist or is not writeable, using /srv/ftp.debian.org/tmp/ instead" % (self.log_dir))
- self.log_dir = '/srv/ftp.debian.org/tmp/'
- # Open the logfile
- self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp)
- self.log_file = utils.open_file(self.log_filename, 'w')
- self.writes = 0
-
- def log (self, source, version, urgency):
- "Log an event"
- self.log_file.write(" ".join([source, version, urgency])+'\n')
- self.log_file.flush()
- self.writes += 1
-
- def close (self):
- "Close a Logger object"
- self.log_file.flush()
- self.log_file.close()
- if self.writes:
- new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp)
- utils.move(self.log_filename, new_filename)
- else:
- os.unlink(self.log_filename)
-
-
-###############################################################################
-
-
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-# Recheck anything that relies on the database; since that's not
-# frozen between accept and our run time.
-
-def check():
- propogate={}
- nopropogate={}
- for checkfile in files.keys():
- # The .orig.tar.gz can disappear out from under us is it's a
- # duplicate of one in the archive.
- if not files.has_key(checkfile):
- continue
- # Check that the source still exists
- if files[checkfile]["type"] == "deb":
- source_version = files[checkfile]["source version"]
- source_package = files[checkfile]["source package"]
- if not changes["architecture"].has_key("source") \
- and not Upload.source_exists(source_package, source_version, changes["distribution"].keys()):
- reject("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
-
- # Version and file overwrite checks
- if not installing_to_stable:
- if files[checkfile]["type"] == "deb":
- reject(Upload.check_binary_against_db(checkfile), "")
- elif files[checkfile]["type"] == "dsc":
- reject(Upload.check_source_against_db(checkfile), "")
- (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(checkfile)
- reject(reject_msg, "")
-
- # propogate in the case it is in the override tables:
- if changes.has_key("propdistribution"):
- for suite in changes["propdistribution"].keys():
- if Upload.in_override_p(files[checkfile]["package"], files[checkfile]["component"], suite, files[checkfile].get("dbtype",""), checkfile):
- propogate[suite] = 1
- else:
- nopropogate[suite] = 1
-
- for suite in propogate.keys():
- if suite in nopropogate:
- continue
- changes["distribution"][suite] = 1
-
- for checkfile in files.keys():
- # Check the package is still in the override tables
- for suite in changes["distribution"].keys():
- if not Upload.in_override_p(files[checkfile]["package"], files[checkfile]["component"], suite, files[checkfile].get("dbtype",""), checkfile):
- reject("%s is NEW for %s." % (checkfile, suite))
###############################################################################
def init():
- global Cnf, Options, Upload, projectB, changes, dsc, dsc_files, files, pkg, Subst
+ global Options
- Cnf = utils.get_conf()
+ # Initialize config and connection to db
+ cnf = Config()
+ DBConn()
Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
('h',"help","Dinstall::Options::Help"),
for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
"version", "directory"]:
- if not Cnf.has_key("Dinstall::Options::%s" % (i)):
- Cnf["Dinstall::Options::%s" % (i)] = ""
+ if not cnf.has_key("Dinstall::Options::%s" % (i)):
+ cnf["Dinstall::Options::%s" % (i)] = ""
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Dinstall::Options")
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Dinstall::Options")
if Options["Help"]:
usage()
# If we have a directory flag, use it to find our files
- if Cnf["Dinstall::Options::Directory"] != "":
+ if cnf["Dinstall::Options::Directory"] != "":
# Note that we clobber the list of files we were given in this case
# so warn if the user has done both
if len(changes_files) > 0:
utils.warn("Directory provided so ignoring files given on command line")
- changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
-
- Upload = queue.Upload(Cnf)
- projectB = Upload.projectB
-
- changes = Upload.pkg.changes
- dsc = Upload.pkg.dsc
- dsc_files = Upload.pkg.dsc_files
- files = Upload.pkg.files
- pkg = Upload.pkg
- Subst = Upload.Subst
+ changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
return changes_files
###############################################################################
-def action (queue=""):
- (summary, short_summary) = Upload.build_summaries()
+def action (u, stable_queue=None, log_urgency=True, session=None):
+ (summary, short_summary) = u.build_summaries()
+ pi = u.package_info()
(prompt, answer) = ("", "XXX")
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
- if reject_message.find("Rejected") != -1:
- print "REJECT\n" + reject_message,
+ if len(u.rejects) > 0:
+ print "REJECT\n" + pi
prompt = "[R]eject, Skip, Quit ?"
if Options["Automatic"]:
answer = 'R'
else:
- print "INSTALL to " + ", ".join(changes["distribution"].keys())
- print reject_message + summary,
+ print "INSTALL to " + ", ".join(u.pkg.changes["distribution"].keys())
+ print pi + summary,
prompt = "[I]nstall, Skip, Quit ?"
if Options["Automatic"]:
answer = 'I'
answer = answer[:1].upper()
if answer == 'R':
- do_reject ()
+ u.do_unaccept()
+ Logger.log(["unaccepted", u.pkg.changes_file])
elif answer == 'I':
- if not installing_to_stable:
- install()
+ if stable_queue:
+ stable_install(u, summary, short_summary, stable_queue, log_urgency)
else:
- stable_install(summary, short_summary, queue)
+ install(u, session, log_urgency)
elif answer == 'Q':
sys.exit(0)
+
###############################################################################
+def add_poolfile(filename, datadict, location_id, session):
+ poolfile = PoolFile()
+ poolfile.filename = filename
+ poolfile.filesize = datadict["size"]
+ poolfile.md5sum = datadict["md5sum"]
+ poolfile.sha1sum = datadict["sha1sum"]
+ poolfile.sha256sum = datadict["sha256sum"]
+ poolfile.location_id = location_id
+
+ session.add(poolfile)
+ # Flush to get a file id (NB: This is not a commit)
+ session.flush()
+
+ return poolfile
+
+def add_dsc_to_db(u, filename, session):
+ entry = u.pkg.files[filename]
+ source = DBSource()
+
+ source.source = u.pkg.dsc["source"]
+ source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
+ source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id
+ source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id
+ source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
+ source.install_date = datetime.now().date()
+
+ dsc_component = entry["component"]
+ dsc_location_id = entry["location id"]
+
+ source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes")
+
+ # Set up a new poolfile if necessary
+ if not entry.has_key("files id") or not entry["files id"]:
+ filename = entry["pool name"] + filename
+ poolfile = add_poolfile(filename, entry, dsc_location_id, session)
+ entry["files id"] = poolfile.file_id
+
+ source.poolfile_id = entry["files id"]
+ session.add(source)
+ session.flush()
+
+ for suite_name in u.pkg.changes["distribution"].keys():
+ sa = SrcAssociation()
+ sa.source_id = source.source_id
+ sa.suite_id = get_suite(suite_name).suite_id
+ session.add(sa)
+
+ session.flush()
+
+ # Add the source files to the DB (files and dsc_files)
+ dscfile = DSCFile()
+ dscfile.source_id = source.source_id
+ dscfile.poolfile_id = entry["files id"]
+ session.add(dscfile)
+
+ for dsc_file, dentry in u.pkg.dsc_files.items():
+ df = DSCFile()
+ df.source_id = source.source_id
+
+ # If the .orig tarball is already in the pool, it's
+ # files id is stored in dsc_files by check_dsc().
+ files_id = dentry.get("files id", None)
+
+ # Find the entry in the files hash
+ # TODO: Bail out here properly
+ dfentry = None
+ for f, e in u.pkg.files.items():
+ if f == dsc_file:
+ dfentry = e
+ break
+
+ if files_id is None:
+ filename = dfentry["pool name"] + dsc_file
+
+ (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id)
+ # FIXME: needs to check for -1/-2 and or handle exception
+ if found and obj is not None:
+ files_id = obj.file_id
+
+ # If still not found, add it
+ if files_id is None:
+ # HACK: Force sha1sum etc into dentry
+ dentry["sha1sum"] = dfentry["sha1sum"]
+ dentry["sha256sum"] = dfentry["sha256sum"]
+ poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
+ files_id = poolfile.file_id
+
+ df.poolfile_id = files_id
+ session.add(df)
+
+ session.flush()
+
+ # Add the src_uploaders to the DB
+ uploader_ids = [source.maintainer_id]
+ if u.pkg.dsc.has_key("uploaders"):
+ for up in u.pkg.dsc["uploaders"].split(","):
+ up = up.strip()
+ uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
+
+ added_ids = {}
+ for up in uploader_ids:
+ if added_ids.has_key(up):
+ utils.warn("Already saw uploader %s for source %s" % (up, source.source))
+ continue
-# Our reject is not really a reject, but an unaccept, but since a) the
-# code for that is non-trivial (reopen bugs, unannounce etc.), b) this
-# should be exteremly rare, for now we'll go with whining at our admin
-# folks...
-
-def do_reject ():
- Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
- Subst["__REJECT_MESSAGE__"] = reject_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
- reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-accepted.unaccept")
-
- # Write the rejection email out as the <foo>.reason file
- reason_filename = os.path.basename(pkg.changes_file[:-8]) + ".reason"
- reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename
- # If we fail here someone is probably trying to exploit the race
- # so let's just raise an exception ...
- if os.path.exists(reject_filename):
- os.unlink(reject_filename)
- fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
- os.write(fd, reject_mail_message)
- os.close(fd)
-
- utils.send_mail(reject_mail_message)
- Logger.log(["unaccepted", pkg.changes_file])
+ added_ids[u]=1
-###############################################################################
+ su = SrcUploader()
+ su.maintainer_id = up
+ su.source_id = source.source_id
+ session.add(su)
-def install ():
- global install_count, install_bytes
+ session.flush()
- print "Installing."
+ return dsc_component, dsc_location_id
+
+def add_deb_to_db(u, filename, session):
+ """
+ Contrary to what you might expect, this routine deals with both
+ debs and udebs. That info is in 'dbtype', whilst 'type' is
+ 'deb' for both of them
+ """
+ cnf = Config()
+ entry = u.pkg.files[filename]
+
+ bin = DBBinary()
+ bin.package = entry["package"]
+ bin.version = entry["version"]
+ bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id
+ bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id
+ bin.arch_id = get_architecture(entry["architecture"], session).arch_id
+ bin.binarytype = entry["dbtype"]
+
+ # Find poolfile id
+ filename = entry["pool name"] + filename
+ fullpath = os.path.join(cnf["Dir::Pool"], filename)
+ if not entry.get("location id", None):
+ entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id
+
+ if not entry.get("files id", None):
+ poolfile = add_poolfile(filename, entry, entry["location id"], session)
+ entry["files id"] = poolfile.file_id
+
+ bin.poolfile_id = entry["files id"]
+
+ # Find source id
+ bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
+ if len(bin_sources) != 1:
+ raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \
+ (bin.package, bin.version, bin.architecture.arch_string,
+ filename, bin.binarytype, u.pkg.changes["fingerprint"])
+
+ bin.source_id = bin_sources[0].source_id
+
+ # Add and flush object so it has an ID
+ session.add(bin)
+ session.flush()
+
+ # Add BinAssociations
+ for suite_name in u.pkg.changes["distribution"].keys():
+ ba = BinAssociation()
+ ba.binary_id = bin.binary_id
+ ba.suite_id = get_suite(suite_name).suite_id
+ session.add(ba)
+
+ session.flush()
+
+ # Deal with contents - disabled for now
+ #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session)
+ #if not contents:
+ # print "REJECT\nCould not determine contents of package %s" % bin.package
+ # session.rollback()
+ # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
- Logger.log(["installing changes",pkg.changes_file])
- # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
- projectB.query("BEGIN WORK")
+def install(u, session, log_urgency=True):
+ cnf = Config()
+ summarystats = SummaryStats()
+
+ print "Installing."
+
+ Logger.log(["installing changes", u.pkg.changes_file])
# Ensure that we have all the hashes we need below.
- rejmsg = utils.ensure_hashes(changes, dsc, files, dsc_files)
- if len(rejmsg) > 0:
+ u.ensure_hashes()
+ if len(u.rejects) > 0:
# There were errors. Print them and SKIP the changes.
- for msg in rejmsg:
+ for msg in u.rejects:
utils.warn(msg)
return
- # Add the .dsc file to the DB
- for newfile in files.keys():
- if files[newfile]["type"] == "dsc":
- package = dsc["source"]
- version = dsc["version"] # NB: not files[file]["version"], that has no epoch
- maintainer = dsc["maintainer"]
- maintainer = maintainer.replace("'", "\\'")
- maintainer_id = database.get_or_set_maintainer_id(maintainer)
- changedby = changes["changed-by"]
- changedby = changedby.replace("'", "\\'")
- changedby_id = database.get_or_set_maintainer_id(changedby)
- fingerprint_id = database.get_or_set_fingerprint_id(dsc["fingerprint"])
- install_date = time.strftime("%Y-%m-%d")
- filename = files[newfile]["pool name"] + newfile
- dsc_component = files[newfile]["component"]
- dsc_location_id = files[newfile]["location id"]
- if dsc.has_key("dm-upload-allowed") and dsc["dm-upload-allowed"] == "yes":
- dm_upload_allowed = "true"
- else:
- dm_upload_allowed = "false"
- if not files[newfile].has_key("files id") or not files[newfile]["files id"]:
- files[newfile]["files id"] = database.set_files_id (filename, files[newfile]["size"], files[newfile]["md5sum"], files[newfile]["sha1sum"], files[newfile]["sha256sum"], dsc_location_id)
- projectB.query("INSERT INTO source (source, version, maintainer, changedby, file, install_date, sig_fpr, dm_upload_allowed) VALUES ('%s', '%s', %d, %d, %d, '%s', %s, %s)"
- % (package, version, maintainer_id, changedby_id, files[newfile]["files id"], install_date, fingerprint_id, dm_upload_allowed))
-
- for suite in changes["distribution"].keys():
- suite_id = database.get_suite_id(suite)
- projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
-
- # Add the source files to the DB (files and dsc_files)
- projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[newfile]["files id"]))
- for dsc_file in dsc_files.keys():
- filename = files[newfile]["pool name"] + dsc_file
- # If the .orig.tar.gz is already in the pool, it's
- # files id is stored in dsc_files by check_dsc().
- files_id = dsc_files[dsc_file].get("files id", None)
- if files_id == None:
- files_id = database.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id)
- # FIXME: needs to check for -1/-2 and or handle exception
- if files_id == None:
- files_id = database.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], files[dsc_file]["sha1sum"], files[dsc_file]["sha256sum"], dsc_location_id)
- projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id))
-
- # Add the src_uploaders to the DB
- uploader_ids = [maintainer_id]
- if dsc.has_key("uploaders"):
- for u in dsc["uploaders"].split(","):
- u = u.replace("'", "\\'")
- u = u.strip()
- uploader_ids.append(
- database.get_or_set_maintainer_id(u))
- added_ids = {}
- for u in uploader_ids:
- if added_ids.has_key(u):
- utils.warn("Already saw uploader %s for source %s" % (u, package))
- continue
- added_ids[u]=1
- projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u))
-
-
- # Add the .deb files to the DB
- for newfile in files.keys():
- if files[newfile]["type"] == "deb":
- package = files[newfile]["package"]
- version = files[newfile]["version"]
- maintainer = files[newfile]["maintainer"]
- maintainer = maintainer.replace("'", "\\'")
- maintainer_id = database.get_or_set_maintainer_id(maintainer)
- fingerprint_id = database.get_or_set_fingerprint_id(changes["fingerprint"])
- architecture = files[newfile]["architecture"]
- architecture_id = database.get_architecture_id (architecture)
- filetype = files[newfile]["dbtype"]
- source = files[newfile]["source package"]
- source_version = files[newfile]["source version"]
- filename = files[newfile]["pool name"] + newfile
- if not files[newfile].has_key("location id") or not files[newfile]["location id"]:
- files[newfile]["location id"] = database.get_location_id(Cnf["Dir::Pool"],files[newfile]["component"],utils.where_am_i())
- if not files[newfile].has_key("files id") or not files[newfile]["files id"]:
- files[newfile]["files id"] = database.set_files_id (filename, files[newfile]["size"], files[newfile]["md5sum"], files[newfile]["sha1sum"], files[newfile]["sha256sum"], files[newfile]["location id"])
- source_id = database.get_source_id (source, source_version)
- if source_id:
- projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, %d, '%s', %d)"
- % (package, version, maintainer_id, source_id, architecture_id, files[newfile]["files id"], filetype, fingerprint_id))
- else:
- raise NoSourceFieldError, "Unable to find a source id for %s (%s), %s, file %s, type %s, signed by %s" % (package, version, architecture, newfile, filetype, changes["fingerprint"])
- for suite in changes["distribution"].keys():
- suite_id = database.get_suite_id(suite)
- projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id))
+ # Add the .dsc file to the DB first
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "dsc":
+ dsc_component, dsc_location_id = add_dsc_to_db(u, newfile, session)
- if not database.copy_temporary_contents(package, version, architecture, newfile, reject):
- print "REJECT\n" + reject_message,
- projectB.query("ROLLBACK")
- raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (package, newfile )
-
-
- orig_tar_id = Upload.pkg.orig_tar_id
- orig_tar_location = Upload.pkg.orig_tar_location
+ # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "deb":
+ add_deb_to_db(u, newfile, session)
# If this is a sourceful diff only upload that is moving
- # cross-component we need to copy the .orig.tar.gz into the new
+ # cross-component we need to copy the .orig files into the new
# component too for the same reasons as above.
- #
- if changes["architecture"].has_key("source") and orig_tar_id and \
- orig_tar_location != dsc_location_id:
- q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum, f.sha1sum, f.sha256sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id))
- ql = q.getresult()[0]
- old_filename = ql[0] + ql[1]
- file_size = ql[2]
- file_md5sum = ql[3]
- file_sha1sum = ql[4]
- file_sha256sum = ql[5]
- new_filename = utils.poolify(changes["source"], dsc_component) + os.path.basename(old_filename)
- new_files_id = database.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id)
- if new_files_id == None:
- utils.copy(old_filename, Cnf["Dir::Pool"] + new_filename)
- new_files_id = database.set_files_id(new_filename, file_size, file_md5sum, file_sha1sum, file_sha256sum, dsc_location_id)
- projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, database.get_source_id(changes["source"], changes["version"]), orig_tar_id))
+ if u.pkg.changes["architecture"].has_key("source"):
+ for orig_file in u.pkg.orig_files.keys():
+ if not u.pkg.orig_files[orig_file].has_key("id"):
+ continue # Skip if it's not in the pool
+ orig_file_id = u.pkg.orig_files[orig_file]["id"]
+ if u.pkg.orig_files[orig_file]["location"] == dsc_location_id:
+ continue # Skip if the location didn't change
+
+ # Do the move
+ oldf = get_poolfile_by_id(orig_file_id, session)
+ old_filename = os.path.join(oldf.location.path, oldf.filename)
+ old_dat = {'size': oldf.filesize, 'md5sum': oldf.md5sum,
+ 'sha1sum': oldf.sha1sum, 'sha256sum': oldf.sha256sum}
+
+ new_filename = os.path.join(utils.poolify(u.pkg.changes["source"], dsc_component), os.path.basename(old_filename))
+
+ # TODO: Care about size/md5sum collisions etc
+ (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
+
+ if newf is None:
+ utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
+ newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)
+
+ # TODO: Check that there's only 1 here
+ source = get_sources_from_name(u.pkg.changes["source"], u.pkg.changes["version"])[0]
+ dscf = get_dscfiles(source_id=source.source_id, poolfile_id=orig_file_id, session=session)[0]
+ dscf.poolfile_id = newf.file_id
+ session.add(dscf)
+ session.flush()
# Install the files into the pool
- for newfile in files.keys():
- destination = Cnf["Dir::Pool"] + files[newfile]["pool name"] + newfile
+ for newfile, entry in u.pkg.files.items():
+ destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
utils.move(newfile, destination)
- Logger.log(["installed", newfile, files[newfile]["type"], files[newfile]["size"], files[newfile]["architecture"]])
- install_bytes += float(files[newfile]["size"])
+ Logger.log(["installed", newfile, entry["type"], entry["size"], entry["architecture"]])
+ summarystats.accept_bytes += float(entry["size"])
# Copy the .changes file across for suite which need it.
copy_changes = {}
copy_dot_dak = {}
- for suite in changes["distribution"].keys():
- if Cnf.has_key("Suite::%s::CopyChanges" % (suite)):
- copy_changes[Cnf["Suite::%s::CopyChanges" % (suite)]] = ""
+ for suite_name in u.pkg.changes["distribution"].keys():
+ if cnf.has_key("Suite::%s::CopyChanges" % (suite_name)):
+ copy_changes[cnf["Suite::%s::CopyChanges" % (suite_name)]] = ""
# and the .dak file...
- if Cnf.has_key("Suite::%s::CopyDotDak" % (suite)):
- copy_dot_dak[Cnf["Suite::%s::CopyDotDak" % (suite)]] = ""
+ if cnf.has_key("Suite::%s::CopyDotDak" % (suite_name)):
+ copy_dot_dak[cnf["Suite::%s::CopyDotDak" % (suite_name)]] = ""
+
for dest in copy_changes.keys():
- utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest)
+ utils.copy(u.pkg.changes_file, os.path.join(cnf["Dir::Root"], dest))
+
for dest in copy_dot_dak.keys():
- utils.copy(Upload.pkg.changes_file[:-8]+".dak", dest)
- projectB.query("COMMIT WORK")
+ utils.copy(u.pkg.changes_file[:-8]+".dak", dest)
+
+ # We're done - commit the database changes
+ session.commit()
# Move the .changes into the 'done' directory
- utils.move (pkg.changes_file,
- os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file)))
+ utils.move(u.pkg.changes_file,
+ os.path.join(cnf["Dir::Queue::Done"], os.path.basename(u.pkg.changes_file)))
# Remove the .dak file
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ os.unlink(u.pkg.changes_file[:-8] + ".dak")
+
+ if u.pkg.changes["architecture"].has_key("source") and log_urgency:
+ UrgencyLog().log(u.pkg.dsc["source"], u.pkg.dsc["version"], u.pkg.changes["urgency"])
- if changes["architecture"].has_key("source") and Urgency_Logger:
- Urgency_Logger.log(dsc["source"], dsc["version"], changes["urgency"])
+ # Our SQL session will automatically start a new transaction after
+ # the last commit
# Undo the work done in queue.py(accept) to help auto-building
# from accepted.
- projectB.query("BEGIN WORK")
- for suite in changes["distribution"].keys():
- if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
+ now_date = datetime.now()
+
+ for suite_name in u.pkg.changes["distribution"].keys():
+ if suite_name not in cnf.ValueList("Dinstall::QueueBuildSuites"):
continue
- now_date = time.strftime("%Y-%m-%d %H:%M")
- suite_id = database.get_suite_id(suite)
- dest_dir = Cnf["Dir::QueueBuild"]
- if Cnf.FindB("Dinstall::SecurityQueueBuild"):
- dest_dir = os.path.join(dest_dir, suite)
- for newfile in files.keys():
+
+ suite = get_suite(suite_name, session)
+ dest_dir = cnf["Dir::QueueBuild"]
+
+ if cnf.FindB("Dinstall::SecurityQueueBuild"):
+ dest_dir = os.path.join(dest_dir, suite_name)
+
+ for newfile, entry in u.pkg.files.items():
dest = os.path.join(dest_dir, newfile)
+
+ qb = get_queue_build(dest, suite.suite_id, session)
+
# Remove it from the list of packages for later processing by apt-ftparchive
- projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, dest, suite_id))
- if not Cnf.FindB("Dinstall::SecurityQueueBuild"):
+ if qb:
+ qb.last_used = now_date
+ qb.in_queue = False
+ session.add(qb)
+
+ if not cnf.FindB("Dinstall::SecurityQueueBuild"):
# Update the symlink to point to the new location in the pool
- pool_location = utils.poolify (changes["source"], files[newfile]["component"])
- src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(newfile))
+ pool_location = utils.poolify(u.pkg.changes["source"], entry["component"])
+ src = os.path.join(cnf["Dir::Pool"], pool_location, os.path.basename(newfile))
if os.path.islink(dest):
os.unlink(dest)
os.symlink(src, dest)
- # Update last_used on any non-upload .orig.tar.gz symlink
- if orig_tar_id:
+
+ # Update last_used on any non-uploaded .orig symlink
+ for orig_file in u.pkg.orig_files.keys():
# Determine the .orig.tar.gz file name
- for dsc_file in dsc_files.keys():
- if dsc_file.endswith(".orig.tar.gz"):
- orig_tar_gz = os.path.join(dest_dir, dsc_file)
+ if not u.pkg.orig_files[orig_file].has_key("id"):
+ continue # Skip files not in the pool
+ # XXX: do we really want to update the orig_files dict here
+ # instead of using a temporary variable?
+ u.pkg.orig_files[orig_file]["path"] = os.path.join(dest_dir, orig_file)
+
# Remove it from the list of packages for later processing by apt-ftparchive
- projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, orig_tar_gz, suite_id))
- projectB.query("COMMIT WORK")
+ qb = get_queue_build(u.pkg.orig_files[orig_file]["path"], suite.suite_id, session)
+ if qb:
+ qb.in_queue = False
+ qb.last_used = now_date
+ session.add(qb)
+
+ session.commit()
# Finally...
- install_count += 1
+ summarystats.accept_count += 1
################################################################################
-def stable_install (summary, short_summary, fromsuite="proposed-updates"):
- global install_count
+def stable_install(u, session, summary, short_summary, fromsuite_name="proposed-updates"):
+ summarystats = SummaryStats()
- fromsuite = fromsuite.lower()
- tosuite = "Stable"
- if fromsuite == "oldstable-proposed-updates":
- tosuite = "OldStable"
+ fromsuite_name = fromsuite_name.lower()
+ tosuite_name = "Stable"
+ if fromsuite_name == "oldstable-proposed-updates":
+ tosuite_name = "OldStable"
- print "Installing from %s to %s." % (fromsuite, tosuite)
+ print "Installing from %s to %s." % (fromsuite_name, tosuite_name)
- # Begin a transaction; if we bomb out anywhere between here and
- # the COMMIT WORK below, the DB won't be changed.
- projectB.query("BEGIN WORK")
+ fromsuite = get_suite(fromsuite_name)
+ tosuite = get_suite(tosuite_name)
# Add the source to stable (and remove it from proposed-updates)
- for newfile in files.keys():
- if files[newfile]["type"] == "dsc":
- package = dsc["source"]
- version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
- q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
- ql = q.getresult()
- if not ql:
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "dsc":
+ package = u.pkg.dsc["source"]
+ # NB: not files[file]["version"], that has no epoch
+ version = u.pkg.dsc["version"]
+
+ source = get_sources_from_name(package, version, session)
+ if len(source) < 1:
utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version))
- source_id = ql[0][0]
- suite_id = database.get_suite_id(fromsuite)
- projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id))
- suite_id = database.get_suite_id(tosuite.lower())
- projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id))
+ source = source[0]
+
+ # Remove from old suite
+ old = session.query(SrcAssociation).filter_by(source_id = source.source_id)
+ old = old.filter_by(suite_id = fromsuite.suite_id)
+ old.delete()
+
+ # Add to new suite
+ new = SrcAssociation()
+ new.source_id = source.source_id
+ new.suite_id = tosuite.suite_id
+ session.add(new)
# Add the binaries to stable (and remove it/them from proposed-updates)
- for newfile in files.keys():
- if files[newfile]["type"] == "deb":
- package = files[newfile]["package"]
- version = files[newfile]["version"]
- architecture = files[newfile]["architecture"]
- q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
- ql = q.getresult()
- if not ql:
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "deb":
+ package = entry["package"]
+ version = entry["version"]
+ architecture = entry["architecture"]
+
+ binary = get_binaries_from_name(package, version, [architecture, 'all'])
+
+ if len(binary) < 1:
utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture))
+ binary = binary[0]
- binary_id = ql[0][0]
- suite_id = database.get_suite_id(fromsuite)
- projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id))
- suite_id = database.get_suite_id(tosuite.lower())
- projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id))
+ # Remove from old suite
+ old = session.query(BinAssociation).filter_by(binary_id = binary.binary_id)
+ old = old.filter_by(suite_id = fromsuite.suite_id)
+ old.delete()
- projectB.query("COMMIT WORK")
+ # Add to new suite
+ new = BinAssociation()
+ new.binary_id = binary.binary_id
+ new.suite_id = tosuite.suite_id
+ session.add(new)
- utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/process-accepted/' + os.path.basename(pkg.changes_file))
+ session.commit()
+
+ utils.move(u.pkg.changes_file,
+ os.path.join(cnf["Dir::Morgue"], 'process-accepted', os.path.basename(u.pkg.changes_file)))
## Update the Stable ChangeLog file
- new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::%s::ChangeLogBase" % (tosuite)] + ".ChangeLog"
- changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::%s::ChangeLogBase" % (tosuite)] + "ChangeLog"
+ # TODO: URGH - Use a proper tmp file
+ new_changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + ".ChangeLog"
+ changelog_filename = cnf["Dir::Root"] + cnf["Suite::%s::ChangeLogBase" % (tosuite.suite_name)] + "ChangeLog"
if os.path.exists(new_changelog_filename):
- os.unlink (new_changelog_filename)
+ os.unlink(new_changelog_filename)
new_changelog = utils.open_file(new_changelog_filename, 'w')
- for newfile in files.keys():
- if files[newfile]["type"] == "deb":
- new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.lower(), files[newfile]["component"], files[newfile]["architecture"], newfile))
+ for newfile, entry in u.pkg.files.items():
+ if entry["type"] == "deb":
+ new_changelog.write("%s/%s/binary-%s/%s\n" % (tosuite.suite_name,
+ entry["component"],
+ entry["architecture"],
+ newfile))
elif re_issource.match(newfile):
- new_changelog.write("%s/%s/source/%s\n" % (tosuite.lower(), files[newfile]["component"], newfile))
+ new_changelog.write("%s/%s/source/%s\n" % (tosuite.suite_name,
+ entry["component"],
+ newfile))
else:
new_changelog.write("%s\n" % (newfile))
- chop_changes = re_fdnic.sub("\n", changes["changes"])
+
+ chop_changes = re_fdnic.sub("\n", u.pkg.changes["changes"])
new_changelog.write(chop_changes + '\n\n')
+
if os.access(changelog_filename, os.R_OK) != 0:
changelog = utils.open_file(changelog_filename)
new_changelog.write(changelog.read())
+
new_changelog.close()
+
if os.access(changelog_filename, os.R_OK) != 0:
os.unlink(changelog_filename)
utils.move(new_changelog_filename, changelog_filename)
- install_count += 1
+ summarystats.accept_count += 1
+
+ if not Options["No-Mail"] and u.pkg.changes["architecture"].has_key("source"):
+ u.Subst["__SUITE__"] = " into %s" % (tosuite)
+ u.Subst["__SUMMARY__"] = summary
+ u.Subst["__BCC__"] = "X-DAK: dak process-accepted"
- if not Options["No-Mail"] and changes["architecture"].has_key("source"):
- Subst["__SUITE__"] = " into %s" % (tosuite)
- Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-accepted.install")
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+
+ template = os.path.join(cnf["Dir::Templates"], 'process-accepted.install')
+
+ mail_message = utils.TemplateSubst(u.Subst, template)
utils.send_mail(mail_message)
- Upload.announce(short_summary, 1)
+ u.announce(short_summary, True)
# Finally remove the .dak file
- dot_dak_file = os.path.join(Cnf["Suite::%s::CopyDotDak" % (fromsuite)], os.path.basename(Upload.pkg.changes_file[:-8]+".dak"))
+ dot_dak_file = os.path.join(cnf["Suite::%s::CopyDotDak" % (fromsuite.suite_name)],
+ os.path.basename(u.pkg.changes_file[:-8]+".dak"))
os.unlink(dot_dak_file)
################################################################################
-def process_it (changes_file, queue=""):
- global reject_message
+def process_it(changes_file, stable_queue, log_urgency, session):
+ cnf = Config()
+ u = Upload()
- reject_message = ""
+ overwrite_checks = True
# Absolutize the filename to avoid the requirement of being in the
# same directory as the .changes file.
- pkg.changes_file = os.path.abspath(changes_file)
+ cfile = os.path.abspath(changes_file)
# And since handling of installs to stable munges with the CWD
# save and restore it.
- pkg.directory = os.getcwd()
+ u.prevdir = os.getcwd()
- if installing_to_stable:
- old = Upload.pkg.changes_file
- Upload.pkg.changes_file = os.path.basename(old)
- os.chdir(Cnf["Suite::%s::CopyDotDak" % (queue)])
+ if stable_queue:
+ old = cfile
+ cfile = os.path.basename(old)
+ os.chdir(cnf["Suite::%s::CopyDotDak" % (stable_queue)])
+ # overwrite_checks should not be performed if installing to stable
+ overwrite_checks = False
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
+ u.pkg.load_dot_dak(cfile)
+ u.update_subst()
- if installing_to_stable:
- Upload.pkg.changes_file = old
+ if stable_queue:
+ u.pkg.changes_file = old
- check()
- action(queue)
+ u.accepted_checks(overwrite_checks, session)
+ action(u, stable_queue, log_urgency, session)
# Restore CWD
- os.chdir(pkg.directory)
+ os.chdir(u.prevdir)
###############################################################################
def main():
- global projectB, Logger, Urgency_Logger, installing_to_stable
+ global Logger
+ cnf = Config()
+ summarystats = SummaryStats()
changes_files = init()
+ log_urgency = False
+ stable_queue = None
# -n/--dry-run invalidates some other options which would involve things happening
if Options["No-Action"]:
# Check that we aren't going to clash with the daily cron job
- if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]:
+ if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (cnf["Dir::Root"])) and not Options["No-Lock"]:
utils.fubar("Archive maintenance in progress. Try again later.")
# If running from within proposed-updates; assume an install to stable
queue = ""
if os.getenv('PWD').find('oldstable-proposed-updates') != -1:
- queue = "Oldstable-Proposed-Updates"
- installing_to_stable = 1
+ stable_queue = "Oldstable-Proposed-Updates"
elif os.getenv('PWD').find('proposed-updates') != -1:
- queue = "Proposed-Updates"
- installing_to_stable = 1
+ stable_queue = "Proposed-Updates"
# Obtain lock if not in no-action mode and initialize the log
if not Options["No-Action"]:
- lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+ lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError, e:
utils.fubar("Couldn't obtain lock; assuming another 'dak process-accepted' is already running.")
else:
raise
- Logger = Upload.Logger = logging.Logger(Cnf, "process-accepted")
- if not installing_to_stable and Cnf.get("Dir::UrgencyLog"):
- Urgency_Logger = Urgency_Log(Cnf)
-
- # Initialize the substitution template mapping global
- bcc = "X-DAK: dak process-accepted\nX-Katie: $Revision: 1.18 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Subst["__BCC__"] = bcc
+ Logger = daklog.Logger(cnf, "process-accepted")
+ if not stable_queue and cnf.get("Dir::UrgencyLog"):
+ # Initialise UrgencyLog()
+ log_urgency = True
+ UrgencyLog()
# Sort the .changes files so that we process sourceful ones first
changes_files.sort(utils.changes_compare)
+
# Process the changes files
for changes_file in changes_files:
print "\n" + changes_file
- process_it (changes_file, queue)
+ session = DBConn().session()
+ process_it(changes_file, stable_queue, log_urgency, session)
+ session.close()
- if install_count:
+ if summarystats.accept_count:
sets = "set"
- if install_count > 1:
+ if summarystats.accept_count > 1:
sets = "sets"
- sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))))
- Logger.log(["total",install_count,install_bytes])
+ sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets,
+ utils.size_type(int(summarystats.accept_bytes))))
+ Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
if not Options["No-Action"]:
Logger.close()
- if Urgency_Logger:
- Urgency_Logger.close()
+ if log_urgency:
+ UrgencyLog().close()
###############################################################################
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Frank Lichtenheld <djpig@debian.org>
@license: GNU General Public License version 2 or later
"""
# This program is free software; you can redistribute it and/or modify
import pwd
import apt_pkg, apt_inst
import examine_package
-from daklib import database
-from daklib import logging
-from daklib import queue
+
+from daklib.dbconn import *
+from daklib.queue import *
+from daklib import daklog
from daklib import utils
from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
+from daklib.summarystats import SummaryStats
+from daklib.config import Config
# Globals
-Cnf = None #: Configuration, apt_pkg.Configuration
Options = None
-Upload = None
-projectB = None #: database connection, pgobject
Logger = None
Priorities = None
Sections = None
-reject_message = ""
-
################################################################################
################################################################################
################################################################################
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-def recheck():
- global reject_message
- files = Upload.pkg.files
- reject_message = ""
+def recheck(upload, session):
+ files = upload.pkg.files
+ cnf = Config()
for f in files.keys():
# The .orig.tar.gz can disappear out from under us is it's a
# duplicate of one in the archive.
if files[f]["type"] == "deb":
source_version = files[f]["source version"]
source_package = files[f]["source package"]
- if not Upload.pkg.changes["architecture"].has_key("source") \
- and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
+ if not upload.pkg.changes["architecture"].has_key("source") \
+ and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()):
source_epochless_version = re_no_epoch.sub('', source_version)
dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
found = 0
for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]:
- if Cnf.has_key("Dir::Queue::%s" % (q)):
- if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
+ if cnf.has_key("Dir::Queue::%s" % (q)):
+ if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
found = 1
if not found:
- reject("no source found for %s %s (%s)." % (source_package, source_version, f))
+ upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
# Version and file overwrite checks
if files[f]["type"] == "deb":
- reject(Upload.check_binary_against_db(f), "")
+ upload.check_binary_against_db(f, session)
elif files[f]["type"] == "dsc":
- reject(Upload.check_source_against_db(f), "")
- (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
- reject(reject_msg, "")
+ upload.check_source_against_db(f, session)
+ upload.check_dsc_against_db(f, session)
- if reject_message.find("Rejected") != -1:
+ if len(upload.rejects) > 0:
answer = "XXX"
if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
answer = 'S'
- print "REJECT\n" + reject_message,
+ print "REJECT\n" + upload.rejects.join("\n"),
prompt = "[R]eject, Skip, Quit ?"
while prompt.find(answer) == -1:
answer = answer[:1].upper()
if answer == 'R':
- Upload.do_reject(0, reject_message)
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ upload.do_reject(manual=0, reject_message=upload.rejects.join("\n"))
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
return 0
elif answer == 'S':
return 0
# Sort by time of oldest upload
return cmp(a["oldest"], b["oldest"])
-def sort_changes(changes_files):
+def sort_changes(changes_files, session):
"""Sort into source groups, then sort each source group by version,
have source, filename. Finally, sort the source groups by have
note, time of oldest upload of each source upload."""
cache = {}
# Read in all the .changes files
for filename in changes_files:
+ u = Upload()
try:
- Upload.pkg.changes_file = filename
- Upload.init_vars()
- Upload.update_vars()
- cache[filename] = copy.copy(Upload.pkg.changes)
+ u.pkg.load_dot_dak(filename)
+ u.update_subst()
+ cache[filename] = copy.copy(u.pkg.changes)
cache[filename]["filename"] = filename
except:
sorted_list.append(filename)
per_source[source]["list"].append(cache[filename])
# Determine oldest time and have note status for each source group
for source in per_source.keys():
- q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
- ql = q.getresult()
- per_source[source]["source_in_database"] = len(ql)>0
+ q = session.query(DBSource).filter_by(source = source).all()
+ per_source[source]["source_in_database"] = len(q)>0
source_list = per_source[source]["list"]
first = source_list[0]
oldest = os.stat(first["filename"])[stat.ST_MTIME]
mtime = os.stat(d["filename"])[stat.ST_MTIME]
if mtime < oldest:
oldest = mtime
- have_note += (database.has_new_comment(d["source"], d["version"], True))
+ have_note += has_new_comment(d["source"], d["version"], session)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
################################################################################
class Section_Completer:
- def __init__ (self):
+ def __init__ (self, session):
self.sections = []
self.matches = []
- q = projectB.query("SELECT section FROM section")
- for i in q.getresult():
- self.sections.append(i[0])
+ for s, in session.query(Section.section):
+ self.sections.append(s)
def complete(self, text, state):
if state == 0:
############################################################
class Priority_Completer:
- def __init__ (self):
+ def __init__ (self, session):
self.priorities = []
self.matches = []
- q = projectB.query("SELECT priority FROM priority")
- for i in q.getresult():
- self.priorities.append(i[0])
+ for p, in session.query(Priority.priority):
+ self.priorities.append(p)
def complete(self, text, state):
if state == 0:
################################################################################
-def print_new (new, indexed, file=sys.stdout):
- queue.check_valid(new)
- broken = 0
+def print_new (new, upload, indexed, file=sys.stdout):
+ check_valid(new)
+ broken = False
index = 0
for pkg in new.keys():
index += 1
priority = new[pkg]["priority"]
if new[pkg]["section id"] == -1:
section += "[!]"
- broken = 1
+ broken = True
if new[pkg]["priority id"] == -1:
priority += "[!]"
- broken = 1
+ broken = True
if indexed:
line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
else:
line = "%-20s %-20s %-20s" % (pkg, priority, section)
line = line.strip()+'\n'
file.write(line)
- note = database.get_new_comments(Upload.pkg.changes.get("source"))
- if len(note) > 0:
- for line in note:
- print line
- return broken, note
+ notes = get_new_comments(upload.pkg.changes.get("source"))
+ for note in notes:
+ print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+ % (note.author, note.version, note.notedate, note.comment)
+ print "-" * 72
+ return broken, len(notes) > 0
################################################################################
################################################################################
################################################################################
-def edit_new (new):
+def edit_new (new, upload):
# Write the current data to a temporary file
(fd, temp_filename) = utils.temp_filename()
temp_file = os.fdopen(fd, 'w')
- print_new (new, 0, temp_file)
+ print_new (new, upload, indexed=0, file=temp_file)
temp_file.close()
# Spawn an editor on that file
editor = os.environ.get("EDITOR","vi")
if priority.endswith("[!]"):
priority = priority[:-3]
for f in new[pkg]["files"]:
- Upload.pkg.files[f]["section"] = section
- Upload.pkg.files[f]["priority"] = priority
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["priority"] = priority
################################################################################
-def edit_index (new, index):
+def edit_index (new, upload, index):
priority = new[index]["priority"]
section = new[index]["section"]
ftype = new[index]["type"]
readline.set_completer(None)
for f in new[index]["files"]:
- Upload.pkg.files[f]["section"] = section
- Upload.pkg.files[f]["priority"] = priority
+ upload.pkg.files[f]["section"] = section
+ upload.pkg.files[f]["priority"] = priority
new[index]["priority"] = priority
new[index]["section"] = section
return new
################################################################################
-def edit_overrides (new):
+def edit_overrides (new, upload, session):
print
done = 0
while not done:
- print_new (new, 1)
+ print_new (new, upload, indexed=1)
new_index = {}
index = 0
for i in new.keys():
got_answer = 1
if answer == 'E':
- edit_new(new)
+ edit_new(new, upload)
elif answer == 'D':
done = 1
else:
- edit_index (new, new_index[answer])
+ edit_index (new, upload, new_index[answer])
return new
################################################################################
-def edit_note(note):
+def edit_note(note, upload, session):
# Write the current data to a temporary file
(fd, temp_filename) = utils.temp_filename()
editor = os.environ.get("EDITOR","vi")
end()
sys.exit(0)
- database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"]))
+ comment = NewComment()
+ comment.package = upload.pkg.changes["source"]
+ comment.version = upload.pkg.changes["version"]
+ comment.comment = newnote
+ comment.author = utils.whoami()
+ comment.trainee = bool(Options["Trainee"])
+ session.add(comment)
+ session.commit()
################################################################################
-def check_pkg ():
+def check_pkg (upload):
try:
less_fd = os.popen("less -R -", 'w', 0)
stdout_fd = sys.stdout
try:
sys.stdout = less_fd
- changes = utils.parse_changes (Upload.pkg.changes_file)
- examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
- files = Upload.pkg.files
+ changes = utils.parse_changes (upload.pkg.changes_file)
+ examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+ files = upload.pkg.files
for f in files.keys():
if files[f].has_key("new"):
ftype = files[f]["type"]
## FIXME: horribly Debian specific
-def do_bxa_notification():
- files = Upload.pkg.files
+def do_bxa_notification(upload):
+ files = upload.pkg.files
summary = ""
for f in files.keys():
if files[f]["type"] == "deb":
summary += "\n"
summary += "Package: %s\n" % (control.Find("Package"))
summary += "Description: %s\n" % (control.Find("Description"))
- Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
- bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
+ upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
+ bxa_mail = utils.TemplateSubst(upload.Subst,Config()["Dir::Templates"]+"/process-new.bxa_notification")
utils.send_mail(bxa_mail)
################################################################################
-def add_overrides (new):
- changes = Upload.pkg.changes
- files = Upload.pkg.files
+def add_overrides (new, upload, session):
+ changes = upload.pkg.changes
+ files = upload.pkg.files
srcpkg = changes.get("source")
- projectB.query("BEGIN WORK")
for suite in changes["suite"].keys():
- suite_id = database.get_suite_id(suite)
+ suite_id = get_suite(suite).suite_id
for pkg in new.keys():
- component_id = database.get_component_id(new[pkg]["component"])
- type_id = database.get_override_type_id(new[pkg]["type"])
+ component_id = get_component(new[pkg]["component"]).component_id
+ type_id = get_override_type(new[pkg]["type"]).overridetype_id
priority_id = new[pkg]["priority id"]
section_id = new[pkg]["section id"]
Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
- projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
+ session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
+ { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
for f in new[pkg]["files"]:
if files[f].has_key("new"):
del files[f]["new"]
del new[pkg]
- projectB.query("COMMIT WORK")
+ session.commit()
- if Cnf.FindB("Dinstall::BXANotify"):
- do_bxa_notification()
+ if Config().FindB("Dinstall::BXANotify"):
+ do_bxa_notification(upload)
################################################################################
-def prod_maintainer (note):
+def prod_maintainer (note, upload):
+ cnf = Config()
# Here we prepare an editor and get them ready to prod...
(fd, temp_filename) = utils.temp_filename()
temp_file = os.fdopen(fd, 'w')
sys.exit(0)
# Otherwise, do the proding...
user_email_address = utils.whoami() + " <%s>" % (
- Cnf["Dinstall::MyAdminAddress"])
+ cnf["Dinstall::MyAdminAddress"])
- Subst = Upload.Subst
+ Subst = upload.Subst
Subst["__FROM_ADDRESS__"] = user_email_address
Subst["__PROD_MESSAGE__"] = prod_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
+ Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
prod_mail_message = utils.TemplateSubst(
- Subst,Cnf["Dir::Templates"]+"/process-new.prod")
+ Subst,cnf["Dir::Templates"]+"/process-new.prod")
# Send the prod mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
+ if not cnf["Dinstall::Options::No-Mail"]:
utils.send_mail(prod_mail_message)
print "Sent proding message"
################################################################################
-def do_new():
+def do_new(upload, session):
print "NEW\n"
- files = Upload.pkg.files
- changes = Upload.pkg.changes
+ files = upload.pkg.files
+ changes = upload.pkg.changes
+ cnf = Config()
# Make a copy of distribution we can happily trample on
changes["suite"] = copy.copy(changes["distribution"])
# Fix up the list of target suites
for suite in changes["suite"].keys():
- override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
+ override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
if override:
- (olderr, newerr) = (database.get_suite_id(suite) == -1,
- database.get_suite_id(override) == -1)
+ (olderr, newerr) = (get_suite(suite, session) == None,
+ get_suite(override, session) == None)
if olderr or newerr:
(oinv, newinv) = ("", "")
if olderr: oinv = "invalid "
changes["suite"][override] = 1
# Validate suites
for suite in changes["suite"].keys():
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
+ if get_suite(suite, session) is None:
utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))
# The main NEW processing loop
done = 0
while not done:
# Find out what's new
- new = queue.determine_new(changes, files, projectB)
+ new = determine_new(changes, files)
if not new:
break
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
- (broken, note) = print_new(new, 0)
+ (broken, note) = print_new(new, upload, indexed=0)
prompt = ""
if not broken and not note:
answer = m.group(1)
answer = answer[:1].upper()
+ if answer in ( 'A', 'E', 'M', 'O', 'R' ) and Options["Trainee"]:
+ utils.warn("Trainees can't do that")
+ continue
+
if answer == 'A' and not Options["Trainee"]:
try:
check_daily_lock()
- done = add_overrides (new)
- Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)])
+ done = add_overrides (new, upload, session)
+ Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
except CantGetLockError:
print "Hello? Operator! Give me the number for 911!"
print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'C':
- check_pkg()
+ check_pkg(upload)
elif answer == 'E' and not Options["Trainee"]:
- new = edit_overrides (new)
+ new = edit_overrides (new, upload, session)
elif answer == 'M' and not Options["Trainee"]:
- aborted = Upload.do_reject(manual=1,
+ aborted = upload.do_reject(manual=1,
reject_message=Options["Manual-Reject"],
- note=database.get_new_comments(changes.get("source", "")))
+ note=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
- Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)])
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
done = 1
elif answer == 'N':
- edit_note(database.get_new_comments(changes.get("source", "")))
+ edit_note(get_new_comments(changes.get("source", ""), session=session),
+ upload, session)
elif answer == 'P' and not Options["Trainee"]:
- prod_maintainer(database.get_new_comments(changes.get("source", "")))
- Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)])
+ prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
+ upload)
+ Logger.log(["NEW PROD: %s" % (upload.pkg.changes_file)])
elif answer == 'R' and not Options["Trainee"]:
confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
if confirm == "y":
- database.delete_new_comments(changes.get("source"), changes.get("version"))
+ for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session):
+ session.delete(c)
+ session.commit()
elif answer == 'O' and not Options["Trainee"]:
confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
if confirm == "y":
- database.delete_all_new_comments(changes.get("source"))
+ for c in get_new_comments(changes.get("source", ""), session=session):
+ session.delete(c)
+ session.commit()
+
elif answer == 'S':
done = 1
elif answer == 'Q':
################################################################################
-def init():
- global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
-
- Cnf = utils.get_conf()
-
- Arguments = [('a',"automatic","Process-New::Options::Automatic"),
- ('h',"help","Process-New::Options::Help"),
- ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
- ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
- ('t',"trainee","Process-New::Options::Trainee"),
- ('n',"no-action","Process-New::Options::No-Action")]
-
- for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
- if not Cnf.has_key("Process-New::Options::%s" % (i)):
- Cnf["Process-New::Options::%s" % (i)] = ""
-
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""):
- changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"])
-
- Options = Cnf.SubTree("Process-New::Options")
-
- if Options["Help"]:
- usage()
-
- Upload = queue.Upload(Cnf)
-
- if not Options["No-Action"]:
- try:
- Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
- except CantOpenError, e:
- Options["Trainee"] = "True"
-
- projectB = Upload.projectB
-
- Sections = Section_Completer()
- Priorities = Priority_Completer()
- readline.parse_and_bind("tab: complete")
-
- return changes_files
-
-################################################################################
-
-def do_byhand():
+def do_byhand(upload, session):
done = 0
while not done:
- files = Upload.pkg.files
+ files = upload.pkg.files
will_install = 1
byhand = []
done = 1
for f in byhand:
del files[f]
- Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)])
+ Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
except CantGetLockError:
print "Hello? Operator! Give me the number for 911!"
print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'M':
- Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)])
- Upload.do_reject(1, Options["Manual-Reject"])
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
+ upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
done = 1
elif answer == 'S':
done = 1
Raises CantGetLockError if the dinstall daily.lock exists.
"""
+ cnf = Config()
try:
- os.open(Cnf["Process-New::DinstallLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ os.open(cnf["Process-New::DinstallLockFile"],
+ os.O_RDONLY | os.O_CREAT | os.O_EXCL)
except OSError, e:
if e.errno == errno.EEXIST or e.errno == errno.EACCES:
raise CantGetLockError
- os.unlink(Cnf["Process-New::DinstallLockFile"])
+ os.unlink(cnf["Process-New::DinstallLockFile"])
@contextlib.contextmanager
@param package: source package name to lock
"""
- path = os.path.join(Cnf["Process-New::LockDir"], package)
+ path = os.path.join(Config()["Process-New::LockDir"], package)
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
except OSError, e:
finally:
os.unlink(path)
-def move_to_dir (dest, perms=0660, changesperms=0664):
- utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
- file_keys = Upload.pkg.files.keys()
- for f in file_keys:
- utils.move (f, dest, perms=perms)
-
-def is_source_in_queue_dir(qdir):
- entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
- and x.endswith(".changes") ]
- for entry in entries:
- # read the .dak
- u = queue.Upload(Cnf)
- u.pkg.changes_file = os.path.join(qdir, entry)
- u.update_vars()
- if not u.pkg.changes["architecture"].has_key("source"):
- # another binary upload, ignore
- continue
- if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
- # another version, ignore
- continue
- # found it!
- return True
- return False
-
-def move_to_holding(suite, queue_dir):
- print "Moving to %s holding area." % (suite.upper(),)
- if Options["No-Action"]:
- return
- Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
- Upload.dump_vars(queue_dir)
- move_to_dir(queue_dir, perms=0664)
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
-
-def _accept():
+# def move_to_dir (upload, dest, perms=0660, changesperms=0664):
+# utils.move (upload.pkg.changes_file, dest, perms=changesperms)
+# file_keys = upload.pkg.files.keys()
+# for f in file_keys:
+# utils.move (f, dest, perms=perms)
+
+# def is_source_in_queue_dir(qdir):
+# entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
+# and x.endswith(".changes") ]
+# for entry in entries:
+# # read the .dak
+# u = queue.Upload(Cnf)
+# u.pkg.changes_file = os.path.join(qdir, entry)
+# u.update_vars()
+# if not u.pkg.changes["architecture"].has_key("source"):
+# # another binary upload, ignore
+# continue
+# if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
+# # another version, ignore
+# continue
+# # found it!
+# return True
+# return False
+
+# def move_to_holding(suite, queue_dir):
+# print "Moving to %s holding area." % (suite.upper(),)
+# if Options["No-Action"]:
+# return
+# Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
+# Upload.dump_vars(queue_dir)
+# move_to_dir(queue_dir, perms=0664)
+# os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+
+def _accept(upload):
if Options["No-Action"]:
return
- (summary, short_summary) = Upload.build_summaries()
- Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
-
-def do_accept_stableupdate(suite, q):
- queue_dir = Cnf["Dir::Queue::%s" % (q,)]
- if not Upload.pkg.changes["architecture"].has_key("source"):
- # It is not a sourceful upload. So its source may be either in p-u
- # holding, in new, in accepted or already installed.
- if is_source_in_queue_dir(queue_dir):
- # It's in p-u holding, so move it there.
- print "Binary-only upload, source in %s." % (q,)
- move_to_holding(suite, queue_dir)
- elif Upload.source_exists(Upload.pkg.changes["source"],
- Upload.pkg.changes["version"]):
- # dak tells us that there is source available. At time of
- # writing this means that it is installed, so put it into
- # accepted.
- print "Binary-only upload, source installed."
- Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
- _accept()
- elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
- # The source is in accepted, the binary cleared NEW: accept it.
- print "Binary-only upload, source in accepted."
- Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
- _accept()
- elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
- # It's in NEW. We expect the source to land in p-u holding
- # pretty soon.
- print "Binary-only upload, source in new."
- move_to_holding(suite, queue_dir)
- elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
- # It's in newstage. Accept into the holding area
- print "Binary-only upload, source in newstage."
- Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
- _accept()
- else:
- # No case applicable. Bail out. Return will cause the upload
- # to be skipped.
- print "ERROR"
- print "Stable update failed. Source not found."
- return
- else:
- # We are handling a sourceful upload. Move to accepted if currently
- # in p-u holding and to p-u holding otherwise.
- if is_source_in_queue_dir(queue_dir):
- print "Sourceful upload in %s, accepting." % (q,)
- _accept()
- else:
- move_to_holding(suite, queue_dir)
-
-def do_accept():
+ (summary, short_summary) = upload.build_summaries()
+ upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
+ os.unlink(upload.pkg.changes_file[:-8]+".dak")
+
+# def do_accept_stableupdate(upload,suite, q):
+# cnf = Config()
+# queue_dir = cnf["Dir::Queue::%s" % (q,)]
+# if not upload.pkg.changes["architecture"].has_key("source"):
+# # It is not a sourceful upload. So its source may be either in p-u
+# # holding, in new, in accepted or already installed.
+# if is_source_in_queue_dir(queue_dir):
+# # It's in p-u holding, so move it there.
+# print "Binary-only upload, source in %s." % (q,)
+# move_to_holding(suite, queue_dir)
+# elif Upload.source_exists(Upload.pkg.changes["source"],
+# Upload.pkg.changes["version"]):
+# # dak tells us that there is source available. At time of
+# # writing this means that it is installed, so put it into
+# # accepted.
+# print "Binary-only upload, source installed."
+# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
+# _accept()
+# elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
+# # The source is in accepted, the binary cleared NEW: accept it.
+# print "Binary-only upload, source in accepted."
+# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
+# _accept()
+# elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
+# # It's in NEW. We expect the source to land in p-u holding
+# # pretty soon.
+# print "Binary-only upload, source in new."
+# move_to_holding(suite, queue_dir)
+# elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]):
+# # It's in newstage. Accept into the holding area
+# print "Binary-only upload, source in newstage."
+# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)])
+# _accept()
+# else:
+# # No case applicable. Bail out. Return will cause the upload
+# # to be skipped.
+# print "ERROR"
+# print "Stable update failed. Source not found."
+# return
+# else:
+# # We are handling a sourceful upload. Move to accepted if currently
+# # in p-u holding and to p-u holding otherwise.
+# if is_source_in_queue_dir(queue_dir):
+# print "Sourceful upload in %s, accepting." % (q,)
+# _accept()
+# else:
+# move_to_holding(suite, queue_dir)
+
+def do_accept(upload):
print "ACCEPT"
+ cnf = Config()
if not Options["No-Action"]:
- (summary, short_summary) = Upload.build_summaries()
- if Cnf.FindB("Dinstall::SecurityQueueHandling"):
- Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
- move_to_dir(Cnf["Dir::Queue::Embargoed"])
- Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- else:
+ (summary, short_summary) = upload.build_summaries()
+# if cnf.FindB("Dinstall::SecurityQueueHandling"):
+# upload.dump_vars(cnf["Dir::Queue::Embargoed"])
+# move_to_dir(cnf["Dir::Queue::Embargoed"])
+# upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
+# # Check for override disparities
+# upload.Subst["__SUMMARY__"] = summary
+# else:
# Stable updates need to be copied to proposed-updates holding
# area instead of accepted. Sourceful uploads need to go
# to it directly, binaries only if the source has not yet been
# accepted into p-u.
for suite, q in [("proposed-updates", "ProposedUpdates"),
("oldstable-proposed-updates", "OldProposedUpdates")]:
- if not Upload.pkg.changes["distribution"].has_key(suite):
+ if not upload.pkg.changes["distribution"].has_key(suite):
continue
- return do_accept_stableupdate(suite, q)
+ utils.fubar("stable accept not supported yet")
+# return do_accept_stableupdate(suite, q)
# Just a normal upload, accept it...
- _accept()
+ _accept(upload)
def check_status(files):
new = byhand = 0
new = 1
return (new, byhand)
-def do_pkg(changes_file):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
- files = Upload.pkg.files
+def do_pkg(changes_file, session):
+ u = Upload()
+ u.pkg.load_dot_dak(changes_file)
+ u.update_subst()
+
+ cnf = Config()
+ bcc = "X-DAK: dak process-new"
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ else:
+ u.Subst["__BCC__"] = bcc
+
+ files = u.pkg.files
try:
- with lock_package(Upload.pkg.changes["source"]):
- if not recheck():
+ with lock_package(u.pkg.changes["source"]):
+ if not recheck(u, session):
return
(new, byhand) = check_status(files)
if new or byhand:
if new:
- do_new()
+ do_new(u, session)
if byhand:
- do_byhand()
+ do_byhand(u, session)
(new, byhand) = check_status(files)
if not new and not byhand:
try:
check_daily_lock()
- do_accept()
+ do_accept(u)
except CantGetLockError:
print "Hello? Operator! Give me the number for 911!"
print "Dinstall in the locked area, cant process packages, come back later"
################################################################################
def end():
- accept_count = Upload.accept_count
- accept_bytes = Upload.accept_bytes
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
if accept_count:
sets = "set"
if accept_count > 1:
sets = "sets"
sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
- Logger.log([utils.getusername(), "total",accept_count,accept_bytes])
+ Logger.log(["total",accept_count,accept_bytes])
if not Options["No-Action"] and not Options["Trainee"]:
Logger.close()
################################################################################
-def do_comments(dir, opref, npref, line, fn):
- for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
- lines = open("%s/%s" % (dir, comm)).readlines()
- if len(lines) == 0 or lines[0] != line + "\n": continue
- changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
- and x.endswith(".changes") ]
- changes_files = sort_changes(changes_files)
- for f in changes_files:
- f = utils.validate_changes_file_arg(f, 0)
- if not f: continue
- print "\n" + f
- fn(f, "".join(lines[1:]))
-
- if opref != npref and not Options["No-Action"]:
- newcomm = npref + comm[len(opref):]
- os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
+# def do_comments(dir, opref, npref, line, fn):
+# for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
+# lines = open("%s/%s" % (dir, comm)).readlines()
+# if len(lines) == 0 or lines[0] != line + "\n": continue
+# changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
+# and x.endswith(".changes") ]
+# changes_files = sort_changes(changes_files)
+# for f in changes_files:
+# f = utils.validate_changes_file_arg(f, 0)
+# if not f: continue
+# print "\n" + f
+# fn(f, "".join(lines[1:]))
+
+# if opref != npref and not Options["No-Action"]:
+# newcomm = npref + comm[len(opref):]
+# os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
+
+# ################################################################################
+
+# def comment_accept(changes_file, comments):
+# Upload.pkg.changes_file = changes_file
+# Upload.init_vars()
+# Upload.update_vars()
+# Upload.update_subst()
+# files = Upload.pkg.files
+
+# if not recheck():
+# return # dak wants to REJECT, crap
+
+# (new, byhand) = check_status(files)
+# if not new and not byhand:
+# do_accept()
+
+# ################################################################################
+
+# def comment_reject(changes_file, comments):
+# Upload.pkg.changes_file = changes_file
+# Upload.init_vars()
+# Upload.update_vars()
+# Upload.update_subst()
+
+# if not recheck():
+# pass # dak has its own reasons to reject as well, which is fine
+
+# reject(comments)
+# print "REJECT\n" + reject_message,
+# if not Options["No-Action"]:
+# Upload.do_reject(0, reject_message)
+# os.unlink(Upload.pkg.changes_file[:-8]+".dak")
################################################################################
-def comment_accept(changes_file, comments):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
- files = Upload.pkg.files
+def main():
+ global Options, Logger, Sections, Priorities
+
+ cnf = Config()
+ session = DBConn().session()
- if not recheck():
- return # dak wants to REJECT, crap
+ Arguments = [('a',"automatic","Process-New::Options::Automatic"),
+ ('h',"help","Process-New::Options::Help"),
+ ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
+ ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
+ ('t',"trainee","Process-New::Options::Trainee"),
+ ('n',"no-action","Process-New::Options::No-Action")]
- (new, byhand) = check_status(files)
- if not new and not byhand:
- do_accept()
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
+ if not cnf.has_key("Process-New::Options::%s" % (i)):
+ cnf["Process-New::Options::%s" % (i)] = ""
-################################################################################
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
+ if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""):
+ changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
-def comment_reject(changes_file, comments):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- Upload.update_subst()
+ Options = cnf.SubTree("Process-New::Options")
- if not recheck():
- pass # dak has its own reasons to reject as well, which is fine
+ if Options["Help"]:
+ usage()
- reject(comments)
- print "REJECT\n" + reject_message,
if not Options["No-Action"]:
- Upload.do_reject(0, reject_message)
- os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+ try:
+ Logger = daklog.Logger(cnf, "process-new")
+ except CantOpenError, e:
+ Options["Trainee"] = "True"
-################################################################################
+ Sections = Section_Completer(session)
+ Priorities = Priority_Completer(session)
+ readline.parse_and_bind("tab: complete")
-def main():
- changes_files = init()
- if len(changes_files) > 50:
+ if len(changes_files) > 1:
sys.stderr.write("Sorting changes...\n")
- changes_files = sort_changes(changes_files)
+ changes_files = sort_changes(changes_files, session)
# Kill me now? **FIXME**
- Cnf["Dinstall::Options::No-Mail"] = ""
- bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Upload.Subst["__BCC__"] = bcc
-
- commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
- if commentsdir:
- if changes_files != []:
- sys.stderr.write("Can't specify any changes files if working with comments-dir")
- sys.exit(1)
- do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
- do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
- else:
+ cnf["Dinstall::Options::No-Mail"] = ""
+
+# commentsdir = cnf.get("Process-New::Options::Comments-Dir","")
+# if commentsdir:
+# if changes_files != []:
+# sys.stderr.write("Can't specify any changes files if working with comments-dir")
+# sys.exit(1)
+# do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
+# do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
+# else:
+ if True:
for changes_file in changes_files:
changes_file = utils.validate_changes_file_arg(changes_file, 0)
if not changes_file:
continue
print "\n" + changes_file
- do_pkg (changes_file)
+ do_pkg (changes_file, session)
end()
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
@license: GNU General Public License version 2 or later
"""
################################################################################
-import commands
import errno
import fcntl
import os
-import re
-import shutil
-import stat
import sys
-import time
import traceback
-import tarfile
-import apt_inst
import apt_pkg
-from debian_bundle import deb822
-from daklib.dbconn import DBConn
-from daklib.binary import Binary
-from daklib import logging
-from daklib import queue
+
+from daklib.dbconn import *
+from daklib import daklog
+from daklib.queue import *
from daklib import utils
+from daklib.textutils import fix_maintainer
from daklib.dak_exceptions import *
-from daklib.regexes import re_valid_version, re_valid_pkg_name, re_changelog_versions, \
- re_strip_revision, re_strip_srcver, re_spacestrip, \
- re_isanum, re_no_epoch, re_no_revision, re_taint_free, \
- re_isadeb, re_extract_src_version, re_issource, re_default_answer
+from daklib.regexes import re_default_answer
+from daklib.summarystats import SummaryStats
+from daklib.holding import Holding
+from daklib.config import Config
from types import *
################################################################################
# Globals
-Cnf = None
Options = None
Logger = None
-Upload = None
-
-reprocess = 0
-in_holding = {}
-
-# Aliases to the real vars in the Upload class; hysterical raisins.
-reject_message = ""
-changes = {}
-dsc = {}
-dsc_files = {}
-files = {}
-pkg = {}
###############################################################################
def init():
- global Cnf, Options, Upload, changes, dsc, dsc_files, files, pkg
+ global Options
apt_pkg.init()
-
- Cnf = apt_pkg.newConfiguration()
- apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file())
+ cnf = Config()
Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
('h',"help","Dinstall::Options::Help"),
for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
"override-distribution", "version", "directory"]:
- Cnf["Dinstall::Options::%s" % (i)] = ""
+ cnf["Dinstall::Options::%s" % (i)] = ""
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Dinstall::Options")
+ changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Dinstall::Options")
if Options["Help"]:
usage()
# If we have a directory flag, use it to find our files
- if Cnf["Dinstall::Options::Directory"] != "":
+ if cnf["Dinstall::Options::Directory"] != "":
# Note that we clobber the list of files we were given in this case
# so warn if the user has done both
if len(changes_files) > 0:
utils.warn("Directory provided so ignoring files given on command line")
- changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
-
- Upload = queue.Upload(Cnf)
-
- changes = Upload.pkg.changes
- dsc = Upload.pkg.dsc
- dsc_files = Upload.pkg.dsc_files
- files = Upload.pkg.files
- pkg = Upload.pkg
+ changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
return changes_files
################################################################################
-def reject (str, prefix="Rejected: "):
- global reject_message
- if str:
- reject_message += prefix + str + "\n"
-
-################################################################################
-
-def copy_to_holding(filename):
- global in_holding
-
- base_filename = os.path.basename(filename)
-
- dest = Cnf["Dir::Queue::Holding"] + '/' + base_filename
- try:
- fd = os.open(dest, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0640)
- os.close(fd)
- except OSError, e:
- # Shouldn't happen, but will if, for example, someone lists a
- # file twice in the .changes.
- if errno.errorcode[e.errno] == 'EEXIST':
- reject("%s: already exists in holding area; can not overwrite." % (base_filename))
- return
- raise
-
- try:
- shutil.copy(filename, dest)
- except IOError, e:
- # In either case (ENOENT or EACCES) we want to remove the
- # O_CREAT | O_EXCLed ghost file, so add the file to the list
- # of 'in holding' even if it's not the real file.
- if errno.errorcode[e.errno] == 'ENOENT':
- reject("%s: can not copy to holding area: file not found." % (base_filename))
- os.unlink(dest)
- return
- elif errno.errorcode[e.errno] == 'EACCES':
- reject("%s: can not copy to holding area: read permission denied." % (base_filename))
- os.unlink(dest)
- return
- raise
-
- in_holding[base_filename] = ""
-
-################################################################################
-
-def clean_holding():
- global in_holding
-
- cwd = os.getcwd()
- os.chdir(Cnf["Dir::Queue::Holding"])
- for f in in_holding.keys():
- if os.path.exists(f):
- if f.find('/') != -1:
- utils.fubar("WTF? clean_holding() got a file ('%s') with / in it!" % (f))
- else:
- os.unlink(f)
- in_holding = {}
- os.chdir(cwd)
-
-################################################################################
-
-def check_changes():
- filename = pkg.changes_file
-
- # Parse the .changes field into a dictionary
- try:
- changes.update(utils.parse_changes(filename))
- except CantOpenError:
- reject("%s: can't read file." % (filename))
- return 0
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (filename, line))
- return 0
- except ChangesUnicodeError:
- reject("%s: changes file not proper utf-8" % (filename))
- return 0
-
- # Parse the Files field from the .changes into another dictionary
- try:
- files.update(utils.build_file_list(changes))
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (filename, line))
- except UnknownFormatError, format:
- reject("%s: unknown format '%s'." % (filename, format))
- return 0
-
- # Check for mandatory fields
- for i in ("source", "binary", "architecture", "version", "distribution",
- "maintainer", "files", "changes", "description"):
- if not changes.has_key(i):
- reject("%s: Missing mandatory field `%s'." % (filename, i))
- return 0 # Avoid <undef> errors during later tests
-
- # Strip a source version in brackets from the source field
- if re_strip_srcver.search(changes["source"]):
- changes["source"] = re_strip_srcver.sub('', changes["source"])
-
- # Ensure the source field is a valid package name.
- if not re_valid_pkg_name.match(changes["source"]):
- reject("%s: invalid source name '%s'." % (filename, changes["source"]))
-
- # Split multi-value fields into a lower-level dictionary
- for i in ("architecture", "distribution", "binary", "closes"):
- o = changes.get(i, "")
- if o != "":
- del changes[i]
- changes[i] = {}
- for j in o.split():
- changes[i][j] = 1
-
- # Fix the Maintainer: field to be RFC822/2047 compatible
- try:
- (changes["maintainer822"], changes["maintainer2047"],
- changes["maintainername"], changes["maintaineremail"]) = \
- utils.fix_maintainer (changes["maintainer"])
- except ParseMaintError, msg:
- reject("%s: Maintainer field ('%s') failed to parse: %s" \
- % (filename, changes["maintainer"], msg))
-
- # ...likewise for the Changed-By: field if it exists.
- try:
- (changes["changedby822"], changes["changedby2047"],
- changes["changedbyname"], changes["changedbyemail"]) = \
- utils.fix_maintainer (changes.get("changed-by", ""))
- except ParseMaintError, msg:
- (changes["changedby822"], changes["changedby2047"],
- changes["changedbyname"], changes["changedbyemail"]) = \
- ("", "", "", "")
- reject("%s: Changed-By field ('%s') failed to parse: %s" \
- % (filename, changes["changed-by"], msg))
-
- # Ensure all the values in Closes: are numbers
- if changes.has_key("closes"):
- for i in changes["closes"].keys():
- if re_isanum.match (i) == None:
- reject("%s: `%s' from Closes field isn't a number." % (filename, i))
-
-
- # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
- changes["chopversion"] = re_no_epoch.sub('', changes["version"])
- changes["chopversion2"] = re_no_revision.sub('', changes["chopversion"])
-
- # Check there isn't already a changes file of the same name in one
- # of the queue directories.
- base_filename = os.path.basename(filename)
- for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
- if os.path.exists(Cnf["Dir::Queue::%s" % (d) ]+'/'+base_filename):
- reject("%s: a file with this name already exists in the %s directory." % (base_filename, d))
-
- # Check the .changes is non-empty
- if not files:
- reject("%s: nothing to do (Files field is empty)." % (base_filename))
- return 0
-
- return 1
-
-################################################################################
-
-def check_distributions():
- "Check and map the Distribution field of a .changes file."
-
- # Handle suite mappings
- for m in Cnf.ValueList("SuiteMappings"):
- args = m.split()
- mtype = args[0]
- if mtype == "map" or mtype == "silent-map":
- (source, dest) = args[1:3]
- if changes["distribution"].has_key(source):
- del changes["distribution"][source]
- changes["distribution"][dest] = 1
- if mtype != "silent-map":
- reject("Mapping %s to %s." % (source, dest),"")
- if changes.has_key("distribution-version"):
- if changes["distribution-version"].has_key(source):
- changes["distribution-version"][source]=dest
- elif mtype == "map-unreleased":
- (source, dest) = args[1:3]
- if changes["distribution"].has_key(source):
- for arch in changes["architecture"].keys():
- if arch not in DBConn().get_suite_architectures(source):
- reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),"")
- del changes["distribution"][source]
- changes["distribution"][dest] = 1
- break
- elif mtype == "ignore":
- suite = args[1]
- if changes["distribution"].has_key(suite):
- del changes["distribution"][suite]
- reject("Ignoring %s as a target suite." % (suite), "Warning: ")
- elif mtype == "reject":
- suite = args[1]
- if changes["distribution"].has_key(suite):
- reject("Uploads to %s are not accepted." % (suite))
- elif mtype == "propup-version":
- # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
- #
- # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
- if changes["distribution"].has_key(args[1]):
- changes.setdefault("distribution-version", {})
- for suite in args[2:]: changes["distribution-version"][suite]=suite
-
- # Ensure there is (still) a target distribution
- if changes["distribution"].keys() == []:
- reject("no valid distribution.")
-
- # Ensure target distributions exist
- for suite in changes["distribution"].keys():
- if not Cnf.has_key("Suite::%s" % (suite)):
- reject("Unknown distribution `%s'." % (suite))
-
-################################################################################
-
-def check_files():
- global reprocess
-
- archive = utils.where_am_i()
- file_keys = files.keys()
-
- # if reprocess is 2 we've already done this and we're checking
- # things again for the new .orig.tar.gz.
- # [Yes, I'm fully aware of how disgusting this is]
- if not Options["No-Action"] and reprocess < 2:
- cwd = os.getcwd()
- os.chdir(pkg.directory)
- for f in file_keys:
- copy_to_holding(f)
- os.chdir(cwd)
-
- # Check there isn't already a .changes or .dak file of the same name in
- # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
- # [NB: this check must be done post-suite mapping]
- base_filename = os.path.basename(pkg.changes_file)
- dot_dak_filename = base_filename[:-8]+".dak"
- for suite in changes["distribution"].keys():
- copychanges = "Suite::%s::CopyChanges" % (suite)
- if Cnf.has_key(copychanges) and \
- os.path.exists(Cnf[copychanges]+"/"+base_filename):
- reject("%s: a file with this name already exists in %s" \
- % (base_filename, Cnf[copychanges]))
-
- copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
- if Cnf.has_key(copy_dot_dak) and \
- os.path.exists(Cnf[copy_dot_dak]+"/"+dot_dak_filename):
- reject("%s: a file with this name already exists in %s" \
- % (dot_dak_filename, Cnf[copy_dot_dak]))
-
- reprocess = 0
- has_binaries = 0
- has_source = 0
-
- cursor = DBConn().cursor()
- # Check for packages that have moved from one component to another
- # STU: this should probably be changed to not join on architecture, suite tables but instead to used their cached name->id mappings from DBConn
- DBConn().prepare("moved_pkg_q", """
- PREPARE moved_pkg_q(text,text,text) AS
- SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
- component c, architecture a, files f
- WHERE b.package = $1 AND s.suite_name = $2
- AND (a.arch_string = $3 OR a.arch_string = 'all')
- AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
- AND f.location = l.id
- AND l.component = c.id
- AND b.file = f.id""")
-
- for f in file_keys:
- # Ensure the file does not already exist in one of the accepted directories
- for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
- if not Cnf.has_key("Dir::Queue::%s" % (d)): continue
- if os.path.exists(Cnf["Dir::Queue::%s" % (d) ] + '/' + f):
- reject("%s file already exists in the %s directory." % (f, d))
- if not re_taint_free.match(f):
- reject("!!WARNING!! tainted filename: '%s'." % (f))
- # Check the file is readable
- if os.access(f, os.R_OK) == 0:
- # When running in -n, copy_to_holding() won't have
- # generated the reject_message, so we need to.
- if Options["No-Action"]:
- if os.path.exists(f):
- reject("Can't read `%s'. [permission denied]" % (f))
- else:
- reject("Can't read `%s'. [file not found]" % (f))
- files[f]["type"] = "unreadable"
- continue
- # If it's byhand skip remaining checks
- if files[f]["section"] == "byhand" or files[f]["section"][:4] == "raw-":
- files[f]["byhand"] = 1
- files[f]["type"] = "byhand"
- # Checks for a binary package...
- elif re_isadeb.match(f):
- has_binaries = 1
- files[f]["type"] = "deb"
-
- # Extract package control information
- deb_file = utils.open_file(f)
- try:
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
- except:
- reject("%s: debExtractControl() raised %s." % (f, sys.exc_type))
- deb_file.close()
- # Can't continue, none of the checks on control would work.
- continue
-
- # Check for mandantory "Description:"
- deb_file.seek ( 0 )
- try:
- apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
- except:
- reject("%s: Missing Description in binary package" % (f))
- continue
-
- deb_file.close()
-
- # Check for mandatory fields
- for field in [ "Package", "Architecture", "Version" ]:
- if control.Find(field) == None:
- reject("%s: No %s field in control." % (f, field))
- # Can't continue
- continue
-
- # Ensure the package name matches the one give in the .changes
- if not changes["binary"].has_key(control.Find("Package", "")):
- reject("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
-
- # Validate the package field
- package = control.Find("Package")
- if not re_valid_pkg_name.match(package):
- reject("%s: invalid package name '%s'." % (f, package))
-
- # Validate the version field
- version = control.Find("Version")
- if not re_valid_version.match(version):
- reject("%s: invalid version number '%s'." % (f, version))
-
- # Ensure the architecture of the .deb is one we know about.
- default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable")
- architecture = control.Find("Architecture")
- upload_suite = changes["distribution"].keys()[0]
- if architecture not in DBConn().get_suite_architectures(default_suite) and architecture not in DBConn().get_suite_architectures(upload_suite):
- reject("Unknown architecture '%s'." % (architecture))
-
- # Ensure the architecture of the .deb is one of the ones
- # listed in the .changes.
- if not changes["architecture"].has_key(architecture):
- reject("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
-
- # Sanity-check the Depends field
- depends = control.Find("Depends")
- if depends == '':
- reject("%s: Depends field is empty." % (f))
-
- # Sanity-check the Provides field
- provides = control.Find("Provides")
- if provides:
- provide = re_spacestrip.sub('', provides)
- if provide == '':
- reject("%s: Provides field is empty." % (f))
- prov_list = provide.split(",")
- for prov in prov_list:
- if not re_valid_pkg_name.match(prov):
- reject("%s: Invalid Provides field content %s." % (f, prov))
-
-
- # Check the section & priority match those given in the .changes (non-fatal)
- if control.Find("Section") and files[f]["section"] != "" and files[f]["section"] != control.Find("Section"):
- reject("%s control file lists section as `%s', but changes file has `%s'." % (f, control.Find("Section", ""), files[f]["section"]), "Warning: ")
- if control.Find("Priority") and files[f]["priority"] != "" and files[f]["priority"] != control.Find("Priority"):
- reject("%s control file lists priority as `%s', but changes file has `%s'." % (f, control.Find("Priority", ""), files[f]["priority"]),"Warning: ")
-
- files[f]["package"] = package
- files[f]["architecture"] = architecture
- files[f]["version"] = version
- files[f]["maintainer"] = control.Find("Maintainer", "")
- if f.endswith(".udeb"):
- files[f]["dbtype"] = "udeb"
- elif f.endswith(".deb"):
- files[f]["dbtype"] = "deb"
- else:
- reject("%s is neither a .deb or a .udeb." % (f))
- files[f]["source"] = control.Find("Source", files[f]["package"])
- # Get the source version
- source = files[f]["source"]
- source_version = ""
- if source.find("(") != -1:
- m = re_extract_src_version.match(source)
- source = m.group(1)
- source_version = m.group(2)
- if not source_version:
- source_version = files[f]["version"]
- files[f]["source package"] = source
- files[f]["source version"] = source_version
-
- # Ensure the filename matches the contents of the .deb
- m = re_isadeb.match(f)
- # package name
- file_package = m.group(1)
- if files[f]["package"] != file_package:
- reject("%s: package part of filename (%s) does not match package name in the %s (%s)." % (f, file_package, files[f]["dbtype"], files[f]["package"]))
- epochless_version = re_no_epoch.sub('', control.Find("Version"))
- # version
- file_version = m.group(2)
- if epochless_version != file_version:
- reject("%s: version part of filename (%s) does not match package version in the %s (%s)." % (f, file_version, files[f]["dbtype"], epochless_version))
- # architecture
- file_architecture = m.group(3)
- if files[f]["architecture"] != file_architecture:
- reject("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % (f, file_architecture, files[f]["dbtype"], files[f]["architecture"]))
-
- # Check for existent source
- source_version = files[f]["source version"]
- source_package = files[f]["source package"]
- if changes["architecture"].has_key("source"):
- if source_version != changes["version"]:
- reject("source version (%s) for %s doesn't match changes version %s." % (source_version, f, changes["version"]))
- else:
- # Check in the SQL database
- if not Upload.source_exists(source_package, source_version, changes["distribution"].keys()):
- # Check in one of the other directories
- source_epochless_version = re_no_epoch.sub('', source_version)
- dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if os.path.exists(Cnf["Dir::Queue::Byhand"] + '/' + dsc_filename):
- files[f]["byhand"] = 1
- elif os.path.exists(Cnf["Dir::Queue::New"] + '/' + dsc_filename):
- files[f]["new"] = 1
- else:
- dsc_file_exists = 0
- for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
- if Cnf.has_key("Dir::Queue::%s" % (myq)):
- if os.path.exists(Cnf["Dir::Queue::"+myq] + '/' + dsc_filename):
- dsc_file_exists = 1
- break
- if not dsc_file_exists:
- reject("no source found for %s %s (%s)." % (source_package, source_version, f))
- # Check the version and for file overwrites
- reject(Upload.check_binary_against_db(f),"")
-
- Binary(f, reject).scan_package()
-
- # Checks for a source package...
- else:
- m = re_issource.match(f)
- if m:
- has_source = 1
- files[f]["package"] = m.group(1)
- files[f]["version"] = m.group(2)
- files[f]["type"] = m.group(3)
-
- # Ensure the source package name matches the Source filed in the .changes
- if changes["source"] != files[f]["package"]:
- reject("%s: changes file doesn't say %s for Source" % (f, files[f]["package"]))
-
- # Ensure the source version matches the version in the .changes file
- if files[f]["type"] == "orig.tar.gz":
- changes_version = changes["chopversion2"]
- else:
- changes_version = changes["chopversion"]
- if changes_version != files[f]["version"]:
- reject("%s: should be %s according to changes file." % (f, changes_version))
-
- # Ensure the .changes lists source in the Architecture field
- if not changes["architecture"].has_key("source"):
- reject("%s: changes file doesn't list `source' in Architecture field." % (f))
-
- # Check the signature of a .dsc file
- if files[f]["type"] == "dsc":
- dsc["fingerprint"] = utils.check_signature(f, reject)
-
- files[f]["architecture"] = "source"
-
- # Not a binary or source package? Assume byhand...
- else:
- files[f]["byhand"] = 1
- files[f]["type"] = "byhand"
-
- # Per-suite file checks
- files[f]["oldfiles"] = {}
- for suite in changes["distribution"].keys():
- # Skip byhand
- if files[f].has_key("byhand"):
- continue
-
- # Handle component mappings
- for m in Cnf.ValueList("ComponentMappings"):
- (source, dest) = m.split()
- if files[f]["component"] == source:
- files[f]["original component"] = source
- files[f]["component"] = dest
-
- # Ensure the component is valid for the target suite
- if Cnf.has_key("Suite:%s::Components" % (suite)) and \
- files[f]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)):
- reject("unknown component `%s' for suite `%s'." % (files[f]["component"], suite))
- continue
-
- # Validate the component
- component = files[f]["component"]
- component_id = DBConn().get_component_id(component)
- if component_id == -1:
- reject("file '%s' has unknown component '%s'." % (f, component))
- continue
-
- # See if the package is NEW
- if not Upload.in_override_p(files[f]["package"], files[f]["component"], suite, files[f].get("dbtype",""), f):
- files[f]["new"] = 1
-
- # Validate the priority
- if files[f]["priority"].find('/') != -1:
- reject("file '%s' has invalid priority '%s' [contains '/']." % (f, files[f]["priority"]))
-
- # Determine the location
- location = Cnf["Dir::Pool"]
- location_id = DBConn().get_location_id(location, component, archive)
- if location_id == -1:
- reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
- files[f]["location id"] = location_id
-
- # Check the md5sum & size against existing files (if any)
- files[f]["pool name"] = utils.poolify (changes["source"], files[f]["component"])
- files_id = DBConn().get_files_id(files[f]["pool name"] + f, files[f]["size"], files[f]["md5sum"], files[f]["location id"])
- if files_id == -1:
- reject("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
- elif files_id == -2:
- reject("md5sum and/or size mismatch on existing copy of %s." % (f))
- files[f]["files id"] = files_id
-
- # Check for packages that have moved from one component to another
- files[f]['suite'] = suite
- cursor.execute("""EXECUTE moved_pkg_q( %(package)s, %(suite)s, %(architecture)s )""", ( files[f] ) )
- ql = cursor.fetchone()
- if ql:
- files[f]["othercomponents"] = ql[0][0]
-
- # If the .changes file says it has source, it must have source.
- if changes["architecture"].has_key("source"):
- if not has_source:
- reject("no source found and Architecture line in changes mention source.")
-
- if not has_binaries and Cnf.FindB("Dinstall::Reject::NoSourceOnly"):
- reject("source only uploads are not supported.")
-
-###############################################################################
-
-def check_dsc():
- global reprocess
-
- # Ensure there is source to check
- if not changes["architecture"].has_key("source"):
- return 1
-
- # Find the .dsc
- dsc_filename = None
- for f in files.keys():
- if files[f]["type"] == "dsc":
- if dsc_filename:
- reject("can not process a .changes file with multiple .dsc's.")
- return 0
- else:
- dsc_filename = f
-
- # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
- if not dsc_filename:
- reject("source uploads must contain a dsc file")
- return 0
-
- # Parse the .dsc file
- try:
- dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
- except CantOpenError:
- # if not -n copy_to_holding() will have done this for us...
- if Options["No-Action"]:
- reject("%s: can't read file." % (dsc_filename))
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (dsc_filename, line))
- except InvalidDscError, line:
- reject("%s: syntax error on line %s." % (dsc_filename, line))
- except ChangesUnicodeError:
- reject("%s: dsc file not proper utf-8." % (dsc_filename))
-
- # Build up the file list of files mentioned by the .dsc
- try:
- dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1))
- except NoFilesFieldError:
- reject("%s: no Files: field." % (dsc_filename))
- return 0
- except UnknownFormatError, format:
- reject("%s: unknown format '%s'." % (dsc_filename, format))
- return 0
- except ParseChangesError, line:
- reject("%s: parse error, can't grok: %s." % (dsc_filename, line))
- return 0
-
- # Enforce mandatory fields
- for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
- if not dsc.has_key(i):
- reject("%s: missing mandatory field `%s'." % (dsc_filename, i))
- return 0
-
- # Validate the source and version fields
- if not re_valid_pkg_name.match(dsc["source"]):
- reject("%s: invalid source name '%s'." % (dsc_filename, dsc["source"]))
- if not re_valid_version.match(dsc["version"]):
- reject("%s: invalid version number '%s'." % (dsc_filename, dsc["version"]))
-
- # Bumping the version number of the .dsc breaks extraction by stable's
- # dpkg-source. So let's not do that...
- if dsc["format"] != "1.0":
- reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename))
-
- # Validate the Maintainer field
- try:
- utils.fix_maintainer (dsc["maintainer"])
- except ParseMaintError, msg:
- reject("%s: Maintainer field ('%s') failed to parse: %s" \
- % (dsc_filename, dsc["maintainer"], msg))
-
- # Validate the build-depends field(s)
- for field_name in [ "build-depends", "build-depends-indep" ]:
- field = dsc.get(field_name)
- if field:
- # Check for broken dpkg-dev lossage...
- if field.startswith("ARRAY"):
- reject("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % (dsc_filename, field_name.title()))
-
- # Have apt try to parse them...
- try:
- apt_pkg.ParseSrcDepends(field)
- except:
- reject("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
- pass
-
- # Ensure the version number in the .dsc matches the version number in the .changes
- epochless_dsc_version = re_no_epoch.sub('', dsc["version"])
- changes_version = files[dsc_filename]["version"]
- if epochless_dsc_version != files[dsc_filename]["version"]:
- reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
-
- # Ensure there is a .tar.gz in the .dsc file
- has_tar = 0
- for f in dsc_files.keys():
- m = re_issource.match(f)
- if not m:
- reject("%s: %s in Files field not recognised as source." % (dsc_filename, f))
- continue
- ftype = m.group(3)
- if ftype == "orig.tar.gz" or ftype == "tar.gz":
- has_tar = 1
- if not has_tar:
- reject("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
-
- # Ensure source is newer than existing source in target suites
- reject(Upload.check_source_against_db(dsc_filename),"")
-
- (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(dsc_filename)
- reject(reject_msg, "")
- if is_in_incoming:
- if not Options["No-Action"]:
- copy_to_holding(is_in_incoming)
- orig_tar_gz = os.path.basename(is_in_incoming)
- files[orig_tar_gz] = {}
- files[orig_tar_gz]["size"] = os.stat(orig_tar_gz)[stat.ST_SIZE]
- files[orig_tar_gz]["md5sum"] = dsc_files[orig_tar_gz]["md5sum"]
- files[orig_tar_gz]["sha1sum"] = dsc_files[orig_tar_gz]["sha1sum"]
- files[orig_tar_gz]["sha256sum"] = dsc_files[orig_tar_gz]["sha256sum"]
- files[orig_tar_gz]["section"] = files[dsc_filename]["section"]
- files[orig_tar_gz]["priority"] = files[dsc_filename]["priority"]
- files[orig_tar_gz]["component"] = files[dsc_filename]["component"]
- files[orig_tar_gz]["type"] = "orig.tar.gz"
- reprocess = 2
-
- return 1
-
-################################################################################
-
-def get_changelog_versions(source_dir):
- """Extracts a the source package and (optionally) grabs the
- version history out of debian/changelog for the BTS."""
-
- # Find the .dsc (again)
- dsc_filename = None
- for f in files.keys():
- if files[f]["type"] == "dsc":
- dsc_filename = f
-
- # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
- if not dsc_filename:
- return
-
- # Create a symlink mirror of the source files in our temporary directory
- for f in files.keys():
- m = re_issource.match(f)
- if m:
- src = os.path.join(source_dir, f)
- # If a file is missing for whatever reason, give up.
- if not os.path.exists(src):
- return
- ftype = m.group(3)
- if ftype == "orig.tar.gz" and pkg.orig_tar_gz:
- continue
- dest = os.path.join(os.getcwd(), f)
- os.symlink(src, dest)
-
- # If the orig.tar.gz is not a part of the upload, create a symlink to the
- # existing copy.
- if pkg.orig_tar_gz:
- dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz))
- os.symlink(pkg.orig_tar_gz, dest)
-
- # Extract the source
- cmd = "dpkg-source -sn -x %s" % (dsc_filename)
- (result, output) = commands.getstatusoutput(cmd)
- if (result != 0):
- reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
- reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "")
- return
-
- if not Cnf.Find("Dir::Queue::BTSVersionTrack"):
- return
-
- # Get the upstream version
- upstr_version = re_no_epoch.sub('', dsc["version"])
- if re_strip_revision.search(upstr_version):
- upstr_version = re_strip_revision.sub('', upstr_version)
-
- # Ensure the changelog file exists
- changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version)
- if not os.path.exists(changelog_filename):
- reject("%s: debian/changelog not found in extracted source." % (dsc_filename))
- return
-
- # Parse the changelog
- dsc["bts changelog"] = ""
- changelog_file = utils.open_file(changelog_filename)
- for line in changelog_file.readlines():
- m = re_changelog_versions.match(line)
- if m:
- dsc["bts changelog"] += line
- changelog_file.close()
-
- # Check we found at least one revision in the changelog
- if not dsc["bts changelog"]:
- reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
-
-########################################
-
-def check_source():
- # Bail out if:
- # a) there's no source
- # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
- # or c) the orig.tar.gz is MIA
- if not changes["architecture"].has_key("source") or reprocess == 2 \
- or pkg.orig_tar_gz == -1:
- return
-
- tmpdir = utils.temp_dirname()
-
- # Move into the temporary directory
- cwd = os.getcwd()
- os.chdir(tmpdir)
-
- # Get the changelog version history
- get_changelog_versions(cwd)
-
- # Move back and cleanup the temporary tree
- os.chdir(cwd)
- try:
- shutil.rmtree(tmpdir)
- except OSError, e:
- if errno.errorcode[e.errno] != 'EACCES':
- utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"]))
-
- reject("%s: source tree could not be cleanly removed." % (dsc["source"]))
- # We probably have u-r or u-w directories so chmod everything
- # and try again.
- cmd = "chmod -R u+rwx %s" % (tmpdir)
- result = os.system(cmd)
- if result != 0:
- utils.fubar("'%s' failed with result %s." % (cmd, result))
- shutil.rmtree(tmpdir)
- except:
- utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"]))
-
-################################################################################
-
-# FIXME: should be a debian specific check called from a hook
-
-def check_urgency ():
- if changes["architecture"].has_key("source"):
- if not changes.has_key("urgency"):
- changes["urgency"] = Cnf["Urgency::Default"]
- # Urgency may be followed by space & comment (policy 5.6.17)
- changes["urgency"] = changes["urgency"].split(" ")[0].lower();
- if changes["urgency"] not in Cnf.ValueList("Urgency::Valid"):
- reject("%s is not a valid urgency; it will be treated as %s by testing." % (changes["urgency"], Cnf["Urgency::Default"]), "Warning: ")
- changes["urgency"] = Cnf["Urgency::Default"]
-
-################################################################################
-
-def check_hashes ():
- utils.check_hash(".changes", files, "md5", apt_pkg.md5sum)
- utils.check_size(".changes", files)
- utils.check_hash(".dsc", dsc_files, "md5", apt_pkg.md5sum)
- utils.check_size(".dsc", dsc_files)
-
- # This is stupid API, but it'll have to do for now until
- # we actually have proper abstraction
- for m in utils.ensure_hashes(changes, dsc, files, dsc_files):
- reject(m)
-
-################################################################################
-
-# Sanity check the time stamps of files inside debs.
-# [Files in the near future cause ugly warnings and extreme time
-# travel can cause errors on extraction]
-
-def check_timestamps():
- class Tar:
- def __init__(self, future_cutoff, past_cutoff):
- self.reset()
- self.future_cutoff = future_cutoff
- self.past_cutoff = past_cutoff
-
- def reset(self):
- self.future_files = {}
- self.ancient_files = {}
-
- def callback(self, Kind,Name,Link,Mode,UID,GID,Size,MTime,Major,Minor):
- if MTime > self.future_cutoff:
- self.future_files[Name] = MTime
- if MTime < self.past_cutoff:
- self.ancient_files[Name] = MTime
- ####
-
- future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
- past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
- tar = Tar(future_cutoff, past_cutoff)
- for filename in files.keys():
- if files[filename]["type"] == "deb":
- tar.reset()
- try:
- deb_file = utils.open_file(filename)
- apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz")
- deb_file.seek(0)
- try:
- apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz")
- except SystemError, e:
- # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
- if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
- raise
- deb_file.seek(0)
- apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
- deb_file.close()
- #
- future_files = tar.future_files.keys()
- if future_files:
- num_future_files = len(future_files)
- future_file = future_files[0]
- future_date = tar.future_files[future_file]
- reject("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
- % (filename, num_future_files, future_file,
- time.ctime(future_date)))
- #
- ancient_files = tar.ancient_files.keys()
- if ancient_files:
- num_ancient_files = len(ancient_files)
- ancient_file = ancient_files[0]
- ancient_date = tar.ancient_files[ancient_file]
- reject("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
- % (filename, num_ancient_files, ancient_file,
- time.ctime(ancient_date)))
- except:
- reject("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
-
-################################################################################
-
-def lookup_uid_from_fingerprint(fpr):
- """
- Return the uid,name,isdm for a given gpg fingerprint
+def action(u):
+ cnf = Config()
- @type fpr: string
- @param fpr: a 40 byte GPG fingerprint
-
- @return: (uid, name, isdm)
- """
- cursor = DBConn().cursor()
- cursor.execute( "SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
- qs = cursor.fetchone()
- if qs:
- return qs
- else:
- return (None, None, False)
-
-def check_signed_by_key():
- """Ensure the .changes is signed by an authorized uploader."""
-
- (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(changes["fingerprint"])
- if uid_name == None:
- uid_name = ""
-
- # match claimed name with actual name:
- if uid is None:
- # This is fundamentally broken but need us to refactor how we get
- # the UIDs/Fingerprints in order for us to fix it properly
- uid, uid_email = changes["fingerprint"], uid
- may_nmu, may_sponsor = 1, 1
- # XXX by default new dds don't have a fingerprint/uid in the db atm,
- # and can't get one in there if we don't allow nmu/sponsorship
- elif is_dm is False:
- # If is_dm is False, we allow full upload rights
- uid_email = "%s@debian.org" % (uid)
- may_nmu, may_sponsor = 1, 1
- else:
- # Assume limited upload rights unless we've discovered otherwise
- uid_email = uid
- may_nmu, may_sponsor = 0, 0
-
-
- if uid_email in [changes["maintaineremail"], changes["changedbyemail"]]:
- sponsored = 0
- elif uid_name in [changes["maintainername"], changes["changedbyname"]]:
- sponsored = 0
- if uid_name == "": sponsored = 1
- else:
- sponsored = 1
- if ("source" in changes["architecture"] and
- uid_email and utils.is_email_alias(uid_email)):
- sponsor_addresses = utils.gpg_get_key_addresses(changes["fingerprint"])
- if (changes["maintaineremail"] not in sponsor_addresses and
- changes["changedbyemail"] not in sponsor_addresses):
- changes["sponsoremail"] = uid_email
-
- if sponsored and not may_sponsor:
- reject("%s is not authorised to sponsor uploads" % (uid))
-
- cursor = DBConn().cursor()
- if not sponsored and not may_nmu:
- source_ids = []
- cursor.execute( "SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = %(source)s AND s.dm_upload_allowed = 'yes'", changes )
-
- highest_sid, highest_version = None, None
-
- should_reject = True
- while True:
- si = cursor.fetchone()
- if not si:
- break
-
- if highest_version == None or apt_pkg.VersionCompare(si[1], highest_version) == 1:
- highest_sid = si[0]
- highest_version = si[1]
-
- if highest_sid == None:
- reject("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % changes["source"])
- else:
-
- cursor.execute("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT su.maintainer FROM src_uploaders su JOIN source s ON (s.id = su.source) WHERE su.source = %s)" % (highest_sid))
-
- while True:
- m = cursor.fetchone()
- if not m:
- break
-
- (rfc822, rfc2047, name, email) = utils.fix_maintainer(m[0])
- if email == uid_email or name == uid_name:
- should_reject=False
- break
-
- if should_reject == True:
- reject("%s is not in Maintainer or Uploaders of source package %s" % (uid, changes["source"]))
-
- for b in changes["binary"].keys():
- for suite in changes["distribution"].keys():
- suite_id = DBConn().get_suite_id(suite)
-
- cursor.execute("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = %(package)s AND ba.suite = %(suite)s" , {'package':b, 'suite':suite_id} )
- while True:
- s = cursor.fetchone()
- if not s:
- break
-
- if s[0] != changes["source"]:
- reject("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
-
- for f in files.keys():
- if files[f].has_key("byhand"):
- reject("%s may not upload BYHAND file %s" % (uid, f))
- if files[f].has_key("new"):
- reject("%s may not upload NEW file %s" % (uid, f))
-
-
-################################################################################
-################################################################################
-
-# If any file of an upload has a recent mtime then chances are good
-# the file is still being uploaded.
-
-def upload_too_new():
- too_new = 0
- # Move back to the original directory to get accurate time stamps
- cwd = os.getcwd()
- os.chdir(pkg.directory)
- file_list = pkg.files.keys()
- file_list.extend(pkg.dsc_files.keys())
- file_list.append(pkg.changes_file)
- for f in file_list:
- try:
- last_modified = time.time()-os.path.getmtime(f)
- if last_modified < int(Cnf["Dinstall::SkipTime"]):
- too_new = 1
- break
- except:
- pass
- os.chdir(cwd)
- return too_new
-
-################################################################################
-
-def action ():
# changes["distribution"] may not exist in corner cases
# (e.g. unreadable changes files)
- if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
- changes["distribution"] = {}
+ if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType):
+ u.pkg.changes["distribution"] = {}
- (summary, short_summary) = Upload.build_summaries()
+ (summary, short_summary) = u.build_summaries()
# q-unapproved hax0ring
queue_info = {
"Autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand },
"Byhand" : { "is": is_byhand, "process": do_byhand },
"OldStableUpdate" : { "is": is_oldstableupdate,
- "process": do_oldstableupdate },
+ "process": do_oldstableupdate },
"StableUpdate" : { "is": is_stableupdate, "process": do_stableupdate },
"Unembargo" : { "is": is_unembargo, "process": queue_unembargo },
"Embargo" : { "is": is_embargo, "process": queue_embargo },
}
+
queues = [ "New", "Autobyhand", "Byhand" ]
- if Cnf.FindB("Dinstall::SecurityQueueHandling"):
+ if cnf.FindB("Dinstall::SecurityQueueHandling"):
queues += [ "Unembargo", "Embargo" ]
else:
queues += [ "OldStableUpdate", "StableUpdate" ]
queuekey = ''
- if reject_message.find("Rejected") != -1:
- if upload_too_new():
- print "SKIP (too new)\n" + reject_message,
+ pi = u.package_info()
+
+ if len(u.rejects) > 0:
+ if u.upload_too_new():
+ print "SKIP (too new)\n" + pi,
prompt = "[S]kip, Quit ?"
else:
- print "REJECT\n" + reject_message,
+ print "REJECT\n" + pi
prompt = "[R]eject, Skip, Quit ?"
if Options["Automatic"]:
answer = 'R'
else:
qu = None
for q in queues:
- if queue_info[q]["is"]():
+ if queue_info[q]["is"](u):
qu = q
break
if qu:
- print "%s for %s\n%s%s" % (
- qu.upper(), ", ".join(changes["distribution"].keys()),
- reject_message, summary),
+ print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary)
queuekey = qu[0].upper()
if queuekey in "RQSA":
queuekey = "D"
if Options["Automatic"]:
answer = queuekey
else:
- print "ACCEPT\n" + reject_message + summary,
+ print "ACCEPT\n" + pi + summary,
prompt = "[A]ccept, Skip, Quit ?"
if Options["Automatic"]:
answer = 'A'
answer = answer[:1].upper()
if answer == 'R':
- os.chdir (pkg.directory)
- Upload.do_reject(0, reject_message)
+ os.chdir(u.pkg.directory)
+ u.do_reject(0, pi)
elif answer == 'A':
- accept(summary, short_summary)
- remove_from_unchecked()
+ u.accept(summary, short_summary)
+ u.check_override()
+ u.remove()
elif answer == queuekey:
- queue_info[qu]["process"](summary, short_summary)
- remove_from_unchecked()
+ queue_info[qu]["process"](u, summary, short_summary)
+ u.remove()
elif answer == 'Q':
sys.exit(0)
-def remove_from_unchecked():
- os.chdir (pkg.directory)
- for f in files.keys():
- os.unlink(f)
- os.unlink(pkg.changes_file)
-
################################################################################
-def accept (summary, short_summary):
- Upload.accept(summary, short_summary)
- Upload.check_override()
+def package_to_suite(u, suite):
+ if not u.pkg.changes["distribution"].has_key(suite):
+ return False
-################################################################################
+ ret = True
-def move_to_dir (dest, perms=0660, changesperms=0664):
- utils.move (pkg.changes_file, dest, perms=changesperms)
- file_keys = files.keys()
- for f in file_keys:
- utils.move (f, dest, perms=perms)
+ if not u.pkg.changes["architecture"].has_key("source"):
+ s = DBConn().session()
+ q = s.query(SrcAssociation.sa_id)
+ q = q.join(Suite).filter_by(suite_name=suite)
+ q = q.join(DBSource).filter_by(source=u.pkg.changes['source'])
+ q = q.filter_by(version=u.pkg.changes['version']).limit(1)
-################################################################################
-
-def is_unembargo ():
- cursor = DBConn().cursor()
- cursor.execute( "SELECT package FROM disembargo WHERE package = %(source)s AND version = %(version)s", changes )
- if cursor.fetchone():
- return 1
-
- oldcwd = os.getcwd()
- os.chdir(Cnf["Dir::Queue::Disembargo"])
- disdir = os.getcwd()
- os.chdir(oldcwd)
+ # NB: Careful, this logic isn't what you would think it is
+ # Source is already in {old-,}proposed-updates so no need to hold
+ # Instead, we don't move to the holding area, we just do an ACCEPT
+ if q.count() > 0:
+ ret = False
- if pkg.directory == disdir:
- if changes["architecture"].has_key("source"):
- if Options["No-Action"]: return 1
+ s.close()
- cursor.execute( "INSERT INTO disembargo (package, version) VALUES ('%(package)s', '%(version)s')",
- changes )
- cursor.execute( "COMMIT" )
- return 1
+ return ret
- return 0
+def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, announce=None):
+ cnf = Config()
+ dir = cnf["Dir::Queue::%s" % queue]
-def queue_unembargo (summary, short_summary):
- print "Moving to UNEMBARGOED holding area."
- Logger.log(["Moving to unembargoed", pkg.changes_file])
+ print "Moving to %s holding area" % queue.upper()
+ Logger.log(["Moving to %s" % queue, u.pkg.changes_file])
- Upload.dump_vars(Cnf["Dir::Queue::Unembargoed"])
- move_to_dir(Cnf["Dir::Queue::Unembargoed"])
- Upload.queue_build("unembargoed", Cnf["Dir::Queue::Unembargoed"])
+ u.pkg.write_dot_dak(dir)
+ u.move_to_dir(dir, perms=perms)
+ if build:
+ get_or_set_queue(queue.lower()).autobuild_upload(u.pkg, dir)
# Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
-
- # Send accept mail, announce to lists, close bugs and check for
- # override disparities
- if not Cnf["Dinstall::Options::No-Mail"]:
- Upload.Subst["__SUITE__"] = ""
- mail_message = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
+ u.check_override()
+
+ # Send accept mail, announce to lists and close bugs
+ if announce and not cnf["Dinstall::Options::No-Mail"]:
+ template = os.path.join(cnf["Dir::Templates"], announce)
+ u.update_subst()
+ u.Subst["__SUITE__"] = ""
+ mail_message = utils.TemplateSubst(u.Subst, template)
utils.send_mail(mail_message)
- Upload.announce(short_summary, 1)
+ u.announce(short_summary, True)
################################################################################
-def is_embargo ():
- # if embargoed queues are enabled always embargo
- return 1
+def is_unembargo(u):
+ session = DBConn().session()
+ cnf = Config()
-def queue_embargo (summary, short_summary):
- print "Moving to EMBARGOED holding area."
- Logger.log(["Moving to embargoed", pkg.changes_file])
+ q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes)
+ if q.rowcount > 0:
+ session.close()
+ return True
- Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
- move_to_dir(Cnf["Dir::Queue::Embargoed"])
- Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
+ oldcwd = os.getcwd()
+ os.chdir(cnf["Dir::Queue::Disembargo"])
+ disdir = os.getcwd()
+ os.chdir(oldcwd)
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
-
- # Send accept mail, announce to lists, close bugs and check for
- # override disparities
- if not Cnf["Dinstall::Options::No-Mail"]:
- Upload.Subst["__SUITE__"] = ""
- mail_message = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
- utils.send_mail(mail_message)
- Upload.announce(short_summary, 1)
+ ret = False
-################################################################################
+ if u.pkg.directory == disdir:
+ if u.pkg.changes["architecture"].has_key("source"):
+ if not Options["No-Action"]:
+ session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes)
+ session.commit()
-def is_stableupdate ():
- if not changes["distribution"].has_key("proposed-updates"):
- return 0
+ ret = True
- if not changes["architecture"].has_key("source"):
- pusuite = DBConn().get_suite_id("proposed-updates")
- cursor = DBConn().cursor()
- cursor.execute( """SELECT 1 FROM source s
- JOIN src_associations sa ON (s.id = sa.source)
- WHERE s.source = %(source)s
- AND s.version = %(version)s
- AND sa.suite = %(suite)s""",
- {'source' : changes['source'],
- 'version' : changes['version'],
- 'suite' : pusuite})
+ session.close()
- if cursor.fetchone():
- # source is already in proposed-updates so no need to hold
- return 0
+ return ret
- return 1
+def queue_unembargo(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "Unembargoed",
+ perms=0660, build=True, announce='process-unchecked.accepted')
-def do_stableupdate (summary, short_summary):
- print "Moving to PROPOSED-UPDATES holding area."
- Logger.log(["Moving to proposed-updates", pkg.changes_file])
+################################################################################
- Upload.dump_vars(Cnf["Dir::Queue::ProposedUpdates"])
- move_to_dir(Cnf["Dir::Queue::ProposedUpdates"], perms=0664)
+def is_embargo(u):
+ # if embargoed queues are enabled always embargo
+ return True
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
+def queue_embargo(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "Unembargoed",
+ perms=0660, build=True, announce='process-unchecked.accepted')
################################################################################
-def is_oldstableupdate ():
- if not changes["distribution"].has_key("oldstable-proposed-updates"):
- return 0
-
- if not changes["architecture"].has_key("source"):
- pusuite = DBConn().get_suite_id("oldstable-proposed-updates")
- cursor = DBConn().cursor()
- cursor.execute( """SELECT 1 FROM source s
- JOIN src_associations sa ON (s.id = sa.source)
- WHERE s.source = %(source)s
- AND s.version = %(version)s
- AND sa.suite = %(suite)s""",
- {'source' : changes['source'],
- 'version' : changes['version'],
- 'suite' : pusuite})
- if cursor.fetchone():
- return 0
-
- return 1
-
-def do_oldstableupdate (summary, short_summary):
- print "Moving to OLDSTABLE-PROPOSED-UPDATES holding area."
- Logger.log(["Moving to oldstable-proposed-updates", pkg.changes_file])
-
- Upload.dump_vars(Cnf["Dir::Queue::OldProposedUpdates"])
- move_to_dir(Cnf["Dir::Queue::OldProposedUpdates"], perms=0664)
+def is_stableupdate(u):
+ return package_to_suite(u, 'proposed-updates')
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
+def do_stableupdate(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "ProposedUpdates",
+ perms=0664, build=False, announce=None)
################################################################################
-def is_autobyhand ():
+def is_oldstableupdate(u):
+ return package_to_suite(u, 'oldstable-proposed-updates')
+
+def do_oldstableupdate(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "OldProposedUpdates",
+ perms=0664, build=False, announce=None)
+
+################################################################################
+
+def is_autobyhand(u):
+ cnf = Config()
+
all_auto = 1
any_auto = 0
- for f in files.keys():
- if files[f].has_key("byhand"):
+ for f in u.pkg.files.keys():
+ if u.pkg.files[f].has_key("byhand"):
any_auto = 1
# filename is of form "PKG_VER_ARCH.EXT" where PKG, VER and ARCH
continue
(pckg, ver, archext) = f.split("_", 2)
- if archext.count(".") < 1 or changes["version"] != ver:
+ if archext.count(".") < 1 or u.pkg.changes["version"] != ver:
all_auto = 0
continue
- ABH = Cnf.SubTree("AutomaticByHandPackages")
+ ABH = cnf.SubTree("AutomaticByHandPackages")
if not ABH.has_key(pckg) or \
- ABH["%s::Source" % (pckg)] != changes["source"]:
- print "not match %s %s" % (pckg, changes["source"])
+ ABH["%s::Source" % (pckg)] != u.pkg.changes["source"]:
+ print "not match %s %s" % (pckg, u.pkg.changes["source"])
all_auto = 0
continue
(arch, ext) = archext.split(".", 1)
- if arch not in changes["architecture"]:
+ if arch not in u.pkg.changes["architecture"]:
all_auto = 0
continue
- files[f]["byhand-arch"] = arch
- files[f]["byhand-script"] = ABH["%s::Script" % (pckg)]
+ u.pkg.files[f]["byhand-arch"] = arch
+ u.pkg.files[f]["byhand-script"] = ABH["%s::Script" % (pckg)]
return any_auto and all_auto
-def do_autobyhand (summary, short_summary):
+def do_autobyhand(u, summary, short_summary):
print "Attempting AUTOBYHAND."
- byhandleft = 0
- for f in files.keys():
+ byhandleft = True
+ for f, entry in u.pkg.files.items():
byhandfile = f
- if not files[f].has_key("byhand"):
+
+ if not entry.has_key("byhand"):
continue
- if not files[f].has_key("byhand-script"):
- byhandleft = 1
+
+ if not entry.has_key("byhand-script"):
+ byhandleft = True
continue
os.system("ls -l %s" % byhandfile)
+
result = os.system("%s %s %s %s %s" % (
- files[f]["byhand-script"], byhandfile,
- changes["version"], files[f]["byhand-arch"],
- os.path.abspath(pkg.changes_file)))
+ entry["byhand-script"],
+ byhandfile,
+ u.pkg.changes["version"],
+ entry["byhand-arch"],
+ os.path.abspath(u.pkg.changes_file)))
+
if result == 0:
os.unlink(byhandfile)
- del files[f]
+ del entry
else:
print "Error processing %s, left as byhand." % (f)
- byhandleft = 1
+ byhandleft = True
if byhandleft:
- do_byhand(summary, short_summary)
+ do_byhand(u, summary, short_summary)
else:
- accept(summary, short_summary)
+ u.accept(summary, short_summary)
+ u.check_override()
+ # XXX: We seem to be missing a u.remove() here
+ # This might explain why we get byhand leftovers in unchecked - mhy
################################################################################
-def is_byhand ():
- for f in files.keys():
- if files[f].has_key("byhand"):
- return 1
- return 0
-
-def do_byhand (summary, short_summary):
- print "Moving to BYHAND holding area."
- Logger.log(["Moving to byhand", pkg.changes_file])
+def is_byhand(u):
+ for f in u.pkg.files.keys():
+ if u.pkg.files[f].has_key("byhand"):
+ return True
+ return False
- Upload.dump_vars(Cnf["Dir::Queue::Byhand"])
- move_to_dir(Cnf["Dir::Queue::Byhand"])
-
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- Upload.check_override()
+def do_byhand(u, summary, short_summary):
+ return package_to_queue(u, summary, short_summary, "Byhand",
+ perms=0660, build=False, announce=None)
################################################################################
-def is_new ():
- for f in files.keys():
- if files[f].has_key("new"):
- return 1
- return 0
+def is_new(u):
+ for f in u.pkg.files.keys():
+ if u.pkg.files[f].has_key("new"):
+ return True
+ return False
-def acknowledge_new (summary, short_summary):
- Subst = Upload.Subst
+def acknowledge_new(u, summary, short_summary):
+ cnf = Config()
print "Moving to NEW holding area."
- Logger.log(["Moving to new", pkg.changes_file])
+ Logger.log(["Moving to new", u.pkg.changes_file])
- Upload.dump_vars(Cnf["Dir::Queue::New"])
- move_to_dir(Cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
+ u.pkg.write_dot_dak(cnf["Dir::Queue::New"])
+ u.move_to_dir(cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
if not Options["No-Mail"]:
print "Sending new ack."
- Subst["__SUMMARY__"] = summary
- new_ack_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.new")
+ template = os.path.join(cnf["Dir::Templates"], 'process-unchecked.new')
+ u.update_subst()
+ u.Subst["__SUMMARY__"] = summary
+ new_ack_message = utils.TemplateSubst(u.Subst, template)
utils.send_mail(new_ack_message)
################################################################################
# we force the .orig.tar.gz into the .changes structure and reprocess
# the .changes file.
-def process_it (changes_file):
- global reprocess, reject_message
+def process_it(changes_file):
+ global Logger
+
+ cnf = Config()
+
+ holding = Holding()
+
+ u = Upload()
+ u.pkg.changes_file = changes_file
+ u.pkg.directory = os.getcwd()
+ u.logger = Logger
+ origchanges = os.path.join(u.pkg.directory, u.pkg.changes_file)
- # Reset some globals
- reprocess = 1
- Upload.init_vars()
# Some defaults in case we can't fully process the .changes file
- changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]
- changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]
- reject_message = ""
+ u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
+ u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
- # Absolutize the filename to avoid the requirement of being in the
- # same directory as the .changes file.
- pkg.changes_file = os.path.abspath(changes_file)
+ # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
+ bcc = "X-DAK: dak process-unchecked"
+ if cnf.has_key("Dinstall::Bcc"):
+ u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+ else:
+ u.Subst["__BCC__"] = bcc
# Remember where we are so we can come back after cd-ing into the
- # holding directory.
- pkg.directory = os.getcwd()
+ # holding directory. TODO: Fix this stupid hack
+ u.prevdir = os.getcwd()
+
+ # TODO: Figure out something better for this (or whether it's even
+ # necessary - it seems to have been for use when we were
+ # still doing the is_unchecked check; reprocess = 2)
+ u.reprocess = 1
try:
# If this is the Real Thing(tm), copy things into a private
# holding directory first to avoid replacable file races.
if not Options["No-Action"]:
- os.chdir(Cnf["Dir::Queue::Holding"])
- copy_to_holding(pkg.changes_file)
+ os.chdir(cnf["Dir::Queue::Holding"])
+
+ # Absolutize the filename to avoid the requirement of being in the
+ # same directory as the .changes file.
+ holding.copy_to_holding(origchanges)
+
# Relativize the filename so we use the copy in holding
# rather than the original...
- pkg.changes_file = os.path.basename(pkg.changes_file)
- changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject)
- if changes["fingerprint"]:
- valid_changes_p = check_changes()
+ changespath = os.path.basename(u.pkg.changes_file)
+
+ (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changespath)
+
+ if u.pkg.changes["fingerprint"]:
+ valid_changes_p = u.load_changes(changespath)
else:
- valid_changes_p = 0
+ valid_changes_p = False
+ u.rejects.extend(rejects)
+
if valid_changes_p:
- while reprocess:
- check_distributions()
- check_files()
- valid_dsc_p = check_dsc()
- if valid_dsc_p:
- check_source()
- check_hashes()
- check_urgency()
- check_timestamps()
- check_signed_by_key()
- Upload.update_subst(reject_message)
- action()
+ while u.reprocess:
+ u.check_distributions()
+ u.check_files(not Options["No-Action"])
+ valid_dsc_p = u.check_dsc(not Options["No-Action"])
+ if valid_dsc_p and not Options["No-Action"]:
+ u.check_source()
+ u.check_lintian()
+ u.check_hashes()
+ u.check_urgency()
+ u.check_timestamps()
+ u.check_signed_by_key()
+
+ action(u)
+
except SystemExit:
raise
+
except:
print "ERROR"
traceback.print_exc(file=sys.stderr)
- pass
# Restore previous WD
- os.chdir(pkg.directory)
+ os.chdir(u.prevdir)
###############################################################################
def main():
- global Cnf, Options, Logger
+ global Options, Logger
+ cnf = Config()
changes_files = init()
# -n/--dry-run invalidates some other options which would involve things happening
if Options["No-Action"]:
Options["Automatic"] = ""
+ # Initialize our Holding singleton
+ holding = Holding()
+
# Ensure all the arguments we were given are .changes files
for f in changes_files:
if not f.endswith(".changes"):
changes_files.remove(f)
if changes_files == []:
- if Cnf["Dinstall::Options::Directory"] == "":
+ if cnf["Dinstall::Options::Directory"] == "":
utils.fubar("Need at least one .changes file as an argument.")
else:
sys.exit(0)
# Check that we aren't going to clash with the daily cron job
-
- if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]:
+ if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
utils.fubar("Archive maintenance in progress. Try again later.")
# Obtain lock if not in no-action mode and initialize the log
-
if not Options["No-Action"]:
- lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+ lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError, e:
utils.fubar("Couldn't obtain lock; assuming another 'dak process-unchecked' is already running.")
else:
raise
- Logger = Upload.Logger = logging.Logger(Cnf, "process-unchecked")
-
- # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
- bcc = "X-DAK: dak process-unchecked\nX-Katie: $Revision: 1.65 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Upload.Subst["__BCC__"] = bcc
-
+ Logger = daklog.Logger(cnf, "process-unchecked")
# Sort the .changes files so that we process sourceful ones first
changes_files.sort(utils.changes_compare)
process_it (changes_file)
finally:
if not Options["No-Action"]:
- clean_holding()
+ holding.clean()
+
+ accept_count = SummaryStats().accept_count
+ accept_bytes = SummaryStats().accept_bytes
- accept_count = Upload.accept_count
- accept_bytes = Upload.accept_bytes
if accept_count:
sets = "set"
if accept_count > 1:
################################################################################
-import copy, glob, os, stat, sys, time
+from copy import copy
+import glob, os, stat, sys, time
import apt_pkg
-import cgi
-from daklib import queue
-from daklib import database
+
from daklib import utils
+from daklib.changes import Changes
+from daklib.dbconn import DBConn, has_new_comment
+from daklib.textutils import fix_maintainer
from daklib.dak_exceptions import *
Cnf = None
-Upload = None
direction = []
row_number = 0
-projectB = None
################################################################################
try:
(login, domain) = sponsor.split("@", 1)
print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span>@debian.org<br/>" % (utils.html_escape(login), utils.html_escape(login))
- except:
+ except Exception, e:
+ print "WARNING: Exception %s" % e
pass
print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
# Read in all the .changes files
for filename in changes_files:
try:
- Upload.pkg.changes_file = filename
- Upload.init_vars()
- Upload.update_vars()
- cache[filename] = copy.copy(Upload.pkg.changes)
+ c = Changes()
+ c.load_dot_dak(filename)
+ cache[filename] = copy(c.changes)
cache[filename]["filename"] = filename
- except:
- break
+ except Exception, e:
+ print "WARNING: Exception %s" % e
+ continue
# Divide the .changes into per-source groups
per_source = {}
for filename in cache.keys():
else:
if mtime < oldest:
oldest = mtime
- have_note += (database.has_new_comment(d["source"], d["version"]))
+ have_note += has_new_comment(d["source"], d["version"])
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
- utils.fix_maintainer (j["maintainer"])
+ fix_maintainer (j["maintainer"])
except ParseMaintError, msg:
print "Problems while parsing maintainer address\n"
maintainer["maintainername"] = "Unknown"
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
- utils.fix_maintainer (j["changed-by"])
+ fix_maintainer (j["changed-by"])
except ParseMaintError, msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
################################################################################
def main():
- global Cnf, Upload
+ global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Queue-Report::Options::Help"),
if Options["Help"]:
usage()
- Upload = queue.Upload(Cnf)
- projectB = Upload.projectB
-
if Cnf.has_key("Queue-Report::Options::New"):
header()
+ # Initialize db so we can get the NEW comments
+ dbconn = DBConn()
+
directories = [ ]
if Cnf.has_key("Queue-Report::Options::Directories"):
+++ /dev/null
-#!/usr/bin/env python
-
-""" Manually reject packages for proprosed-updates """
-# Copyright (C) 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-################################################################################
-
-import os, pg, sys
-import apt_pkg
-from daklib import database
-from daklib import logging
-from daklib import queue
-from daklib import utils
-from daklib.regexes import re_default_answer
-
-################################################################################
-
-# Globals
-Cnf = None
-Options = None
-projectB = None
-Upload = None
-Logger = None
-
-################################################################################
-
-def usage(exit_code=0):
- print """Usage: dak reject-proposed-updates .CHANGES[...]
-Manually reject the .CHANGES file(s).
-
- -h, --help show this help and exit.
- -m, --message=MSG use this message for the rejection.
- -s, --no-mail don't send any mail."""
- sys.exit(exit_code)
-
-################################################################################
-
-def main():
- global Cnf, Logger, Options, projectB, Upload
-
- Cnf = utils.get_conf()
- Arguments = [('h',"help","Reject-Proposed-Updates::Options::Help"),
- ('m',"manual-reject","Reject-Proposed-Updates::Options::Manual-Reject", "HasArg"),
- ('s',"no-mail", "Reject-Proposed-Updates::Options::No-Mail")]
- for i in [ "help", "manual-reject", "no-mail" ]:
- if not Cnf.has_key("Reject-Proposed-Updates::Options::%s" % (i)):
- Cnf["Reject-Proposed-Updates::Options::%s" % (i)] = ""
-
- arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
-
- Options = Cnf.SubTree("Reject-Proposed-Updates::Options")
- if Options["Help"]:
- usage()
- if not arguments:
- utils.fubar("need at least one .changes filename as an argument.")
-
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
- Upload = queue.Upload(Cnf)
- Logger = Upload.Logger = logging.Logger(Cnf, "reject-proposed-updates")
-
- bcc = "X-DAK: dak rejected-proposed-updates\nX-Katie: lauren $Revision: 1.4 $"
- if Cnf.has_key("Dinstall::Bcc"):
- Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
- else:
- Upload.Subst["__BCC__"] = bcc
-
- for arg in arguments:
- arg = utils.validate_changes_file_arg(arg)
- Upload.pkg.changes_file = arg
- Upload.init_vars()
- cwd = os.getcwd()
- os.chdir(Cnf["Suite::Proposed-Updates::CopyDotDak"])
- Upload.update_vars()
- os.chdir(cwd)
- Upload.update_subst()
-
- print arg
- done = 0
- prompt = "Manual reject, [S]kip, Quit ?"
- while not done:
- answer = "XXX"
-
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt)
- m = re_default_answer.search(prompt)
- if answer == "":
- answer = m.group(1)
- answer = answer[:1].upper()
-
- if answer == 'M':
- aborted = reject(Options["Manual-Reject"])
- if not aborted:
- done = 1
- elif answer == 'S':
- done = 1
- elif answer == 'Q':
- sys.exit(0)
-
- Logger.close()
-
-################################################################################
-
-def reject (reject_message = ""):
- files = Upload.pkg.files
- dsc = Upload.pkg.dsc
- changes_file = Upload.pkg.changes_file
-
- # If we weren't given a manual rejection message, spawn an editor
- # so the user can add one in...
- if not reject_message:
- (fd, temp_filename) = utils.temp_filename()
- editor = os.environ.get("EDITOR","vi")
- answer = 'E'
- while answer == 'E':
- os.system("%s %s" % (editor, temp_filename))
- f = os.fdopen(fd)
- reject_message = "".join(f.readlines())
- f.close()
- print "Reject message:"
- print utils.prefix_multi_line_string(reject_message," ", include_blank_lines=1)
- prompt = "[R]eject, Edit, Abandon, Quit ?"
- answer = "XXX"
- while prompt.find(answer) == -1:
- answer = utils.our_raw_input(prompt)
- m = re_default_answer.search(prompt)
- if answer == "":
- answer = m.group(1)
- answer = answer[:1].upper()
- os.unlink(temp_filename)
- if answer == 'A':
- return 1
- elif answer == 'Q':
- sys.exit(0)
-
- print "Rejecting.\n"
-
- # Reject the .changes file
- Upload.force_reject([changes_file])
-
- # Setup the .reason file
- reason_filename = changes_file[:-8] + ".reason"
- reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename
-
- # If we fail here someone is probably trying to exploit the race
- # so let's just raise an exception ...
- if os.path.exists(reject_filename):
- os.unlink(reject_filename)
- reject_fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
-
- # Build up the rejection email
- user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"])
-
- Upload.Subst["__REJECTOR_ADDRESS__"] = user_email_address
- Upload.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
- Upload.Subst["__STABLE_REJECTOR__"] = Cnf["Reject-Proposed-Updates::StableRejector"]
- Upload.Subst["__STABLE_MAIL__"] = Cnf["Reject-Proposed-Updates::StableMail"]
- Upload.Subst["__MORE_INFO_URL__"] = Cnf["Reject-Proposed-Updates::MoreInfoURL"]
- Upload.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
- reject_mail_message = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/reject-proposed-updates.rejected")
-
- # Write the rejection email out as the <foo>.reason file
- os.write(reject_fd, reject_mail_message)
- os.close(reject_fd)
-
- # Remove the packages from proposed-updates
- suite_id = database.get_suite_id('proposed-updates')
-
- projectB.query("BEGIN WORK")
- # Remove files from proposed-updates suite
- for f in files.keys():
- if files[f]["type"] == "dsc":
- package = dsc["source"]
- version = dsc["version"]; # NB: not files[f]["version"], that has no epoch
- q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
- ql = q.getresult()
- if not ql:
- utils.fubar("reject: Couldn't find %s_%s in source table." % (package, version))
- source_id = ql[0][0]
- projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id))
- elif files[f]["type"] == "deb":
- package = files[f]["package"]
- version = files[f]["version"]
- architecture = files[f]["architecture"]
- q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
- ql = q.getresult()
-
- # Horrible hack to work around partial replacement of
- # packages with newer versions (from different source
- # packages). This, obviously, should instead check for a
- # newer version of the package and only do the
- # warn&continue thing if it finds one.
- if not ql:
- utils.warn("reject: Couldn't find %s_%s_%s in binaries table." % (package, version, architecture))
- else:
- binary_id = ql[0][0]
- projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id))
- projectB.query("COMMIT WORK")
-
- # Send the rejection mail if appropriate
- if not Options["No-Mail"]:
- utils.send_mail(reject_mail_message)
-
- # Finally remove the .dak file
- dot_dak_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyDotDak"], os.path.basename(changes_file[:-8]+".dak"))
- os.unlink(dot_dak_file)
-
- Logger.log(["rejected", changes_file])
- return 0
-
-################################################################################
-
-if __name__ == '__main__':
- main()
import commands
import os
-import pg
-import re
import sys
import apt_pkg
import apt_inst
-from daklib import database
+
+from daklib.config import Config
+from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import *
from daklib.regexes import re_strip_source_version, re_build_dep_arch
################################################################################
-Cnf = None
Options = None
-projectB = None
################################################################################
################################################################################
def reverse_depends_check(removals, suites, arches=None):
+ cnf = Config()
+
print "Checking reverse dependencies..."
- components = Cnf.ValueList("Suite::%s::Components" % suites[0])
+ components = cnf.ValueList("Suite::%s::Components" % suites[0])
dep_problem = 0
p2c = {}
all_broken = {}
if arches:
all_arches = set(arches)
else:
- all_arches = set(database.get_suite_architectures(suites[0]))
+ all_arches = set([x.arch_string for x in get_suite_architectures(suites[0])])
all_arches -= set(["source", "all"])
for architecture in all_arches:
deps = {}
sources = {}
virtual_packages = {}
for component in components:
- filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suites[0], component, architecture)
+ filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suites[0], component, architecture)
# apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
# Check source dependencies (Build-Depends and Build-Depends-Indep)
all_broken.clear()
for component in components:
- filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suites[0], component)
+ filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suites[0], component)
# apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
result, output = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
################################################################################
def main ():
- global Cnf, Options, projectB
+ global Options
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Rm::Options::Help"),
('a',"architecture","Rm::Options::Architecture", "HasArg"),
for i in [ "architecture", "binary-only", "carbon-copy", "component",
"done", "help", "no-action", "partial", "rdep-check", "reason",
"source-only" ]:
- if not Cnf.has_key("Rm::Options::%s" % (i)):
- Cnf["Rm::Options::%s" % (i)] = ""
- if not Cnf.has_key("Rm::Options::Suite"):
- Cnf["Rm::Options::Suite"] = "unstable"
+ if not cnf.has_key("Rm::Options::%s" % (i)):
+ cnf["Rm::Options::%s" % (i)] = ""
+ if not cnf.has_key("Rm::Options::Suite"):
+ cnf["Rm::Options::Suite"] = "unstable"
- arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
- Options = Cnf.SubTree("Rm::Options")
+ arguments = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Rm::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# Sanity check options
if not arguments:
carbon_copy = []
for copy_to in utils.split_args(Options.get("Carbon-Copy")):
if copy_to.isdigit():
- carbon_copy.append(copy_to + "@" + Cnf["Dinstall::BugServer"])
+ carbon_copy.append(copy_to + "@" + cnf["Dinstall::BugServer"])
elif copy_to == 'package':
for package in arguments:
- carbon_copy.append(package + "@" + Cnf["Dinstall::PackagesServer"])
- if Cnf.has_key("Dinstall::TrackingServer"):
- carbon_copy.append(package + "@" + Cnf["Dinstall::TrackingServer"])
+ carbon_copy.append(package + "@" + cnf["Dinstall::PackagesServer"])
+ if cnf.has_key("Dinstall::TrackingServer"):
+ carbon_copy.append(package + "@" + cnf["Dinstall::TrackingServer"])
elif '@' in copy_to:
carbon_copy.append(copy_to)
else:
suites_list = utils.join_with_commas_and(suites)
if not Options["No-Action"]:
for suite in suites:
- suite_id = database.get_suite_id(suite)
- if suite_id != -1:
- suite_ids_list.append(suite_id)
+ s = get_suite(suite, session=session)
+ if s is not None:
+ suite_ids_list.append(s.suite_id)
if suite == "stable":
print "**WARNING** About to remove from the stable suite!"
print "This should only be done just prior to a (point) release and not at"
# latter is a nasty mess, but very nice from a UI perspective so
# we try to support it.
+ # XXX: TODO: This all needs converting to use placeholders or the object
+ # API. It's an SQL injection dream at the moment
+
if Options["Binary-Only"]:
# Binary-only
- q = projectB.query("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
- for i in q.getresult():
+ q = session.execute("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures))
+ for i in q.fetchall():
to_remove.append(i)
else:
# Source-only
source_packages = {}
- q = projectB.query("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
- for i in q.getresult():
+ q = session.execute("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
+ for i in q.fetchall():
source_packages[i[2]] = i[:2]
to_remove.append(i[2:])
if not Options["Source-Only"]:
# Source + Binary
binary_packages = {}
# First get a list of binary package names we suspect are linked to the source
- q = projectB.query("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
- for i in q.getresult():
+ q = session.execute("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components))
+ for i in q.fetchall():
binary_packages[i[0]] = ""
# Then parse each .dsc that we found earlier to see what binary packages it thinks it produces
for i in source_packages.keys():
# source package and if so add it to the list of packages
# to be removed.
for package in binary_packages.keys():
- q = projectB.query("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package))
- for i in q.getresult():
+ q = session.execute("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package))
+ for i in q.fetchall():
filename = "/".join(i[:2])
control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(filename)))
source = control.Find("Source", control.Find("Package"))
maintainer_list = []
for maintainer_id in maintainers.keys():
- maintainer_list.append(database.get_maintainer(maintainer_id))
+ maintainer_list.append(get_maintainer(maintainer_id).name)
summary = ""
removals = d.keys()
removals.sort()
date = commands.getoutput('date -R')
# Log first; if it all falls apart I want a record that we at least tried.
- logfile = utils.open_file(Cnf["Rm::LogFile"], 'a')
+ logfile = utils.open_file(cnf["Rm::LogFile"], 'a')
logfile.write("=========================================================================\n")
logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary))
logfile.write("----------------------------------------------\n")
logfile.flush()
- dsc_type_id = database.get_override_type_id('dsc')
- deb_type_id = database.get_override_type_id('deb')
+ dsc_type_id = get_override_type('dsc', session).overridetype_id
+ deb_type_id = get_override_type('deb', session).overridetype_id
# Do the actual deletion
print "Deleting...",
sys.stdout.flush()
- projectB.query("BEGIN WORK")
+
for i in to_remove:
package = i[0]
architecture = i[2]
package_id = i[3]
for suite_id in suite_ids_list:
if architecture == "source":
- projectB.query("DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id))
+ session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
#print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)
else:
- projectB.query("DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id))
+ session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid",
+ {'packageid': package_id, 'suiteid': suite_id})
#print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)
# Delete from the override file
if not Options["Partial"]:
type_id = dsc_type_id
else:
type_id = deb_type_id
- projectB.query("DELETE FROM override WHERE package = '%s' AND type = %s AND suite = %s %s" % (package, type_id, suite_id, over_con_components))
- projectB.query("COMMIT WORK")
+ # TODO: Again, fix this properly to remove the remaining non-bind argument
+ session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (over_con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id})
+ session.commit()
print "done."
# Send the bug closing messages
if Options["Done"]:
Subst = {}
- Subst["__RM_ADDRESS__"] = Cnf["Rm::MyEmailAddress"]
- Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
+ Subst["__RM_ADDRESS__"] = cnf["Rm::MyEmailAddress"]
+ Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
bcc = []
- if Cnf.Find("Dinstall::Bcc") != "":
- bcc.append(Cnf["Dinstall::Bcc"])
- if Cnf.Find("Rm::Bcc") != "":
- bcc.append(Cnf["Rm::Bcc"])
+ if cnf.Find("Dinstall::Bcc") != "":
+ bcc.append(cnf["Dinstall::Bcc"])
+ if cnf.Find("Rm::Bcc") != "":
+ bcc.append(cnf["Rm::Bcc"])
if bcc:
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst["__BCC__"] = "X-Filler: 42"
- Subst["__CC__"] = "X-DAK: dak rm\nX-Katie: melanie"
+ Subst["__CC__"] = "X-DAK: dak rm"
if carbon_copy:
Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy)
Subst["__SUITE_LIST__"] = suites_list
Subst["__SUMMARY__"] = summary
- Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
- Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
+ Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst["__WHOAMI__"] = whoami
whereami = utils.where_am_i()
- Archive = Cnf.SubTree("Archive::%s" % (whereami))
+ Archive = cnf.SubTree("Archive::%s" % (whereami))
Subst["__MASTER_ARCHIVE__"] = Archive["OriginServer"]
Subst["__PRIMARY_MIRROR__"] = Archive["PrimaryMirror"]
for bug in utils.split_args(Options["Done"]):
Subst["__BUG_NUMBER__"] = bug
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/rm.bug-close")
+ mail_message = utils.TemplateSubst(Subst,cnf["Dir::Templates"]+"/rm.bug-close")
utils.send_mail(mail_message)
logfile.write("=========================================================================\n")
import sys, os, re, time
import apt_pkg
-import tempfile
from debian_bundle import deb822
-from daklib import database
-from daklib import queue
+from daklib.dbconn import *
from daklib import utils
from daklib.regexes import re_html_escaping, html_escaping
uploader = re.sub(r'^\s*(\S.*)\s+<.*>',r'\1',uploader)
if Cnf.has_key("Show-Deferred::LinkPath"):
isnew = 0
- suites = database.get_suites(achanges['source'],src=1)
+ suites = get_suites_source_in(achanges['source'])
if 'unstable' not in suites and 'experimental' not in suites:
isnew = 1
+
for b in achanges['binary'].split():
- suites = database.get_suites(b)
+ suites = get_suites_binary_in(b)
if 'unstable' not in suites and 'experimental' not in suites:
isnew = 1
+
if not isnew:
# we don't link .changes because we don't want other people to
# upload it with the existing signature.
sys.exit(exit_code)
def init():
- global Cnf, Options, Upload, projectB
+ global Cnf, Options
Cnf = utils.get_conf()
Arguments = [('h',"help","Show-Deferred::Options::Help"),
("p","link-path","Show-Deferred::LinkPath","HasArg"),
Options = Cnf.SubTree("Show-Deferred::Options")
if Options["help"]:
usage()
- Upload = queue.Upload(Cnf)
- projectB = Upload.projectB
+
+ # Initialise database connection
+ DBConn()
+
return args
def main():
################################################################################
-import copy, os, sys, time
+from copy import copy
+import os, sys, time
import apt_pkg
import examine_package
-from daklib import database
-from daklib import queue
+
+from daklib.queue import determine_new, check_valid
from daklib import utils
+from daklib.regexes import re_source_ext
# Globals
Cnf = None
Options = None
-Upload = None
-projectB = None
sources = set()
def do_pkg(changes_file):
- Upload.pkg.changes_file = changes_file
- Upload.init_vars()
- Upload.update_vars()
- files = Upload.pkg.files
- changes = Upload.pkg.changes
-
- changes["suite"] = copy.copy(changes["distribution"])
- distribution = changes["distribution"].keys()[0]
+ c = Changes()
+ c.load_dot_dak(changes_file)
+ files = c.files
+ changes = c.changes
+
+ c.changes["suite"] = copy(c.changes["distribution"])
+ distribution = c.changes["distribution"].keys()[0]
# Find out what's new
- new = queue.determine_new(changes, files, projectB, 0)
+ new = determine_new(c.changes, c.files, 0)
stdout_fd = sys.stdout
- htmlname = changes["source"] + "_" + changes["version"] + ".html"
+ htmlname = c.changes["source"] + "_" + c.changes["version"] + ".html"
sources.add(htmlname)
# do not generate html output if that source/version already has one.
if not os.path.exists(os.path.join(Cnf["Show-New::HTMLPath"],htmlname)):
filestoexamine = []
for pkg in new.keys():
for fn in new[pkg]["files"]:
- if ( files[fn].has_key("new") and not
- files[fn]["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2"] ):
+ if (c.files[fn].has_key("new") and
+ (c.files[fn]["type"] == "dsc" or
+ not re_source_ext.match(c.files[fn]["type"]))):
filestoexamine.append(fn)
- html_header(changes["source"], filestoexamine)
+ html_header(c.changes["source"], filestoexamine)
- queue.check_valid(new)
- examine_package.display_changes( distribution, Upload.pkg.changes_file)
+ check_valid(new)
+ examine_package.display_changes( distribution, changes_file)
for fn in filter(lambda fn: fn.endswith(".dsc"), filestoexamine):
examine_package.check_dsc(distribution, fn)
################################################################################
def init():
- global Cnf, Options, Upload, projectB
+ global Cnf, Options
Cnf = utils.get_conf()
if Options["help"]:
usage()
- Upload = queue.Upload(Cnf)
-
- projectB = Upload.projectB
-
return changes_files
################################################################################
-import pg, sys
+import sys
import apt_pkg
+
from daklib import utils
-from daklib import database
+from daklib.dbconn import DBConn, get_suite_architectures, Suite, Architecture
################################################################################
Cnf = None
-projectB = None
################################################################################
################################################################################
def per_arch_space_use():
- q = projectB.query("""
-SELECT a.arch_string as Architecture, sum(f.size)
+ session = DBConn().session()
+ q = session.execute("""
+SELECT a.arch_string as Architecture, sum(f.size) AS sum
FROM files f, binaries b, architecture a
WHERE a.id=b.architecture AND f.id=b.file
- GROUP BY a.arch_string""")
- print q
- q = projectB.query("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'")
- print q
+ GROUP BY a.arch_string ORDER BY sum""").fetchall()
+ for j in q:
+ print "%-15.15s %s" % (j[0], j[1])
+ print
+ q = session.execute("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'").fetchall()
+ print "%-15.15s %s" % ("Source", q[0][0])
################################################################################
output_suite.append(word[0])
return "-".join(output_suite)
-# Obvious query with GROUP BY and mapped names -> 50 seconds
-# GROUP BY but ids instead of suite/architecture names -> 28 seconds
-# Simple query -> 14 seconds
-# Simple query into large dictionary + processing -> 21 seconds
-# Simple query into large pre-created dictionary + processing -> 18 seconds
-
def number_of_packages():
arches = {}
arch_ids = {}
suites = {}
suite_ids = {}
d = {}
+ session = DBConn().session()
# Build up suite mapping
- q = projectB.query("SELECT id, suite_name FROM suite")
- suite_ql = q.getresult()
- for i in suite_ql:
- (sid, name) = i
- suites[sid] = name
- suite_ids[name] = sid
+ for i in session.query(Suite).all():
+ suites[i.suite_id] = i.suite_name
+ suite_ids[i.suite_name] = i.suite_id
# Build up architecture mapping
- q = projectB.query("SELECT id, arch_string FROM architecture")
- for i in q.getresult():
- (aid, name) = i
- arches[aid] = name
- arch_ids[name] = aid
+ for i in session.query(Architecture).all():
+ arches[i.arch_id] = i.arch_string
+ arch_ids[i.arch_string] = i.arch_id
# Pre-create the dictionary
for suite_id in suites.keys():
d[suite_id] = {}
for arch_id in arches.keys():
d[suite_id][arch_id] = 0
# Get the raw data for binaries
- q = projectB.query("""
-SELECT ba.suite, b.architecture
- FROM binaries b, bin_associations ba
- WHERE b.id = ba.bin""")
# Simultate 'GROUP by suite, architecture' with a dictionary
- for i in q.getresult():
- (suite_id, arch_id) = i
- d[suite_id][arch_id] = d[suite_id][arch_id] + 1
+ # XXX: Why don't we just get the DB to do this?
+ for i in session.execute("""SELECT suite, architecture, COUNT(suite)
+ FROM bin_associations
+ LEFT JOIN binaries ON bin = binaries.id
+ GROUP BY suite, architecture""").fetchall():
+ d[ i[0] ][ i[1] ] = i[2]
# Get the raw data for source
arch_id = arch_ids["source"]
- q = projectB.query("""
-SELECT suite, count(suite) FROM src_associations GROUP BY suite;""")
- for i in q.getresult():
+ for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall():
(suite_id, count) = i
d[suite_id][arch_id] = d[suite_id][arch_id] + count
## Print the results
for suite in suite_list:
suite_id = suite_ids[suite]
suite_arches[suite_id] = {}
- for arch in database.get_suite_architectures(suite_id):
- suite_arches[suite_id][arch] = ""
+ for arch in get_suite_architectures(suite):
+ suite_arches[suite_id][arch.arch_string] = ""
suite_id_list.append(suite_id)
output_list = [ output_format(i) for i in suite_list ]
longest_suite = longest(output_list)
output = output + arch.center(longest_arch)+" |"
for suite_id in suite_id_list:
if suite_arches[suite_id].has_key(arch):
- count = repr(d[suite_id][arch_id])
+ count = "%d" % d[suite_id][arch_id]
else:
count = "-"
output = output + count.rjust(longest_suite)+" |"
################################################################################
def main ():
- global Cnf, projectB
+ global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Stats::Options::Help")]
usage(1)
mode = args[0].lower()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
-
if mode == "arch-space":
per_arch_space_use()
elif mode == "pkg-nums":
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-# Test utils.fix_maintainer()
+# Test textutils.fix_maintainer()
# Copyright (C) 2004, 2006 James Troup <james@nocrew.org>
# This program is free software; you can redistribute it and/or modify
sys.path.append(os.path.abspath('../../'))
-import utils
+import textutils
################################################################################
################################################################################
def check_valid(s, xa, xb, xc, xd):
- (a, b, c, d) = utils.fix_maintainer(s)
+ (a, b, c, d) = textutils.fix_maintainer(s)
if a != xa:
fail("rfc822_maint: %s (returned) != %s (expected [From: '%s']" % (a, xa, s))
if b != xb:
def check_invalid(s):
try:
- utils.fix_maintainer(s)
+ textutils.fix_maintainer(s)
fail("%s was parsed successfully but is expected to be invalid." % (s))
except utils.ParseMaintError, unused:
pass
################################################################################
import os
-import pg
import sys
import time
import errno
import fcntl
import tempfile
-import pwd
import apt_pkg
-from daklib import database
+
+from daklib.dbconn import *
from daklib import utils
from daklib.dak_exceptions import TransitionsError
from daklib.regexes import re_broken_package
# Globals
Cnf = None #: Configuration, apt_pkg.Configuration
Options = None #: Parsed CommandLine arguments
-projectB = None #: database connection, pgobject
################################################################################
@attention: This function may run B{within sudo}
"""
- global Cnf, Options, projectB
+ global Cnf, Options
apt_pkg.init()
print "Non-dak user: %s" % username
Options["sudo"] = "y"
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ # Initialise DB connection
+ DBConn()
################################################################################
to_dump = 0
to_remove = []
info = {}
+
+ session = DBConn().session()
+
# Now look through all defined transitions
for trans in transitions:
t = transitions[trans]
source = t["source"]
expected = t["new"]
- # Will be None if nothing is in testing.
- current = database.get_suite_version(source, "testing")
+ # Will be an empty list if nothing is in testing.
+ sourceobj = get_source_in_suite(source, "testing", session)
info[trans] = get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
print info[trans]
- if current == None:
+ if sourceobj is None:
# No package in testing
print "Transition source %s not in testing, transition still ongoing." % (source)
else:
+ current = sourceobj.version
compare = apt_pkg.VersionCompare(current, expected)
if compare < 0:
# This is still valid, the current version in database is older than
@type transitions: dict
@param transitions: defined transitions
"""
+
+ session = DBConn().session()
+
for trans in transitions:
t = transitions[trans]
source = t["source"]
expected = t["new"]
# Will be None if nothing is in testing.
- current = database.get_suite_version(source, "testing")
+ sourceobj = get_source_in_suite(source, "testing", session)
print get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
- if current == None:
+ if sourceobj is None:
# No package in testing
print "Transition source %s not in testing, transition still ongoing." % (source)
else:
- compare = apt_pkg.VersionCompare(current, expected)
+ compare = apt_pkg.VersionCompare(sourceobj.version, expected)
print "Apt compare says: %s" % (compare)
if compare < 0:
# This is still valid, the current version in database is older than
# the new version we wait for
- print "This transition is still ongoing, we currently have version %s" % (current)
+ print "This transition is still ongoing, we currently have version %s" % (sourceobj.version)
else:
print "This transition is over, the target package reached testing, should be removed"
- print "%s wanted version: %s, has %s" % (source, expected, current)
+ print "%s wanted version: %s, has %s" % (source, expected, sourceobj.version)
print "-------------------------------------------------------------------------"
################################################################################
import apt_pkg
import time
import errno
-from daklib import database
+
from daklib import utils
from daklib.dak_exceptions import DBUpdateError
################################################################################
Cnf = None
-projectB = None
-required_database_schema = 14
+required_database_schema = 15
################################################################################
################################################################################
def get_db_rev(self):
- global projectB
-
# We keep database revision info the config table
# Try and access it
################################################################################
def init (self):
- global Cnf, projectB
+ global Cnf
Cnf = utils.get_conf()
arguments = [('h', "help", "Update-DB::Options::Help")]
import os
import sys
import shutil
-import tempfile
import tarfile
import commands
import traceback
import atexit
+
from debian_bundle import deb822
-from dbconn import DBConn
+
+from dbconn import *
from config import Config
-import logging
import utils
+################################################################################
+
+__all__ = []
+
+################################################################################
+
class Binary(object):
def __init__(self, filename, reject=None):
"""
self.tmpdir = None
self.chunks = None
self.wrapped_reject = reject
+ # Store rejects for later use
+ self.rejects = []
def reject(self, message):
"""
otherwise send it to stderr.
"""
print >> sys.stderr, message
+ self.rejects.append(message)
if self.wrapped_reject:
self.wrapped_reject(message)
return not rejected
- def scan_package(self, bootstrap_id=0, relaxed=False):
+ def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
"""
Unpack the .deb, do sanity checking, and gather info from it.
data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
if bootstrap_id:
- result = DBConn().insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
else:
pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
pkg = pkgs.next()
- result = DBConn().insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ result = insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
except:
traceback.print_exc()
os.chdir(cwd)
+__all__.append('Binary')
+
+def copy_temporary_contents(package, version, archname, deb, reject, session=None):
+ """
+ copy the previously stored contents from the temp table to the permanant one
+
+ during process-unchecked, the deb should have been scanned and the
+ contents stored in pending_content_associations
+ """
+
+ cnf = Config()
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ arch = get_architecture(archname, session=session)
+
+ # first see if contents exist:
+ in_pcaq = """SELECT 1 FROM pending_content_associations
+ WHERE package=:package
+ AND version=:version
+ AND architecture=:archid LIMIT 1"""
+
+ vals = {'package': package,
+ 'version': version,
+ 'archid': arch.arch_id}
+
+ exists = None
+ check = session.execute(in_pcaq, vals)
+
+ if check.rowcount > 0:
+ # This should NOT happen. We should have added contents
+ # during process-unchecked. if it did, log an error, and send
+ # an email.
+ subst = {
+ "__PACKAGE__": package,
+ "__VERSION__": version,
+ "__ARCH__": arch,
+ "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
+ "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
+
+ message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
+ utils.send_mail(message)
+
+ # Temporarily disable contents storage until we re-do the table layout
+ #exists = Binary(deb, reject).scan_package()
+
+ if exists:
+ sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
+ SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations
+ WHERE package=:package AND version=:version AND architecture=:archid"""
+ session.execute(sql, vals)
+
+ sql = """DELETE from pending_content_associations
+ WHERE package=:package AND version=:version AND architecture=:archid"""
+ session.execute(sql, vals)
+ session.commit()
+
+ if privatetrans:
+ session.close()
+
+ return exists
+__all__.append('copy_temporary_contents')
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Changes class for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+import os
+import stat
+from cPickle import Unpickler, Pickler
+from errno import EPERM
+
+from apt_inst import debExtractControl
+from apt_pkg import ParseSection
+
+from utils import open_file, fubar, poolify
+
+###############################################################################
+
+__all__ = []
+
+###############################################################################
+
+CHANGESFIELDS_MANDATORY = [ "distribution", "source", "architecture",
+ "version", "maintainer", "urgency", "fingerprint", "changedby822",
+ "changedby2047", "changedbyname", "maintainer822", "maintainer2047",
+ "maintainername", "maintaineremail", "closes", "changes" ]
+
+__all__.append('CHANGESFIELDS_MANDATORY')
+
+CHANGESFIELDS_OPTIONAL = [ "changed-by", "filecontents", "format",
+ "process-new note", "adv id", "distribution-version", "sponsoremail" ]
+
+__all__.append('CHANGESFIELDS_OPTIONAL')
+
+CHANGESFIELDS_FILES = [ "package", "version", "architecture", "type", "size",
+ "md5sum", "sha1sum", "sha256sum", "component", "location id",
+ "source package", "source version", "maintainer", "dbtype", "files id",
+ "new", "section", "priority", "othercomponents", "pool name",
+ "original component" ]
+
+__all__.append('CHANGESFIELDS_FILES')
+
+CHANGESFIELDS_DSC = [ "source", "version", "maintainer", "fingerprint",
+ "uploaders", "bts changelog", "dm-upload-allowed" ]
+
+__all__.append('CHANGESFIELDS_DSC')
+
+CHANGESFIELDS_DSCFILES_MANDATORY = [ "size", "md5sum" ]
+
+__all__.append('CHANGESFIELDS_DSCFILES_MANDATORY')
+
+CHANGESFIELDS_DSCFILES_OPTIONAL = [ "files id" ]
+
+__all__.append('CHANGESFIELDS_DSCFILES_OPTIONAL')
+
+CHANGESFIELDS_ORIGFILES = [ "id", "location" ]
+
+__all__.append('CHANGESFIELDS_ORIGFILES')
+
+###############################################################################
+
+class Changes(object):
+ """ Convenience wrapper to carry around all the package information """
+
+ def __init__(self, **kwds):
+ self.reset()
+
+ def reset(self):
+ self.changes_file = ""
+
+ self.changes = {}
+ self.dsc = {}
+ self.files = {}
+ self.dsc_files = {}
+ self.orig_files = {}
+
+ def file_summary(self):
+ # changes["distribution"] may not exist in corner cases
+ # (e.g. unreadable changes files)
+ if not self.changes.has_key("distribution") or not \
+ isinstance(self.changes["distribution"], dict):
+ self.changes["distribution"] = {}
+
+ byhand = False
+ new = False
+ summary = ""
+ override_summary = ""
+
+ for name, entry in sorted(self.files.items()):
+ if entry.has_key("byhand"):
+ byhand = True
+ summary += name + " byhand\n"
+
+ elif entry.has_key("new"):
+ new = True
+ summary += "(new) %s %s %s\n" % (name, entry["priority"], entry["section"])
+
+ if entry.has_key("othercomponents"):
+ summary += "WARNING: Already present in %s distribution.\n" % (entry["othercomponents"])
+
+ if entry["type"] == "deb":
+ deb_fh = open_file(name)
+ summary += ParseSection(debExtractControl(deb_fh))["Description"] + '\n'
+ deb_fh.close()
+
+ else:
+ entry["pool name"] = poolify(self.changes.get("source", ""), entry["component"])
+ destination = entry["pool name"] + name
+ summary += name + "\n to " + destination + "\n"
+
+ if not entry.has_key("type"):
+ entry["type"] = "unknown"
+
+ if entry["type"] in ["deb", "udeb", "dsc"]:
+ # (queue/unchecked), there we have override entries already, use them
+ # (process-new), there we dont have override entries, use the newly generated ones.
+ override_prio = entry.get("override priority", entry["priority"])
+ override_sect = entry.get("override section", entry["section"])
+ override_summary += "%s - %s %s\n" % (name, override_prio, override_sect)
+
+ return (byhand, new, summary, override_summary)
+
+ def check_override(self):
+ """
+ Checks override entries for validity.
+
+ Returns an empty string if there are no problems
+ or the text of a warning if there are
+ """
+
+ summary = ""
+
+ # Abandon the check if it's a non-sourceful upload
+ if not self.changes["architecture"].has_key("source"):
+ return summary
+
+ for name, entry in sorted(self.files.items()):
+ if not entry.has_key("new") and entry["type"] == "deb":
+ if entry["section"] != "-":
+ if entry["section"].lower() != entry["override section"].lower():
+ summary += "%s: package says section is %s, override says %s.\n" % (name,
+ entry["section"],
+ entry["override section"])
+
+ if entry["priority"] != "-":
+ if entry["priority"] != entry["override priority"]:
+ summary += "%s: package says priority is %s, override says %s.\n" % (name,
+ entry["priority"],
+ entry["override priority"])
+
+ return summary
+
+
+ def load_dot_dak(self, changesfile):
+ """
+ Update ourself by reading a previously created cPickle .dak dumpfile.
+ """
+
+ self.changes_file = changesfile
+ dump_filename = self.changes_file[:-8]+".dak"
+ dump_file = open_file(dump_filename)
+
+ p = Unpickler(dump_file)
+
+ self.changes.update(p.load())
+ self.dsc.update(p.load())
+ self.files.update(p.load())
+ self.dsc_files.update(p.load())
+
+ next_obj = p.load()
+ if isinstance(next_obj, dict):
+ self.orig_files.update(next_obj)
+ else:
+ # Auto-convert old dak files to new format supporting
+ # multiple tarballs
+ orig_tar_gz = None
+ for dsc_file in self.dsc_files.keys():
+ if dsc_file.endswith(".orig.tar.gz"):
+ orig_tar_gz = dsc_file
+ self.orig_files[orig_tar_gz] = {}
+ if next_obj != None:
+ self.orig_files[orig_tar_gz]["id"] = next_obj
+ next_obj = p.load()
+ if next_obj != None and next_obj != "":
+ self.orig_files[orig_tar_gz]["location"] = next_obj
+ if len(self.orig_files[orig_tar_gz]) == 0:
+ del self.orig_files[orig_tar_gz]
+
+ dump_file.close()
+
+ def sanitised_files(self):
+ ret = {}
+ for name, entry in self.files.items():
+ ret[name] = {}
+ for i in CHANGESFIELDS_FILES:
+ if entry.has_key(i):
+ ret[name][i] = entry[i]
+
+ return ret
+
+ def sanitised_changes(self):
+ ret = {}
+ # Mandatory changes fields
+ for i in CHANGESFIELDS_MANDATORY:
+ ret[i] = self.changes[i]
+
+ # Optional changes fields
+ for i in CHANGESFIELDS_OPTIONAL:
+ if self.changes.has_key(i):
+ ret[i] = self.changes[i]
+
+ return ret
+
+ def sanitised_dsc(self):
+ ret = {}
+ for i in CHANGESFIELDS_DSC:
+ if self.dsc.has_key(i):
+ ret[i] = self.dsc[i]
+
+ return ret
+
+ def sanitised_dsc_files(self):
+ ret = {}
+ for name, entry in self.dsc_files.items():
+ ret[name] = {}
+ # Mandatory dsc_files fields
+ for i in CHANGESFIELDS_DSCFILES_MANDATORY:
+ ret[name][i] = entry[i]
+
+ # Optional dsc_files fields
+ for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
+ if entry.has_key(i):
+ ret[name][i] = entry[i]
+
+ return ret
+
+ def sanitised_orig_files(self):
+ ret = {}
+ for name, entry in self.orig_files.items():
+ ret[name] = {}
+ # Optional orig_files fields
+ for i in CHANGESFIELDS_ORIGFILES:
+ if entry.has_key(i):
+ ret[name][i] = entry[i]
+
+ return ret
+
+ def write_dot_dak(self, dest_dir):
+ """
+ Dump ourself into a cPickle file.
+
+ @type dest_dir: string
+ @param dest_dir: Path where the dumpfile should be stored
+
+ @note: This could just dump the dictionaries as is, but I'd like to avoid this so
+ there's some idea of what process-accepted & process-new use from
+ process-unchecked. (JT)
+
+ """
+
+ dump_filename = os.path.join(dest_dir, self.changes_file[:-8] + ".dak")
+ dump_file = open_file(dump_filename, 'w')
+
+ try:
+ os.chmod(dump_filename, 0664)
+ except OSError, e:
+ # chmod may fail when the dumpfile is not owned by the user
+ # invoking dak (like e.g. when NEW is processed by a member
+ # of ftpteam)
+ if e.errno == EPERM:
+ perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE])
+ # security precaution, should never happen unless a weird
+ # umask is set anywhere
+ if perms & stat.S_IWOTH:
+ fubar("%s is world writable and chmod failed." % \
+ (dump_filename,))
+ # ignore the failed chmod otherwise as the file should
+ # already have the right privileges and is just, at worst,
+ # unreadable for world
+ else:
+ raise
+
+ p = Pickler(dump_file, 1)
+
+ p.dump(self.sanitised_changes())
+ p.dump(self.sanitised_dsc())
+ p.dump(self.sanitised_files())
+ p.dump(self.sanitised_dsc_files())
+ p.dump(self.sanitised_orig_files())
+
+ dump_file.close()
+
+ def unknown_files_fields(self, name):
+ return sorted(list( set(self.files[name].keys()) -
+ set(CHANGESFIELDS_FILES)))
+
+ def unknown_changes_fields(self):
+ return sorted(list( set(self.changes.keys()) -
+ set(CHANGESFIELDS_MANDATORY + CHANGESFIELDS_OPTIONAL)))
+
+ def unknown_dsc_fields(self):
+ return sorted(list( set(self.dsc.keys()) -
+ set(CHANGESFIELDS_DSC)))
+
+ def unknown_dsc_files_fields(self, name):
+ return sorted(list( set(self.dsc_files[name].keys()) -
+ set(CHANGESFIELDS_DSCFILES_MANDATORY + CHANGESFIELDS_DSCFILES_OPTIONAL)))
+
+ def str_files(self):
+ r = []
+ for name, entry in self.files.items():
+ r.append(" %s:" % (name))
+ for i in CHANGESFIELDS_FILES:
+ if entry.has_key(i):
+ r.append(" %s: %s" % (i.capitalize(), entry[i]))
+ xfields = self.unknown_files_fields(name)
+ if len(xfields) > 0:
+ r.append("files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
+
+ return r
+
+ def str_changes(self):
+ r = []
+ for i in CHANGESFIELDS_MANDATORY:
+ val = self.changes[i]
+ if isinstance(val, list):
+ val = " ".join(val)
+ elif isinstance(val, dict):
+ val = " ".join(val.keys())
+ r.append(' %s: %s' % (i.capitalize(), val))
+
+ for i in CHANGESFIELDS_OPTIONAL:
+ if self.changes.has_key(i):
+ r.append(' %s: %s' % (i.capitalize(), self.changes[i]))
+
+ xfields = self.unknown_changes_fields()
+ if len(xfields) > 0:
+ r.append("Warning: changes still has the following unrecognised fields: %s" % ", ".join(xfields))
+
+ return r
+
+ def str_dsc(self):
+ r = []
+ for i in CHANGESFIELDS_DSC:
+ if self.dsc.has_key(i):
+ r.append(' %s: %s' % (i.capitalize(), self.dsc[i]))
+
+ xfields = self.unknown_dsc_fields()
+ if len(xfields) > 0:
+ r.append("Warning: dsc still has the following unrecognised fields: %s" % ", ".join(xfields))
+
+ return r
+
+ def str_dsc_files(self):
+ r = []
+ for name, entry in self.dsc_files.items():
+ r.append(" %s:" % (name))
+ for i in CHANGESFIELDS_DSCFILES_MANDATORY:
+ r.append(" %s: %s" % (i.capitalize(), entry[i]))
+ for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
+ if entry.has_key(i):
+ r.append(" %s: %s" % (i.capitalize(), entry[i]))
+ xfields = self.unknown_dsc_files_fields(name)
+ if len(xfields) > 0:
+ r.append("dsc_files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
+
+ return r
+
+ def __str__(self):
+ r = []
+
+ r.append(" Changes:")
+ r += self.str_changes()
+
+ r.append("")
+
+ r.append(" Dsc:")
+ r += self.str_dsc()
+
+ r.append("")
+
+ r.append(" Files:")
+ r += self.str_files()
+
+ r.append("")
+
+ r.append(" Dsc Files:")
+ r += self.str_dsc_files()
+
+ return "\n".join(r)
+
+__all__.append('Changes')
self.get = self.Cnf.get
self.SubTree = self.Cnf.SubTree
self.ValueList = self.Cnf.ValueList
+ self.Find = self.Cnf.Find
+ self.FindB = self.Cnf.FindB
def _startup(self, *args, **kwargs):
self._readconf()
def __getitem__(self, name):
return self.Cnf[name]
+ def __setitem__(self, name, value):
+ self.Cnf[name] = value
--- /dev/null
+#!/usr/bin/env python
+
+"""
+Logging functions
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001, 2002, 2006 James Troup <james@nocrew.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import os
+import pwd
+import time
+import sys
+import utils
+
+################################################################################
+
+class Logger:
+ "Logger object"
+ Cnf = None
+ logfile = None
+ program = None
+
+ def __init__ (self, Cnf, program, debug=0):
+ "Initialize a new Logger object"
+ self.Cnf = Cnf
+ self.program = program
+ # Create the log directory if it doesn't exist
+ logdir = Cnf["Dir::Log"]
+ if not os.path.exists(logdir):
+ umask = os.umask(00000)
+ os.makedirs(logdir, 02775)
+ os.umask(umask)
+ # Open the logfile
+ logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m"))
+ logfile = None
+ if debug:
+ logfile = sys.stderr
+ else:
+ umask = os.umask(00002)
+ logfile = utils.open_file(logfilename, 'a')
+ os.umask(umask)
+ self.logfile = logfile
+ self.log(["program start"])
+
+ def log (self, details):
+ "Log an event"
+ # Prepend timestamp, program name, and user name
+ details.insert(0, utils.getusername())
+ details.insert(0, self.program)
+ timestamp = time.strftime("%Y%m%d%H%M%S")
+ details.insert(0, timestamp)
+ # Force the contents of the list to be string.join-able
+ details = [ str(i) for i in details ]
+ # Write out the log in TSV
+ self.logfile.write("|".join(details)+'\n')
+ # Flush the output to enable tail-ing
+ self.logfile.flush()
+
+ def close (self):
+ "Close a Logger object"
+ self.log(["program end"])
+ self.logfile.flush()
+ self.logfile.close()
+++ /dev/null
-#!/usr/bin/env python
-
-""" DB access functions
-@group readonly: get_suite_id, get_section_id, get_priority_id, get_override_type_id,
- get_architecture_id, get_archive_id, get_component_id, get_location_id,
- get_source_id, get_suite_version, get_files_id, get_maintainer, get_suites,
- get_suite_architectures, get_new_comments, has_new_comment
-@group read/write: get_or_set*, set_files_id
-@group writeonly: add_new_comment, delete_new_comments
-
-@contact: Debian FTP Master <ftpmaster@debian.org>
-@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
-@copyright: 2009 Joerg Jaspert <joerg@debian.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-################################################################################
-
-import sys
-import time
-import types
-import utils
-import pg
-from binary import Binary
-
-################################################################################
-
-Cnf = None #: Configuration, apt_pkg.Configuration
-projectB = None #: database connection, pgobject
-suite_id_cache = {} #: cache for suites
-section_id_cache = {} #: cache for sections
-priority_id_cache = {} #: cache for priorities
-override_type_id_cache = {} #: cache for overrides
-architecture_id_cache = {} #: cache for architectures
-archive_id_cache = {} #: cache for archives
-component_id_cache = {} #: cache for components
-location_id_cache = {} #: cache for locations
-maintainer_id_cache = {} #: cache for maintainers
-keyring_id_cache = {} #: cache for keyrings
-source_id_cache = {} #: cache for sources
-
-files_id_cache = {} #: cache for files
-maintainer_cache = {} #: cache for maintainer names
-fingerprint_id_cache = {} #: cache for fingerprints
-queue_id_cache = {} #: cache for queues
-uid_id_cache = {} #: cache for uids
-suite_version_cache = {} #: cache for suite_versions (packages)
-suite_bin_version_cache = {}
-cache_preloaded = False
-
-################################################################################
-
-def init (config, sql):
- """
- database module init.
-
- @type config: apt_pkg.Configuration
- @param config: apt config, see U{http://apt.alioth.debian.org/python-apt-doc/apt_pkg/cache.html#Configuration}
-
- @type sql: pgobject
- @param sql: database connection
-
- """
- global Cnf, projectB
-
- Cnf = config
- projectB = sql
-
-
-def do_query(query):
- """
- Executes a database query. Writes statistics / timing to stderr.
-
- @type query: string
- @param query: database query string, passed unmodified
-
- @return: db result
-
- @warning: The query is passed B{unmodified}, so be careful what you use this for.
- """
- sys.stderr.write("query: \"%s\" ... " % (query))
- before = time.time()
- r = projectB.query(query)
- time_diff = time.time()-before
- sys.stderr.write("took %.3f seconds.\n" % (time_diff))
- if type(r) is int:
- sys.stderr.write("int result: %s\n" % (r))
- elif type(r) is types.NoneType:
- sys.stderr.write("result: None\n")
- else:
- sys.stderr.write("pgresult: %s\n" % (r.getresult()))
- return r
-
-################################################################################
-
-def get_suite_id (suite):
- """
- Returns database id for given C{suite}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type suite: string
- @param suite: The name of the suite
-
- @rtype: int
- @return: the database id for the given suite
-
- """
- global suite_id_cache
-
- if suite_id_cache.has_key(suite):
- return suite_id_cache[suite]
-
- q = projectB.query("SELECT id FROM suite WHERE suite_name = '%s'" % (suite))
- ql = q.getresult()
- if not ql:
- return -1
-
- suite_id = ql[0][0]
- suite_id_cache[suite] = suite_id
-
- return suite_id
-
-def get_section_id (section):
- """
- Returns database id for given C{section}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type section: string
- @param section: The name of the section
-
- @rtype: int
- @return: the database id for the given section
-
- """
- global section_id_cache
-
- if section_id_cache.has_key(section):
- return section_id_cache[section]
-
- q = projectB.query("SELECT id FROM section WHERE section = '%s'" % (section))
- ql = q.getresult()
- if not ql:
- return -1
-
- section_id = ql[0][0]
- section_id_cache[section] = section_id
-
- return section_id
-
-def get_priority_id (priority):
- """
- Returns database id for given C{priority}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type priority: string
- @param priority: The name of the priority
-
- @rtype: int
- @return: the database id for the given priority
-
- """
- global priority_id_cache
-
- if priority_id_cache.has_key(priority):
- return priority_id_cache[priority]
-
- q = projectB.query("SELECT id FROM priority WHERE priority = '%s'" % (priority))
- ql = q.getresult()
- if not ql:
- return -1
-
- priority_id = ql[0][0]
- priority_id_cache[priority] = priority_id
-
- return priority_id
-
-def get_override_type_id (type):
- """
- Returns database id for given override C{type}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type type: string
- @param type: The name of the override type
-
- @rtype: int
- @return: the database id for the given override type
-
- """
- global override_type_id_cache
-
- if override_type_id_cache.has_key(type):
- return override_type_id_cache[type]
-
- q = projectB.query("SELECT id FROM override_type WHERE type = '%s'" % (type))
- ql = q.getresult()
- if not ql:
- return -1
-
- override_type_id = ql[0][0]
- override_type_id_cache[type] = override_type_id
-
- return override_type_id
-
-def get_architecture_id (architecture):
- """
- Returns database id for given C{architecture}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type architecture: string
- @param architecture: The name of the override type
-
- @rtype: int
- @return: the database id for the given architecture
-
- """
- global architecture_id_cache
-
- if architecture_id_cache.has_key(architecture):
- return architecture_id_cache[architecture]
-
- q = projectB.query("SELECT id FROM architecture WHERE arch_string = '%s'" % (architecture))
- ql = q.getresult()
- if not ql:
- return -1
-
- architecture_id = ql[0][0]
- architecture_id_cache[architecture] = architecture_id
-
- return architecture_id
-
-def get_archive_id (archive):
- """
- Returns database id for given C{archive}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type archive: string
- @param archive: The name of the override type
-
- @rtype: int
- @return: the database id for the given archive
-
- """
- global archive_id_cache
-
- archive = archive.lower()
-
- if archive_id_cache.has_key(archive):
- return archive_id_cache[archive]
-
- q = projectB.query("SELECT id FROM archive WHERE lower(name) = '%s'" % (archive))
- ql = q.getresult()
- if not ql:
- return -1
-
- archive_id = ql[0][0]
- archive_id_cache[archive] = archive_id
-
- return archive_id
-
-def get_component_id (component):
- """
- Returns database id for given C{component}.
- Results are kept in a cache during runtime to minimize database queries.
-
- @type component: string
- @param component: The name of the component
-
- @rtype: int
- @return: the database id for the given component
-
- """
- global component_id_cache
-
- component = component.lower()
-
- if component_id_cache.has_key(component):
- return component_id_cache[component]
-
- q = projectB.query("SELECT id FROM component WHERE lower(name) = '%s'" % (component))
- ql = q.getresult()
- if not ql:
- return -1
-
- component_id = ql[0][0]
- component_id_cache[component] = component_id
-
- return component_id
-
-def get_location_id (location, component, archive):
- """
- Returns database id for the location behind the given combination of
- - B{location} - the path of the location, eg. I{/srv/ftp.debian.org/ftp/pool/}
- - B{component} - the id of the component as returned by L{get_component_id}
- - B{archive} - the id of the archive as returned by L{get_archive_id}
- Results are kept in a cache during runtime to minimize database queries.
-
- @type location: string
- @param location: the path of the location
-
- @type component: int
- @param component: the id of the component
-
- @type archive: int
- @param archive: the id of the archive
-
- @rtype: int
- @return: the database id for the location
-
- """
- global location_id_cache
-
- cache_key = location + '_' + component + '_' + location
- if location_id_cache.has_key(cache_key):
- return location_id_cache[cache_key]
-
- archive_id = get_archive_id (archive)
- if component != "":
- component_id = get_component_id (component)
- if component_id != -1:
- q = projectB.query("SELECT id FROM location WHERE path = '%s' AND component = %d AND archive = %d" % (location, component_id, archive_id))
- else:
- q = projectB.query("SELECT id FROM location WHERE path = '%s' AND archive = %d" % (location, archive_id))
- ql = q.getresult()
- if not ql:
- return -1
-
- location_id = ql[0][0]
- location_id_cache[cache_key] = location_id
-
- return location_id
-
-def get_source_id (source, version):
- """
- Returns database id for the combination of C{source} and C{version}
- - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{version}
- Results are kept in a cache during runtime to minimize database queries.
-
- @type source: string
- @param source: source package name
-
- @type version: string
- @param version: the source version
-
- @rtype: int
- @return: the database id for the source
-
- """
- global source_id_cache
-
- cache_key = source + '_' + version + '_'
- if source_id_cache.has_key(cache_key):
- return source_id_cache[cache_key]
-
- q = projectB.query("SELECT id FROM source s WHERE s.source = '%s' AND s.version = '%s'" % (source, version))
-
- if not q.getresult():
- return None
-
- source_id = q.getresult()[0][0]
- source_id_cache[cache_key] = source_id
-
- return source_id
-
-def get_suite_version(source, suite):
- """
- Returns database id for a combination of C{source} and C{suite}.
-
- - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{suite} - a suite name, eg. I{unstable}
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type source: string
- @param source: source package name
-
- @type suite: string
- @param suite: the suite name
-
- @rtype: string
- @return: the version for I{source} in I{suite}
-
- """
-
- global suite_version_cache
- cache_key = "%s_%s" % (source, suite)
-
- if suite_version_cache.has_key(cache_key):
- return suite_version_cache[cache_key]
-
- q = projectB.query("""
- SELECT s.version FROM source s, suite su, src_associations sa
- WHERE sa.source=s.id
- AND sa.suite=su.id
- AND su.suite_name='%s'
- AND s.source='%s'"""
- % (suite, source))
-
- if not q.getresult():
- return None
-
- version = q.getresult()[0][0]
- suite_version_cache[cache_key] = version
-
- return version
-
-def get_latest_binary_version_id(binary, section, suite, arch):
- global suite_bin_version_cache
- cache_key = "%s_%s_%s_%s" % (binary, section, suite, arch)
- cache_key_all = "%s_%s_%s_%s" % (binary, section, suite, get_architecture_id("all"))
-
- # Check for the cache hit for its arch, then arch all
- if suite_bin_version_cache.has_key(cache_key):
- return suite_bin_version_cache[cache_key]
- if suite_bin_version_cache.has_key(cache_key_all):
- return suite_bin_version_cache[cache_key_all]
- if cache_preloaded == True:
- return # package does not exist
-
- q = projectB.query("SELECT DISTINCT b.id FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.package = '%s' AND b.architecture = '%d' AND ba.suite = '%d' AND o.section = '%d'" % (binary, int(arch), int(suite), int(section)))
-
- if not q.getresult():
- return False
-
- highest_bid = q.getresult()[0][0]
-
- suite_bin_version_cache[cache_key] = highest_bid
- return highest_bid
-
-def preload_binary_id_cache():
- global suite_bin_version_cache, cache_preloaded
-
- # Get suite info
- q = projectB.query("SELECT id FROM suite")
- suites = q.getresult()
-
- # Get arch mappings
- q = projectB.query("SELECT id FROM architecture")
- arches = q.getresult()
-
- for suite in suites:
- for arch in arches:
- q = projectB.query("SELECT DISTINCT b.id, b.package, o.section FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN override o ON (o.package=b.package) WHERE b.architecture = '%d' AND ba.suite = '%d'" % (int(arch[0]), int(suite[0])))
-
- for bi in q.getresult():
- cache_key = "%s_%s_%s_%s" % (bi[1], bi[2], suite[0], arch[0])
- suite_bin_version_cache[cache_key] = int(bi[0])
-
- cache_preloaded = True
-
-def get_suite_architectures(suite):
- """
- Returns list of architectures for C{suite}.
-
- @type suite: string, int
- @param suite: the suite name or the suite_id
-
- @rtype: list
- @return: the list of architectures for I{suite}
- """
-
- suite_id = None
- if type(suite) == str:
- suite_id = get_suite_id(suite)
- elif type(suite) == int:
- suite_id = suite
- else:
- return None
-
- sql = """ SELECT a.arch_string FROM suite_architectures sa
- JOIN architecture a ON (a.id = sa.architecture)
- WHERE suite='%s' """ % (suite_id)
-
- q = projectB.query(sql)
- return map(lambda x: x[0], q.getresult())
-
-def get_suite_untouchable(suite):
- """
- Returns true if the C{suite} is untouchable, otherwise false.
-
- @type suite: string, int
- @param suite: the suite name or the suite_id
-
- @rtype: boolean
- @return: status of suite
- """
-
- suite_id = None
- if type(suite) == str:
- suite_id = get_suite_id(suite.lower())
- elif type(suite) == int:
- suite_id = suite
- else:
- return None
-
- sql = """ SELECT untouchable FROM suite WHERE id='%s' """ % (suite_id)
-
- q = projectB.query(sql)
- if q.getresult()[0][0] == "f":
- return False
- else:
- return True
-
-################################################################################
-
-def get_or_set_maintainer_id (maintainer):
- """
- If C{maintainer} does not have an entry in the maintainer table yet, create one
- and return the new id.
- If C{maintainer} already has an entry, simply return the existing id.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type maintainer: string
- @param maintainer: the maintainer name
-
- @rtype: int
- @return: the database id for the maintainer
-
- """
- global maintainer_id_cache
-
- if maintainer_id_cache.has_key(maintainer):
- return maintainer_id_cache[maintainer]
-
- q = projectB.query("SELECT id FROM maintainer WHERE name = '%s'" % (maintainer))
- if not q.getresult():
- projectB.query("INSERT INTO maintainer (name) VALUES ('%s')" % (maintainer))
- q = projectB.query("SELECT id FROM maintainer WHERE name = '%s'" % (maintainer))
- maintainer_id = q.getresult()[0][0]
- maintainer_id_cache[maintainer] = maintainer_id
-
- return maintainer_id
-
-################################################################################
-
-def get_or_set_keyring_id (keyring):
- """
- If C{keyring} does not have an entry in the C{keyrings} table yet, create one
- and return the new id.
- If C{keyring} already has an entry, simply return the existing id.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type keyring: string
- @param keyring: the keyring name
-
- @rtype: int
- @return: the database id for the keyring
-
- """
- global keyring_id_cache
-
- if keyring_id_cache.has_key(keyring):
- return keyring_id_cache[keyring]
-
- q = projectB.query("SELECT id FROM keyrings WHERE name = '%s'" % (keyring))
- if not q.getresult():
- projectB.query("INSERT INTO keyrings (name) VALUES ('%s')" % (keyring))
- q = projectB.query("SELECT id FROM keyrings WHERE name = '%s'" % (keyring))
- keyring_id = q.getresult()[0][0]
- keyring_id_cache[keyring] = keyring_id
-
- return keyring_id
-
-################################################################################
-
-def get_or_set_uid_id (uid):
- """
- If C{uid} does not have an entry in the uid table yet, create one
- and return the new id.
- If C{uid} already has an entry, simply return the existing id.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type uid: string
- @param uid: the uid.
-
- @rtype: int
- @return: the database id for the uid
-
- """
-
- global uid_id_cache
-
- if uid_id_cache.has_key(uid):
- return uid_id_cache[uid]
-
- q = projectB.query("SELECT id FROM uid WHERE uid = '%s'" % (uid))
- if not q.getresult():
- projectB.query("INSERT INTO uid (uid) VALUES ('%s')" % (uid))
- q = projectB.query("SELECT id FROM uid WHERE uid = '%s'" % (uid))
- uid_id = q.getresult()[0][0]
- uid_id_cache[uid] = uid_id
-
- return uid_id
-
-################################################################################
-
-def get_or_set_fingerprint_id (fingerprint):
- """
- If C{fingerprint} does not have an entry in the fingerprint table yet, create one
- and return the new id.
- If C{fingerprint} already has an entry, simply return the existing id.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type fingerprint: string
- @param fingerprint: the fingerprint
-
- @rtype: int
- @return: the database id for the fingerprint
-
- """
- global fingerprint_id_cache
-
- if fingerprint_id_cache.has_key(fingerprint):
- return fingerprint_id_cache[fingerprint]
-
- q = projectB.query("SELECT id FROM fingerprint WHERE fingerprint = '%s'" % (fingerprint))
- if not q.getresult():
- projectB.query("INSERT INTO fingerprint (fingerprint) VALUES ('%s')" % (fingerprint))
- q = projectB.query("SELECT id FROM fingerprint WHERE fingerprint = '%s'" % (fingerprint))
- fingerprint_id = q.getresult()[0][0]
- fingerprint_id_cache[fingerprint] = fingerprint_id
-
- return fingerprint_id
-
-################################################################################
-
-def get_files_id (filename, size, md5sum, location_id):
- """
- Returns -1, -2 or the file_id for filename, if its C{size} and C{md5sum} match an
- existing copy.
-
- The database is queried using the C{filename} and C{location_id}. If a file does exist
- at that location, the existing size and md5sum are checked against the provided
- parameters. A size or checksum mismatch returns -2. If more than one entry is
- found within the database, a -1 is returned, no result returns None, otherwise
- the file id.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type filename: string
- @param filename: the filename of the file to check against the DB
-
- @type size: int
- @param size: the size of the file to check against the DB
-
- @type md5sum: string
- @param md5sum: the md5sum of the file to check against the DB
-
- @type location_id: int
- @param location_id: the id of the location as returned by L{get_location_id}
-
- @rtype: int / None
- @return: Various return values are possible:
- - -2: size/checksum error
- - -1: more than one file found in database
- - None: no file found in database
- - int: file id
-
- """
- global files_id_cache
-
- cache_key = "%s_%d" % (filename, location_id)
-
- if files_id_cache.has_key(cache_key):
- return files_id_cache[cache_key]
-
- size = int(size)
- q = projectB.query("SELECT id, size, md5sum FROM files WHERE filename = '%s' AND location = %d" % (filename, location_id))
- ql = q.getresult()
- if ql:
- if len(ql) != 1:
- return -1
- ql = ql[0]
- orig_size = int(ql[1])
- orig_md5sum = ql[2]
- if orig_size != size or orig_md5sum != md5sum:
- return -2
- files_id_cache[cache_key] = ql[0]
- return files_id_cache[cache_key]
- else:
- return None
-
-################################################################################
-
-def get_or_set_queue_id (queue):
- """
- If C{queue} does not have an entry in the queue table yet, create one
- and return the new id.
- If C{queue} already has an entry, simply return the existing id.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type queue: string
- @param queue: the queue name (no full path)
-
- @rtype: int
- @return: the database id for the queue
-
- """
- global queue_id_cache
-
- if queue_id_cache.has_key(queue):
- return queue_id_cache[queue]
-
- q = projectB.query("SELECT id FROM queue WHERE queue_name = '%s'" % (queue))
- if not q.getresult():
- projectB.query("INSERT INTO queue (queue_name) VALUES ('%s')" % (queue))
- q = projectB.query("SELECT id FROM queue WHERE queue_name = '%s'" % (queue))
- queue_id = q.getresult()[0][0]
- queue_id_cache[queue] = queue_id
-
- return queue_id
-
-################################################################################
-
-def set_files_id (filename, size, md5sum, sha1sum, sha256sum, location_id):
- """
- Insert a new entry into the files table and return its id.
-
- @type filename: string
- @param filename: the filename
-
- @type size: int
- @param size: the size in bytes
-
- @type md5sum: string
- @param md5sum: md5sum of the file
-
- @type sha1sum: string
- @param sha1sum: sha1sum of the file
-
- @type sha256sum: string
- @param sha256sum: sha256sum of the file
-
- @type location_id: int
- @param location_id: the id of the location as returned by L{get_location_id}
-
- @rtype: int
- @return: the database id for the new file
-
- """
- global files_id_cache
-
- projectB.query("INSERT INTO files (filename, size, md5sum, sha1sum, sha256sum, location) VALUES ('%s', %d, '%s', '%s', '%s', %d)" % (filename, long(size), md5sum, sha1sum, sha256sum, location_id))
-
- return get_files_id (filename, size, md5sum, location_id)
-
- ### currval has issues with postgresql 7.1.3 when the table is big
- ### it was taking ~3 seconds to return on auric which is very Not
- ### Cool(tm).
- ##
- ##q = projectB.query("SELECT id FROM files WHERE id = currval('files_id_seq')")
- ##ql = q.getresult()[0]
- ##cache_key = "%s_%d" % (filename, location_id)
- ##files_id_cache[cache_key] = ql[0]
- ##return files_id_cache[cache_key]
-
-################################################################################
-
-def get_maintainer (maintainer_id):
- """
- Return the name of the maintainer behind C{maintainer_id}.
-
- Results are kept in a cache during runtime to minimize database queries.
-
- @type maintainer_id: int
- @param maintainer_id: the id of the maintainer, eg. from L{get_or_set_maintainer_id}
-
- @rtype: string
- @return: the name of the maintainer
-
- """
- global maintainer_cache
-
- if not maintainer_cache.has_key(maintainer_id):
- q = projectB.query("SELECT name FROM maintainer WHERE id = %s" % (maintainer_id))
- maintainer_cache[maintainer_id] = q.getresult()[0][0]
-
- return maintainer_cache[maintainer_id]
-
-################################################################################
-
-def get_suites(pkgname, src=False):
- """
- Return the suites in which C{pkgname} can be found. If C{src} is True query for source
- package, else binary package.
-
- @type pkgname: string
- @param pkgname: name of the package
-
- @type src: bool
- @param src: if True look for source packages, false (default) looks for binary.
-
- @rtype: list
- @return: list of suites, or empty list if no match
-
- """
- if src:
- sql = """
- SELECT suite_name
- FROM source,
- src_associations,
- suite
- WHERE source.id = src_associations.source
- AND source.source = '%s'
- AND src_associations.suite = suite.id
- """ % (pkgname)
- else:
- sql = """
- SELECT suite_name
- FROM binaries,
- bin_associations,
- suite
- WHERE binaries.id = bin_associations.bin
- AND package = '%s'
- AND bin_associations.suite = suite.id
- """ % (pkgname)
-
- q = projectB.query(sql)
- return map(lambda x: x[0], q.getresult())
-
-
-################################################################################
-
-def get_new_comments(package):
- """
- Returns all the possible comments attached to C{package} in NEW. All versions.
-
- @type package: string
- @param package: name of the package
-
- @rtype: list
- @return: list of strings containing comments for all versions from all authors for package
- """
-
- comments = []
- query = projectB.query(""" SELECT version, comment, author, notedate
- FROM new_comments
- WHERE package = '%s'
- ORDER BY notedate
- """ % (package))
-
- for row in query.getresult():
- comments.append("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s\n" % (row[2], row[0], row[3], row[1]))
- comments.append("-"*72)
-
- return comments
-
-def has_new_comment(package, version, ignore_trainee=False):
- """
- Returns true if the given combination of C{package}, C{version} has a comment.
- If C{ignore_trainee} is true, comments from a trainee are ignored.
-
- @type package: string
- @param package: name of the package
-
- @type version: string
- @param version: package version
-
- @type ignore_trainee: boolean
- @param ignore_trainee: ignore trainee comments
-
- @rtype: boolean
- @return: true/false
- """
-
- trainee=""
- if ignore_trainee:
- trainee='AND trainee=false'
-
- exists = projectB.query("""SELECT 1 FROM new_comments
- WHERE package='%s'
- AND version='%s'
- %s
- LIMIT 1"""
- % (package, version, trainee) ).getresult()
-
- if not exists:
- return False
- else:
- return True
-
-def add_new_comment(package, version, comment, author, trainee=False):
- """
- Add a new comment for C{package}, C{version} written by C{author}
-
- @type package: string
- @param package: name of the package
-
- @type version: string
- @param version: package version
-
- @type comment: string
- @param comment: the comment
-
- @type author: string
- @param author: the authorname
-
- @type trainee: boolean
- @param trainee: trainee comment
- """
-
- projectB.query(""" INSERT INTO new_comments (package, version, comment, author, trainee)
- VALUES ('%s', '%s', '%s', '%s', '%s')
- """ % (package, version, pg.escape_string(comment), pg.escape_string(author), trainee))
-
- return
-
-def delete_new_comments(package, version):
- """
- Delete a comment for C{package}, C{version}, if one exists
- """
-
- projectB.query(""" DELETE FROM new_comments
- WHERE package = '%s' AND version = '%s'
- """ % (package, version))
- return
-
-def delete_all_new_comments(package):
- """
- Delete all comments for C{package}, if they exist
- """
-
- projectB.query(""" DELETE FROM new_comments
- WHERE package = '%s'
- """ % (package))
- return
-
-################################################################################
-def copy_temporary_contents(package, version, arch, deb, reject):
- """
- copy the previously stored contents from the temp table to the permanant one
-
- during process-unchecked, the deb should have been scanned and the
- contents stored in pending_content_associations
- """
-
- # first see if contents exist:
-
- arch_id = get_architecture_id (arch)
-
- exists = projectB.query("""SELECT 1 FROM pending_content_associations
- WHERE package='%s'
- AND version='%s'
- AND architecture=%d LIMIT 1"""
- % (package, version, arch_id) ).getresult()
-
- if not exists:
- # This should NOT happen. We should have added contents
- # during process-unchecked. if it did, log an error, and send
- # an email.
- subst = {
- "__PACKAGE__": package,
- "__VERSION__": version,
- "__ARCH__": arch,
- "__TO_ADDRESS__": Cnf["Dinstall::MyAdminAddress"],
- "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"] }
-
- message = utils.TemplateSubst(subst, Cnf["Dir::Templates"]+"/missing-contents")
- utils.send_mail( message )
-
- exists = Binary(deb, reject).scan_package()
-
- if exists:
- sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
- SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations
- WHERE package='%s'
- AND version='%s'
- AND architecture=%d""" % (package, version, arch_id)
- projectB.query(sql)
- projectB.query("""DELETE from pending_content_associations
- WHERE package='%s'
- AND version='%s'
- AND architecture=%d""" % (package, version, arch_id))
-
- return exists
import psycopg2
import traceback
-from singleton import Singleton
+from inspect import getargspec
+
+from sqlalchemy import create_engine, Table, MetaData
+from sqlalchemy.orm import sessionmaker, mapper, relation
+
+# Don't remove this, we re-export the exceptions to scripts which import us
+from sqlalchemy.exc import *
+from sqlalchemy.orm.exc import NoResultFound
+
+# Only import Config until Queue stuff is changed to store its config
+# in the database
from config import Config
+from singleton import Singleton
+from textutils import fix_maintainer
+
+################################################################################
+
+__all__ = ['IntegrityError', 'SQLAlchemyError']
################################################################################
-class Cache(object):
- def __init__(self, hashfunc=None):
- if hashfunc:
- self.hashfunc = hashfunc
+def session_wrapper(fn):
+ """
+ Wrapper around common ".., session=None):" handling. If the wrapped
+ function is called without passing 'session', we create a local one
+ and destroy it when the function ends.
+
+ Also attaches a commit_or_flush method to the session; if we created a
+ local session, this is a synonym for session.commit(), otherwise it is a
+ synonym for session.flush().
+ """
+
+ def wrapped(*args, **kwargs):
+ private_transaction = False
+
+ # Find the session object
+ session = kwargs.get('session')
+
+ if session is None:
+ if len(args) <= len(getargspec(fn)[0]) - 1:
+ # No session specified as last argument or in kwargs
+ private_transaction = True
+ session = kwargs['session'] = DBConn().session()
+ else:
+ # Session is last argument in args
+ session = args[-1]
+ if session is None:
+ args = list(args)
+ session = args[-1] = DBConn().session()
+ private_transaction = True
+
+ if private_transaction:
+ session.commit_or_flush = session.commit
else:
- self.hashfunc = lambda x: x['value']
+ session.commit_or_flush = session.flush
- self.data = {}
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ if private_transaction:
+ # We created a session; close it.
+ session.close()
- def SetValue(self, keys, value):
- self.data[self.hashfunc(keys)] = value
+ wrapped.__doc__ = fn.__doc__
+ wrapped.func_name = fn.func_name
- def GetValue(self, keys):
- return self.data.get(self.hashfunc(keys))
+ return wrapped
################################################################################
-class DBConn(Singleton):
+class Architecture(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.arch_string== val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.arch_string != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Architecture %s>' % self.arch_string
+
+__all__.append('Architecture')
+
+@session_wrapper
+def get_architecture(architecture, session=None):
"""
- database module init.
+ Returns database id for given C{architecture}.
+
+ @type architecture: string
+ @param architecture: The name of the architecture
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Architecture
+ @return: Architecture object for the given arch (None if not present)
+ """
+
+ q = session.query(Architecture).filter_by(arch_string=architecture)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_architecture')
+
+@session_wrapper
+def get_architecture_suites(architecture, session=None):
"""
+ Returns list of Suite objects for given C{architecture} name
+
+ @type source: str
+ @param source: Architecture name to search for
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of Suite objects for the given name (may be empty)
+ """
+
+ q = session.query(Suite)
+ q = q.join(SuiteArchitecture)
+ q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name')
+
+ ret = q.all()
+
+ return ret
+
+__all__.append('get_architecture_suites')
+
+################################################################################
+
+class Archive(object):
def __init__(self, *args, **kwargs):
- super(DBConn, self).__init__(*args, **kwargs)
+ pass
- def _startup(self, *args, **kwargs):
- self.__createconn()
- self.__init_caches()
+ def __repr__(self):
+ return '<Archive %s>' % self.archive_name
- ## Connection functions
- def __createconn(self):
- cnf = Config()
- connstr = "dbname=%s" % "projectbstew" #cnf["DB::Name"]
- print( "connstr: %s "% connstr)
- if cnf["DB::Host"]:
- connstr += " host=%s" % cnf["DB::Host"]
- if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
- connstr += " port=%s" % cnf["DB::Port"]
+__all__.append('Archive')
- self.db_con = psycopg2.connect(connstr)
+@session_wrapper
+def get_archive(archive, session=None):
+ """
+ returns database id for given C{archive}.
- def reconnect(self):
- try:
- self.db_con.close()
- except psycopg2.InterfaceError:
- pass
+ @type archive: string
+ @param archive: the name of the arhive
- self.db_con = None
- self.__createconn()
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
- ## Cache functions
- def __init_caches(self):
- self.caches = {'suite': Cache(),
- 'section': Cache(),
- 'priority': Cache(),
- 'override_type': Cache(),
- 'architecture': Cache(),
- 'archive': Cache(),
- 'component': Cache(),
- 'content_path_names': Cache(),
- 'content_file_names': Cache(),
- 'location': Cache(lambda x: '%s_%s_%s' % (x['location'], x['component'], x['location'])),
- 'maintainer': {}, # TODO
- 'keyring': {}, # TODO
- 'source': Cache(lambda x: '%s_%s_' % (x['source'], x['version'])),
- 'files': Cache(lambda x: '%s_%s_' % (x['filename'], x['location'])),
- 'maintainer': {}, # TODO
- 'fingerprint': {}, # TODO
- 'queue': {}, # TODO
- 'uid': {}, # TODO
- 'suite_version': Cache(lambda x: '%s_%s' % (x['source'], x['suite'])),
- }
-
- self.prepared_statements = {}
-
- def prepare(self,name,statement):
- if not self.prepared_statements.has_key(name):
- c = self.cursor()
- c.execute(statement)
- self.prepared_statements[name] = statement
-
- def clear_caches(self):
- self.__init_caches()
-
- ## Functions to pass through to the database connector
- def cursor(self):
- return self.db_con.cursor()
-
- def commit(self):
- return self.db_con.commit()
-
- ## Get functions
- def __get_single_id(self, query, values, cachename=None):
- # This is a bit of a hack but it's an internal function only
- if cachename is not None:
- res = self.caches[cachename].GetValue(values)
- if res:
- return res
-
- c = self.db_con.cursor()
- c.execute(query, values)
-
- if c.rowcount != 1:
- return None
-
- res = c.fetchone()[0]
-
- if cachename is not None:
- self.caches[cachename].SetValue(values, res)
-
- return res
-
- def __get_id(self, retfield, table, qfield, value):
- query = "SELECT %s FROM %s WHERE %s = %%(value)s" % (retfield, table, qfield)
- return self.__get_single_id(query, {'value': value}, cachename=table)
-
- def get_suite_id(self, suite):
- """
- Returns database id for given C{suite}.
- Results are kept in a cache during runtime to minimize database queries.
+ @rtype: Archive
+ @return: Archive object for the given name (None if not present)
- @type suite: string
- @param suite: The name of the suite
+ """
+ archive = archive.lower()
- @rtype: int
- @return: the database id for the given suite
+ q = session.query(Archive).filter_by(archive_name=archive)
- """
- suiteid = self.__get_id('id', 'suite', 'suite_name', suite)
- if suiteid is None:
- return None
- else:
- return int(suiteid)
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
- def get_section_id(self, section):
- """
- Returns database id for given C{section}.
- Results are kept in a cache during runtime to minimize database queries.
+__all__.append('get_archive')
+
+################################################################################
- @type section: string
- @param section: The name of the section
+class BinAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
- @rtype: int
- @return: the database id for the given section
+ def __repr__(self):
+ return '<BinAssociation %s (%s, %s)>' % (self.ba_id, self.binary, self.suite)
- """
- return self.__get_id('id', 'section', 'section', section)
+__all__.append('BinAssociation')
- def get_priority_id(self, priority):
- """
- Returns database id for given C{priority}.
- Results are kept in a cache during runtime to minimize database queries.
+################################################################################
- @type priority: string
- @param priority: The name of the priority
+class DBBinary(object):
+ def __init__(self, *args, **kwargs):
+ pass
- @rtype: int
- @return: the database id for the given priority
+ def __repr__(self):
+ return '<DBBinary %s (%s, %s)>' % (self.package, self.version, self.architecture)
- """
- return self.__get_id('id', 'priority', 'priority', priority)
+__all__.append('DBBinary')
- def get_override_type_id(self, override_type):
- """
- Returns database id for given override C{type}.
- Results are kept in a cache during runtime to minimize database queries.
+@session_wrapper
+def get_suites_binary_in(package, session=None):
+ """
+ Returns list of Suite objects which given C{package} name is in
- @type override_type: string
- @param override_type: The name of the override type
+ @type source: str
+ @param source: DBBinary package name to search for
- @rtype: int
- @return: the database id for the given override type
+ @rtype: list
+ @return: list of Suite objects for the given package
+ """
- """
- return self.__get_id('id', 'override_type', 'type', override_type)
+ return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all()
- def get_architecture_id(self, architecture):
- """
- Returns database id for given C{architecture}.
- Results are kept in a cache during runtime to minimize database queries.
+__all__.append('get_suites_binary_in')
- @type architecture: string
- @param architecture: The name of the override type
+@session_wrapper
+def get_binary_from_id(id, session=None):
+ """
+ Returns DBBinary object for given C{id}
- @rtype: int
- @return: the database id for the given architecture
+ @type id: int
+ @param id: Id of the required binary
- """
- return self.__get_id('id', 'architecture', 'arch_string', architecture)
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
- def get_archive_id(self, archive):
- """
- returns database id for given c{archive}.
- results are kept in a cache during runtime to minimize database queries.
+ @rtype: DBBinary
+ @return: DBBinary object for the given binary (None if not present)
+ """
- @type archive: string
- @param archive: the name of the override type
+ q = session.query(DBBinary).filter_by(binary_id=id)
- @rtype: int
- @return: the database id for the given archive
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
- """
- return self.__get_id('id', 'archive', 'lower(name)', archive)
+__all__.append('get_binary_from_id')
- def get_component_id(self, component):
- """
- Returns database id for given C{component}.
- Results are kept in a cache during runtime to minimize database queries.
+@session_wrapper
+def get_binaries_from_name(package, version=None, architecture=None, session=None):
+ """
+ Returns list of DBBinary objects for given C{package} name
- @type component: string
- @param component: The name of the override type
+ @type package: str
+ @param package: DBBinary package name to search for
- @rtype: int
- @return: the database id for the given component
+ @type version: str or None
+ @param version: Version to search for (or None)
- """
- return self.__get_id('id', 'component', 'lower(name)', component)
+ @type package: str, list or None
+ @param package: Architectures to limit to (or None if no limit)
- def get_location_id(self, location, component, archive):
- """
- Returns database id for the location behind the given combination of
- - B{location} - the path of the location, eg. I{/srv/ftp.debian.org/ftp/pool/}
- - B{component} - the id of the component as returned by L{get_component_id}
- - B{archive} - the id of the archive as returned by L{get_archive_id}
- Results are kept in a cache during runtime to minimize database queries.
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
- @type location: string
- @param location: the path of the location
+ @rtype: list
+ @return: list of DBBinary objects for the given name (may be empty)
+ """
- @type component: int
- @param component: the id of the component
+ q = session.query(DBBinary).filter_by(package=package)
- @type archive: int
- @param archive: the id of the archive
+ if version is not None:
+ q = q.filter_by(version=version)
- @rtype: int
- @return: the database id for the location
+ if architecture is not None:
+ if not isinstance(architecture, list):
+ architecture = [architecture]
+ q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture))
- """
+ ret = q.all()
- archive_id = self.get_archive_id(archive)
+ return ret
- if not archive_id:
- return None
+__all__.append('get_binaries_from_name')
- res = None
+@session_wrapper
+def get_binaries_from_source_id(source_id, session=None):
+ """
+ Returns list of DBBinary objects for given C{source_id}
- if component:
- component_id = self.get_component_id(component)
- if component_id:
- res = self.__get_single_id("SELECT id FROM location WHERE path=%(location)s AND component=%(component)s AND archive=%(archive)s",
- {'location': location,
- 'archive': int(archive_id),
- 'component': component_id}, cachename='location')
- else:
- res = self.__get_single_id("SELECT id FROM location WHERE path=%(location)s AND archive=%(archive)d",
- {'location': location, 'archive': archive_id, 'component': ''}, cachename='location')
+ @type source_id: int
+ @param source_id: source_id to search for
- return res
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
- def get_source_id(self, source, version):
- """
- Returns database id for the combination of C{source} and C{version}
- - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{version}
- Results are kept in a cache during runtime to minimize database queries.
+ @rtype: list
+ @return: list of DBBinary objects for the given name (may be empty)
+ """
- @type source: string
- @param source: source package name
+ return session.query(DBBinary).filter_by(source_id=source_id).all()
- @type version: string
- @param version: the source version
+__all__.append('get_binaries_from_source_id')
- @rtype: int
- @return: the database id for the source
+@session_wrapper
+def get_binary_from_name_suite(package, suitename, session=None):
+ ### For dak examine-package
+ ### XXX: Doesn't use object API yet
- """
- return self.__get_single_id("SELECT id FROM source s WHERE s.source=%(source)s AND s.version=%(version)s",
- {'source': source, 'version': version}, cachename='source')
+ sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name
+ FROM binaries b, files fi, location l, component c, bin_associations ba, suite su
+ WHERE b.package=:package
+ AND b.file = fi.id
+ AND fi.location = l.id
+ AND l.component = c.id
+ AND ba.bin=b.id
+ AND ba.suite = su.id
+ AND su.suite_name=:suitename
+ ORDER BY b.version DESC"""
- def get_suite_version(self, source, suite):
- """
- Returns database id for a combination of C{source} and C{suite}.
+ return session.execute(sql, {'package': package, 'suitename': suitename})
- - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
- - B{suite} - a suite name, eg. I{unstable}
+__all__.append('get_binary_from_name_suite')
- Results are kept in a cache during runtime to minimize database queries.
+@session_wrapper
+def get_binary_components(package, suitename, arch, session=None):
+ # Check for packages that have moved from one component to another
+ query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f
+ WHERE b.package=:package AND s.suite_name=:suitename
+ AND (a.arch_string = :arch OR a.arch_string = 'all')
+ AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
+ AND f.location = l.id
+ AND l.component = c.id
+ AND b.file = f.id"""
- @type source: string
- @param source: source package name
+ vals = {'package': package, 'suitename': suitename, 'arch': arch}
- @type suite: string
- @param suite: the suite name
+ return session.execute(query, vals)
- @rtype: string
- @return: the version for I{source} in I{suite}
+__all__.append('get_binary_components')
- """
- return self.__get_single_id("""
- SELECT s.version FROM source s, suite su, src_associations sa
- WHERE sa.source=s.id
- AND sa.suite=su.id
- AND su.suite_name=%(suite)s
- AND s.source=%(source)""", {'suite': suite, 'source': source}, cachename='suite_version')
+################################################################################
+class Component(object):
+ def __init__(self, *args, **kwargs):
+ pass
- def get_files_id (self, filename, size, md5sum, location_id):
- """
- Returns -1, -2 or the file_id for filename, if its C{size} and C{md5sum} match an
- existing copy.
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.component_name == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
- The database is queried using the C{filename} and C{location_id}. If a file does exist
- at that location, the existing size and md5sum are checked against the provided
- parameters. A size or checksum mismatch returns -2. If more than one entry is
- found within the database, a -1 is returned, no result returns None, otherwise
- the file id.
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.component_name != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
- Results are kept in a cache during runtime to minimize database queries.
+ def __repr__(self):
+ return '<Component %s>' % self.component_name
- @type filename: string
- @param filename: the filename of the file to check against the DB
- @type size: int
- @param size: the size of the file to check against the DB
+__all__.append('Component')
- @type md5sum: string
- @param md5sum: the md5sum of the file to check against the DB
+@session_wrapper
+def get_component(component, session=None):
+ """
+ Returns database id for given C{component}.
- @type location_id: int
- @param location_id: the id of the location as returned by L{get_location_id}
+ @type component: string
+ @param component: The name of the override type
- @rtype: int / None
- @return: Various return values are possible:
- - -2: size/checksum error
- - -1: more than one file found in database
- - None: no file found in database
- - int: file id
+ @rtype: int
+ @return: the database id for the given component
- """
- values = {'filename' : filename,
- 'location' : location_id}
+ """
+ component = component.lower()
+
+ q = session.query(Component).filter_by(component_name=component)
- res = self.caches['files'].GetValue( values )
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
- if not res:
- query = """SELECT id, size, md5sum
- FROM files
- WHERE filename = %(filename)s AND location = %(location)s"""
+__all__.append('get_component')
- cursor = self.db_con.cursor()
- cursor.execute( query, values )
+################################################################################
- if cursor.rowcount == 0:
- res = None
+class DBConfig(object):
+ def __init__(self, *args, **kwargs):
+ pass
- elif cursor.rowcount != 1:
- res = -1
+ def __repr__(self):
+ return '<DBConfig %s>' % self.name
- else:
- row = cursor.fetchone()
+__all__.append('DBConfig')
- if row[1] != int(size) or row[2] != md5sum:
- res = -2
+################################################################################
- else:
- self.caches['files'].SetValue(values, row[0])
- res = row[0]
+class ContentFilename(object):
+ def __init__(self, *args, **kwargs):
+ pass
- return res
+ def __repr__(self):
+ return '<ContentFilename %s>' % self.filename
+__all__.append('ContentFilename')
- def get_or_set_contents_file_id(self, filename):
- """
- Returns database id for given filename.
+@session_wrapper
+def get_or_set_contents_file_id(filename, session=None):
+ """
+ Returns database id for given filename.
- Results are kept in a cache during runtime to minimize database queries.
- If no matching file is found, a row is inserted.
+ If no matching file is found, a row is inserted.
- @type filename: string
- @param filename: The filename
+ @type filename: string
+ @param filename: The filename
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
- @rtype: int
- @return: the database id for the given component
- """
- try:
- values={'value': filename}
- query = "SELECT id FROM content_file_names WHERE file = %(value)s"
- id = self.__get_single_id(query, values, cachename='content_file_names')
- if not id:
- c = self.db_con.cursor()
- c.execute( "INSERT INTO content_file_names VALUES (DEFAULT, %(value)s) RETURNING id",
- values )
-
- id = c.fetchone()[0]
- self.caches['content_file_names'].SetValue(values, id)
-
- return id
- except:
- traceback.print_exc()
- raise
-
- def get_or_set_contents_path_id(self, path):
- """
- Returns database id for given path.
+ @rtype: int
+ @return: the database id for the given component
+ """
- Results are kept in a cache during runtime to minimize database queries.
- If no matching file is found, a row is inserted.
+ q = session.query(ContentFilename).filter_by(filename=filename)
- @type path: string
- @param path: The filename
+ try:
+ ret = q.one().cafilename_id
+ except NoResultFound:
+ cf = ContentFilename()
+ cf.filename = filename
+ session.add(cf)
+ session.commit_or_flush()
+ ret = cf.cafilename_id
- @rtype: int
- @return: the database id for the given component
- """
- try:
- values={'value': path}
- query = "SELECT id FROM content_file_paths WHERE path = %(value)s"
- id = self.__get_single_id(query, values, cachename='content_path_names')
- if not id:
- c = self.db_con.cursor()
- c.execute( "INSERT INTO content_file_paths VALUES (DEFAULT, %(value)s) RETURNING id",
- values )
-
- id = c.fetchone()[0]
- self.caches['content_path_names'].SetValue(values, id)
-
- return id
- except:
- traceback.print_exc()
- raise
-
- def get_suite_architectures(self, suite):
- """
- Returns list of architectures for C{suite}.
+ return ret
- @type suite: string, int
- @param suite: the suite name or the suite_id
+__all__.append('get_or_set_contents_file_id')
- @rtype: list
- @return: the list of architectures for I{suite}
- """
+@session_wrapper
+def get_contents(suite, overridetype, section=None, session=None):
+ """
+ Returns contents for a suite / overridetype combination, limiting
+ to a section if not None.
+
+ @type suite: Suite
+ @param suite: Suite object
+
+ @type overridetype: OverrideType
+ @param overridetype: OverrideType object
+
+ @type section: Section
+ @param section: Optional section object to limit results to
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: ResultsProxy
+ @return: ResultsProxy object set up to return tuples of (filename, section,
+ package, arch_id)
+ """
+
+ # find me all of the contents for a given suite
+ contents_q = """SELECT (p.path||'/'||n.file) AS fn,
+ s.section,
+ b.package,
+ b.architecture
+ FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
+ JOIN content_file_names n ON (c.filename=n.id)
+ JOIN binaries b ON (b.id=c.binary_pkg)
+ JOIN override o ON (o.package=b.package)
+ JOIN section s ON (s.id=o.section)
+ WHERE o.suite = :suiteid AND o.type = :overridetypeid
+ AND b.type=:overridetypename"""
+
+ vals = {'suiteid': suite.suite_id,
+ 'overridetypeid': overridetype.overridetype_id,
+ 'overridetypename': overridetype.overridetype}
+
+ if section is not None:
+ contents_q += " AND s.id = :sectionid"
+ vals['sectionid'] = section.section_id
+
+ contents_q += " ORDER BY fn"
+
+ return session.execute(contents_q, vals)
+
+__all__.append('get_contents')
+
+################################################################################
+
+class ContentFilepath(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<ContentFilepath %s>' % self.filepath
+
+__all__.append('ContentFilepath')
+
+@session_wrapper
+def get_or_set_contents_path_id(filepath, session=None):
+ """
+ Returns database id for given path.
+
+ If no matching file is found, a row is inserted.
+
+ @type filename: string
+ @param filename: The filepath
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: int
+ @return: the database id for the given path
+ """
+
+ q = session.query(ContentFilepath).filter_by(filepath=filepath)
+
+ try:
+ ret = q.one().cafilepath_id
+ except NoResultFound:
+ cf = ContentFilepath()
+ cf.filepath = filepath
+ session.add(cf)
+ session.commit_or_flush()
+ ret = cf.cafilepath_id
+
+ return ret
+
+__all__.append('get_or_set_contents_path_id')
+
+################################################################################
+
+class ContentAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<ContentAssociation %s>' % self.ca_id
+
+__all__.append('ContentAssociation')
+
+def insert_content_paths(binary_id, fullpaths, session=None):
+ """
+ Make sure given path is associated with given binary id
+
+ @type binary_id: int
+ @param binary_id: the id of the binary
+ @type fullpaths: list
+ @param fullpaths: the list of paths of the file being associated with the binary
+ @type session: SQLAlchemy session
+ @param session: Optional SQLAlchemy session. If this is passed, the caller
+ is responsible for ensuring a transaction has begun and committing the
+ results or rolling back based on the result code. If not passed, a commit
+ will be performed at the end of the function, otherwise the caller is
+ responsible for commiting.
+
+ @return: True upon success
+ """
- suite_id = None
- if type(suite) == str:
- suite_id = self.get_suite_id(suite)
- elif type(suite) == int:
- suite_id = suite
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ try:
+ # Insert paths
+ pathcache = {}
+ for fullpath in fullpaths:
+ # Get the necessary IDs ...
+ (path, file) = os.path.split(fullpath)
+
+ filepath_id = get_or_set_contents_path_id(path, session)
+ filename_id = get_or_set_contents_file_id(file, session)
+
+ pathcache[fullpath] = (filepath_id, filename_id)
+
+ for fullpath, dat in pathcache.items():
+ ca = ContentAssociation()
+ ca.binary_id = binary_id
+ ca.filepath_id = dat[0]
+ ca.filename_id = dat[1]
+ session.add(ca)
+
+ # Only commit if we set up the session ourself
+ if privatetrans:
+ session.commit()
+ session.close()
else:
- return None
+ session.flush()
- c = self.db_con.cursor()
- c.execute( """SELECT a.arch_string FROM suite_architectures sa
- JOIN architecture a ON (a.id = sa.architecture)
- WHERE suite='%s'""" % suite_id )
+ return True
- return map(lambda x: x[0], c.fetchall())
+ except:
+ traceback.print_exc()
- def insert_content_paths(self, bin_id, fullpaths):
- """
- Make sure given path is associated with given binary id
+ # Only rollback if we set up the session ourself
+ if privatetrans:
+ session.rollback()
+ session.close()
- @type bin_id: int
- @param bin_id: the id of the binary
- @type fullpaths: list
- @param fullpaths: the list of paths of the file being associated with the binary
+ return False
- @return: True upon success
- """
+__all__.append('insert_content_paths')
- c = self.db_con.cursor()
+################################################################################
- c.execute("BEGIN WORK")
- try:
+class DSCFile(object):
+ def __init__(self, *args, **kwargs):
+ pass
- for fullpath in fullpaths:
- (path, file) = os.path.split(fullpath)
+ def __repr__(self):
+ return '<DSCFile %s>' % self.dscfile_id
- if path.startswith( "./" ):
- path = path[2:]
- # Get the necessary IDs ...
- file_id = self.get_or_set_contents_file_id(file)
- path_id = self.get_or_set_contents_path_id(path)
+__all__.append('DSCFile')
- c.execute("""INSERT INTO deb_contents
-
- (binary_pkg, filepath, filename)
- VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) )
+@session_wrapper
+def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None):
+ """
+ Returns a list of DSCFiles which may be empty
- c.execute("COMMIT")
- return True
- except:
- traceback.print_exc()
- c.execute("ROLLBACK")
- return False
+ @type dscfile_id: int (optional)
+ @param dscfile_id: the dscfile_id of the DSCFiles to find
- def insert_pending_content_paths(self, package, fullpaths):
- """
- Make sure given paths are temporarily associated with given
- package
+ @type source_id: int (optional)
+ @param source_id: the source id related to the DSCFiles to find
- @type package: dict
- @param package: the package to associate with should have been read in from the binary control file
- @type fullpaths: list
- @param fullpaths: the list of paths of the file being associated with the binary
+ @type poolfile_id: int (optional)
+ @param poolfile_id: the poolfile id related to the DSCFiles to find
- @return: True upon success
- """
+ @rtype: list
+ @return: Possibly empty list of DSCFiles
+ """
+
+ q = session.query(DSCFile)
+
+ if dscfile_id is not None:
+ q = q.filter_by(dscfile_id=dscfile_id)
+
+ if source_id is not None:
+ q = q.filter_by(source_id=source_id)
+
+ if poolfile_id is not None:
+ q = q.filter_by(poolfile_id=poolfile_id)
+
+ return q.all()
+
+__all__.append('get_dscfiles')
+
+################################################################################
+
+class PoolFile(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<PoolFile %s>' % self.filename
+
+__all__.append('PoolFile')
+
+@session_wrapper
+def check_poolfile(filename, filesize, md5sum, location_id, session=None):
+ """
+ Returns a tuple:
+ (ValidFileFound [boolean or None], PoolFile object or None)
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @type filesize: int
+ @param filesize: the size of the file to check against the DB
+
+ @type md5sum: string
+ @param md5sum: the md5sum of the file to check against the DB
+
+ @type location_id: int
+ @param location_id: the id of the location to look in
+
+ @rtype: tuple
+ @return: Tuple of length 2.
+ If more than one file found with that name:
+ (None, None)
+ If valid pool file found: (True, PoolFile object)
+ If valid pool file not found:
+ (False, None) if no file found
+ (False, PoolFile object) if file found with size/md5sum mismatch
+ """
+
+ q = session.query(PoolFile).filter_by(filename=filename)
+ q = q.join(Location).filter_by(location_id=location_id)
+
+ ret = None
+
+ if q.count() > 1:
+ ret = (None, None)
+ elif q.count() < 1:
+ ret = (False, None)
+ else:
+ obj = q.one()
+ if obj.md5sum != md5sum or obj.filesize != filesize:
+ ret = (False, obj)
+
+ if ret is None:
+ ret = (True, obj)
+
+ return ret
+
+__all__.append('check_poolfile')
+
+@session_wrapper
+def get_poolfile_by_id(file_id, session=None):
+ """
+ Returns a PoolFile objects or None for the given id
+
+ @type file_id: int
+ @param file_id: the id of the file to look for
+
+ @rtype: PoolFile or None
+ @return: either the PoolFile object or None
+ """
+
+ q = session.query(PoolFile).filter_by(file_id=file_id)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_poolfile_by_id')
+
+
+@session_wrapper
+def get_poolfile_by_name(filename, location_id=None, session=None):
+ """
+ Returns an array of PoolFile objects for the given filename and
+ (optionally) location_id
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @type location_id: int
+ @param location_id: the id of the location to look in (optional)
+
+ @rtype: array
+ @return: array of PoolFile objects
+ """
+
+ q = session.query(PoolFile).filter_by(filename=filename)
+
+ if location_id is not None:
+ q = q.join(Location).filter_by(location_id=location_id)
+
+ return q.all()
+
+__all__.append('get_poolfile_by_name')
+
+@session_wrapper
+def get_poolfile_like_name(filename, session=None):
+ """
+ Returns an array of PoolFile objects which are like the given name
+
+ @type filename: string
+ @param filename: the filename of the file to check against the DB
+
+ @rtype: array
+ @return: array of PoolFile objects
+ """
+
+ # TODO: There must be a way of properly using bind parameters with %FOO%
+ q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename))
+
+ return q.all()
+
+__all__.append('get_poolfile_like_name')
+
+################################################################################
+
+class Fingerprint(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Fingerprint %s>' % self.fingerprint
+
+__all__.append('Fingerprint')
+
+@session_wrapper
+def get_or_set_fingerprint(fpr, session=None):
+ """
+ Returns Fingerprint object for given fpr.
+
+ If no matching fpr is found, a row is inserted.
+
+ @type fpr: string
+ @param fpr: The fpr to find / add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+ A flush will be performed either way.
+
+ @rtype: Fingerprint
+ @return: the Fingerprint object for the given fpr
+ """
+
+ q = session.query(Fingerprint).filter_by(fingerprint=fpr)
+
+ try:
+ ret = q.one()
+ except NoResultFound:
+ fingerprint = Fingerprint()
+ fingerprint.fingerprint = fpr
+ session.add(fingerprint)
+ session.commit_or_flush()
+ ret = fingerprint
+
+ return ret
+
+__all__.append('get_or_set_fingerprint')
+
+################################################################################
+
+class Keyring(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Keyring %s>' % self.keyring_name
+
+__all__.append('Keyring')
+
+@session_wrapper
+def get_or_set_keyring(keyring, session=None):
+ """
+ If C{keyring} does not have an entry in the C{keyrings} table yet, create one
+ and return the new Keyring
+ If C{keyring} already has an entry, simply return the existing Keyring
+
+ @type keyring: string
+ @param keyring: the keyring name
+
+ @rtype: Keyring
+ @return: the Keyring object for this keyring
+ """
+
+ q = session.query(Keyring).filter_by(keyring_name=keyring)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ obj = Keyring(keyring_name=keyring)
+ session.add(obj)
+ session.commit_or_flush()
+ return obj
+
+__all__.append('get_or_set_keyring')
+
+################################################################################
+
+class Location(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Location %s (%s)>' % (self.path, self.location_id)
+
+__all__.append('Location')
+
+@session_wrapper
+def get_location(location, component=None, archive=None, session=None):
+ """
+ Returns Location object for the given combination of location, component
+ and archive
+
+ @type location: string
+ @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/}
+
+ @type component: string
+ @param component: the component name (if None, no restriction applied)
+
+ @type archive: string
+ @param archive_id: the archive name (if None, no restriction applied)
+
+ @rtype: Location / None
+ @return: Either a Location object or None if one can't be found
+ """
+
+ q = session.query(Location).filter_by(path=location)
+
+ if archive is not None:
+ q = q.join(Archive).filter_by(archive_name=archive)
+
+ if component is not None:
+ q = q.join(Component).filter_by(component_name=component)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_location')
+
+################################################################################
+
+class Maintainer(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '''<Maintainer '%s' (%s)>''' % (self.name, self.maintainer_id)
+
+ def get_split_maintainer(self):
+ if not hasattr(self, 'name') or self.name is None:
+ return ('', '', '', '')
+
+ return fix_maintainer(self.name.strip())
+
+__all__.append('Maintainer')
+
+@session_wrapper
+def get_or_set_maintainer(name, session=None):
+ """
+ Returns Maintainer object for given maintainer name.
+
+ If no matching maintainer name is found, a row is inserted.
+
+ @type name: string
+ @param name: The maintainer name to add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+ A flush will be performed either way.
+
+ @rtype: Maintainer
+ @return: the Maintainer object for the given maintainer
+ """
+
+ q = session.query(Maintainer).filter_by(name=name)
+ try:
+ ret = q.one()
+ except NoResultFound:
+ maintainer = Maintainer()
+ maintainer.name = name
+ session.add(maintainer)
+ session.commit_or_flush()
+ ret = maintainer
+
+ return ret
+
+__all__.append('get_or_set_maintainer')
+
+@session_wrapper
+def get_maintainer(maintainer_id, session=None):
+ """
+ Return the name of the maintainer behind C{maintainer_id} or None if that
+ maintainer_id is invalid.
+
+ @type maintainer_id: int
+ @param maintainer_id: the id of the maintainer
+
+ @rtype: Maintainer
+ @return: the Maintainer with this C{maintainer_id}
+ """
+
+ return session.query(Maintainer).get(maintainer_id)
+
+__all__.append('get_maintainer')
+
+################################################################################
+
+class NewComment(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '''<NewComment for '%s %s' (%s)>''' % (self.package, self.version, self.comment_id)
+
+__all__.append('NewComment')
+
+@session_wrapper
+def has_new_comment(package, version, session=None):
+ """
+ Returns true if the given combination of C{package}, C{version} has a comment.
+
+ @type package: string
+ @param package: name of the package
+
+ @type version: string
+ @param version: package version
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: boolean
+ @return: true/false
+ """
+
+ q = session.query(NewComment)
+ q = q.filter_by(package=package)
+ q = q.filter_by(version=version)
+
+ return bool(q.count() > 0)
+
+__all__.append('has_new_comment')
+
+@session_wrapper
+def get_new_comments(package=None, version=None, comment_id=None, session=None):
+ """
+ Returns (possibly empty) list of NewComment objects for the given
+ parameters
+
+ @type package: string (optional)
+ @param package: name of the package
+
+ @type version: string (optional)
+ @param version: package version
+
+ @type comment_id: int (optional)
+ @param comment_id: An id of a comment
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: A (possibly empty) list of NewComment objects will be returned
+ """
+
+ q = session.query(NewComment)
+ if package is not None: q = q.filter_by(package=package)
+ if version is not None: q = q.filter_by(version=version)
+ if comment_id is not None: q = q.filter_by(comment_id=comment_id)
+
+ return q.all()
+
+__all__.append('get_new_comments')
+
+################################################################################
+
+class Override(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Override %s (%s)>' % (self.package, self.suite_id)
+
+__all__.append('Override')
+
+@session_wrapper
+def get_override(package, suite=None, component=None, overridetype=None, session=None):
+ """
+ Returns Override object for the given parameters
+
+ @type package: string
+ @param package: The name of the package
+
+ @type suite: string, list or None
+ @param suite: The name of the suite (or suites if a list) to limit to. If
+ None, don't limit. Defaults to None.
+
+ @type component: string, list or None
+ @param component: The name of the component (or components if a list) to
+ limit to. If None, don't limit. Defaults to None.
+
+ @type overridetype: string, list or None
+ @param overridetype: The name of the overridetype (or overridetypes if a list) to
+ limit to. If None, don't limit. Defaults to None.
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: A (possibly empty) list of Override objects will be returned
+ """
+
+ q = session.query(Override)
+ q = q.filter_by(package=package)
+
+ if suite is not None:
+ if not isinstance(suite, list): suite = [suite]
+ q = q.join(Suite).filter(Suite.suite_name.in_(suite))
+
+ if component is not None:
+ if not isinstance(component, list): component = [component]
+ q = q.join(Component).filter(Component.component_name.in_(component))
+
+ if overridetype is not None:
+ if not isinstance(overridetype, list): overridetype = [overridetype]
+ q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype))
+
+ return q.all()
+
+__all__.append('get_override')
+
+
+################################################################################
+
+class OverrideType(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<OverrideType %s>' % self.overridetype
+
+__all__.append('OverrideType')
+
+@session_wrapper
+def get_override_type(override_type, session=None):
+ """
+ Returns OverrideType object for given C{override type}.
+
+ @type override_type: string
+ @param override_type: The name of the override type
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: int
+ @return: the database id for the given override type
+ """
+
+ q = session.query(OverrideType).filter_by(overridetype=override_type)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_override_type')
+
+################################################################################
+
+class PendingContentAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<PendingContentAssociation %s>' % self.pca_id
+
+__all__.append('PendingContentAssociation')
+
+def insert_pending_content_paths(package, fullpaths, session=None):
+ """
+ Make sure given paths are temporarily associated with given
+ package
+
+ @type package: dict
+ @param package: the package to associate with should have been read in from the binary control file
+ @type fullpaths: list
+ @param fullpaths: the list of paths of the file being associated with the binary
+ @type session: SQLAlchemy session
+ @param session: Optional SQLAlchemy session. If this is passed, the caller
+ is responsible for ensuring a transaction has begun and committing the
+ results or rolling back based on the result code. If not passed, a commit
+ will be performed at the end of the function
+
+ @return: True upon success, False if there is a problem
+ """
+
+ privatetrans = False
+
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ try:
+ arch = get_architecture(package['Architecture'], session)
+ arch_id = arch.arch_id
+
+ # Remove any already existing recorded files for this package
+ q = session.query(PendingContentAssociation)
+ q = q.filter_by(package=package['Package'])
+ q = q.filter_by(version=package['Version'])
+ q = q.filter_by(architecture=arch_id)
+ q.delete()
+
+ # Insert paths
+ pathcache = {}
+ for fullpath in fullpaths:
+ (path, file) = os.path.split(fullpath)
+
+ if path.startswith( "./" ):
+ path = path[2:]
+
+ filepath_id = get_or_set_contents_path_id(path, session)
+ filename_id = get_or_set_contents_file_id(file, session)
+
+ pathcache[fullpath] = (filepath_id, filename_id)
+
+ for fullpath, dat in pathcache.items():
+ pca = PendingContentAssociation()
+ pca.package = package['Package']
+ pca.version = package['Version']
+ pca.filepath_id = dat[0]
+ pca.filename_id = dat[1]
+ pca.architecture = arch_id
+ session.add(pca)
+
+ # Only commit if we set up the session ourself
+ if privatetrans:
+ session.commit()
+ session.close()
+ else:
+ session.flush()
+
+ return True
+ except Exception, e:
+ traceback.print_exc()
+
+ # Only rollback if we set up the session ourself
+ if privatetrans:
+ session.rollback()
+ session.close()
+
+ return False
+
+__all__.append('insert_pending_content_paths')
+
+################################################################################
+
+class Priority(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.priority == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.priority != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Priority %s (%s)>' % (self.priority, self.priority_id)
+
+__all__.append('Priority')
+
+@session_wrapper
+def get_priority(priority, session=None):
+ """
+ Returns Priority object for given C{priority name}.
+
+ @type priority: string
+ @param priority: The name of the priority
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Priority
+ @return: Priority object for the given priority
+ """
+
+ q = session.query(Priority).filter_by(priority=priority)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_priority')
+
+@session_wrapper
+def get_priorities(session=None):
+ """
+ Returns dictionary of priority names -> id mappings
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: dictionary
+ @return: dictionary of priority names -> id mappings
+ """
+
+ ret = {}
+ q = session.query(Priority)
+ for x in q.all():
+ ret[x.priority] = x.priority_id
+
+ return ret
+
+__all__.append('get_priorities')
+
+################################################################################
+
+class Queue(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Queue %s>' % self.queue_name
+
+ def autobuild_upload(self, changes, srcpath, session=None):
+ """
+ Update queue_build database table used for incoming autobuild support.
+
+ @type changes: Changes
+ @param changes: changes object for the upload to process
+
+ @type srcpath: string
+ @param srcpath: path for the queue file entries/link destinations
+
+ @type session: SQLAlchemy session
+ @param session: Optional SQLAlchemy session. If this is passed, the
+ caller is responsible for ensuring a transaction has begun and
+ committing the results or rolling back based on the result code. If
+ not passed, a commit will be performed at the end of the function,
+ otherwise the caller is responsible for commiting.
+
+ @rtype: NoneType or string
+ @return: None if the operation failed, a string describing the error if not
+ """
+
+ privatetrans = False
+ if session is None:
+ session = DBConn().session()
+ privatetrans = True
+
+ # TODO: Remove by moving queue config into the database
+ conf = Config()
+
+ for suitename in changes.changes["distribution"].keys():
+ # TODO: Move into database as:
+ # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build)
+ # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink)
+ # This also gets rid of the SecurityQueueBuild hack below
+ if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"):
+ continue
+
+ # Find suite object
+ s = get_suite(suitename, session)
+ if s is None:
+ return "INTERNAL ERROR: Could not find suite %s" % suitename
+
+ # TODO: Get from database as above
+ dest_dir = conf["Dir::QueueBuild"]
+
+ # TODO: Move into database as above
+ if conf.FindB("Dinstall::SecurityQueueBuild"):
+ dest_dir = os.path.join(dest_dir, suitename)
+
+ for file_entry in changes.files.keys():
+ src = os.path.join(srcpath, file_entry)
+ dest = os.path.join(dest_dir, file_entry)
+
+ # TODO: Move into database as above
+ if conf.FindB("Dinstall::SecurityQueueBuild"):
+ # Copy it since the original won't be readable by www-data
+ import utils
+ utils.copy(src, dest)
+ else:
+ # Create a symlink to it
+ os.symlink(src, dest)
+
+ qb = QueueBuild()
+ qb.suite_id = s.suite_id
+ qb.queue_id = self.queue_id
+ qb.filename = dest
+ qb.in_queue = True
+
+ session.add(qb)
+
+ exists, symlinked = utils.ensure_orig_files(changes, dest, session)
+
+ # Add symlinked files to the list of packages for later processing
+ # by apt-ftparchive
+ for filename in symlinked:
+ qb = QueueBuild()
+ qb.suite_id = s.suite_id
+ qb.queue_id = self.queue_id
+ qb.filename = filename
+ qb.in_queue = True
+ session.add(qb)
+
+ # Update files to ensure they are not removed prematurely
+ for filename in exists:
+ qb = get_queue_build(filename, s.suite_id, session)
+ if qb is None:
+ qb.in_queue = True
+ qb.last_used = None
+ session.add(qb)
+
+ if privatetrans:
+ session.commit()
+ session.close()
+
+ return None
+
+__all__.append('Queue')
+
+@session_wrapper
+def get_or_set_queue(queuename, session=None):
+ """
+ Returns Queue object for given C{queue name}, creating it if it does not
+ exist.
+
+ @type queuename: string
+ @param queuename: The name of the queue
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Queue
+ @return: Queue object for the given queue
+ """
+
+ q = session.query(Queue).filter_by(queue_name=queuename)
+
+ try:
+ ret = q.one()
+ except NoResultFound:
+ queue = Queue()
+ queue.queue_name = queuename
+ session.add(queue)
+ session.commit_or_flush()
+ ret = queue
+
+ return ret
+
+__all__.append('get_or_set_queue')
+
+################################################################################
+
+class QueueBuild(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<QueueBuild %s (%s)>' % (self.filename, self.queue_id)
+
+__all__.append('QueueBuild')
+
+@session_wrapper
+def get_queue_build(filename, suite, session=None):
+ """
+ Returns QueueBuild object for given C{filename} and C{suite}.
+
+ @type filename: string
+ @param filename: The name of the file
+
+ @type suiteid: int or str
+ @param suiteid: Suite name or ID
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Queue
+ @return: Queue object for the given queue
+ """
+
+ if isinstance(suite, int):
+ q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite)
+ else:
+ q = session.query(QueueBuild).filter_by(filename=filename)
+ q = q.join(Suite).filter_by(suite_name=suite)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_queue_build')
+
+################################################################################
+
+class Section(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.section == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.section != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Section %s>' % self.section
+
+__all__.append('Section')
+
+@session_wrapper
+def get_section(section, session=None):
+ """
+ Returns Section object for given C{section name}.
+
+ @type section: string
+ @param section: The name of the section
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Section
+ @return: Section object for the given section name
+ """
+
+ q = session.query(Section).filter_by(section=section)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_section')
+
+@session_wrapper
+def get_sections(session=None):
+ """
+ Returns dictionary of section names -> id mappings
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: dictionary
+ @return: dictionary of section names -> id mappings
+ """
+
+ ret = {}
+ q = session.query(Section)
+ for x in q.all():
+ ret[x.section] = x.section_id
+
+ return ret
+
+__all__.append('get_sections')
+
+################################################################################
+
+class DBSource(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<DBSource %s (%s)>' % (self.source, self.version)
+
+__all__.append('DBSource')
+
+@session_wrapper
+def source_exists(source, source_version, suites = ["any"], session=None):
+ """
+ Ensure that source exists somewhere in the archive for the binary
+ upload being processed.
+ 1. exact match => 1.0-3
+ 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
+
+ @type package: string
+ @param package: package source name
+
+ @type source_version: string
+ @param source_version: expected source version
+
+ @type suites: list
+ @param suites: list of suites to check in, default I{any}
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: int
+ @return: returns 1 if a source with expected version is found, otherwise 0
+
+ """
+
+ cnf = Config()
+ ret = 1
+
+ for suite in suites:
+ q = session.query(DBSource).filter_by(source=source)
+ if suite != "any":
+ # source must exist in suite X, or in some other suite that's
+ # mapped to X, recursively... silent-maps are counted too,
+ # unreleased-maps aren't.
+ maps = cnf.ValueList("SuiteMappings")[:]
+ maps.reverse()
+ maps = [ m.split() for m in maps ]
+ maps = [ (x[1], x[2]) for x in maps
+ if x[0] == "map" or x[0] == "silent-map" ]
+ s = [suite]
+ for x in maps:
+ if x[1] in s and x[0] not in s:
+ s.append(x[0])
+
+ q = q.join(SrcAssociation).join(Suite)
+ q = q.filter(Suite.suite_name.in_(s))
+
+ # Reduce the query results to a list of version numbers
+ ql = [ j.version for j in q.all() ]
+
+ # Try (1)
+ if source_version in ql:
+ continue
+
+ # Try (2)
+ from daklib.regexes import re_bin_only_nmu
+ orig_source_version = re_bin_only_nmu.sub('', source_version)
+ if orig_source_version in ql:
+ continue
+
+ # No source found so return not ok
+ ret = 0
+
+ return ret
+
+__all__.append('source_exists')
+
+@session_wrapper
+def get_suites_source_in(source, session=None):
+ """
+ Returns list of Suite objects which given C{source} name is in
+
+ @type source: str
+ @param source: DBSource package name to search for
+
+ @rtype: list
+ @return: list of Suite objects for the given source
+ """
+
+ return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all()
+
+__all__.append('get_suites_source_in')
+
+@session_wrapper
+def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None):
+ """
+ Returns list of DBSource objects for given C{source} name and other parameters
+
+ @type source: str
+ @param source: DBSource package name to search for
+
+ @type source: str or None
+ @param source: DBSource version name to search for or None if not applicable
+
+ @type dm_upload_allowed: bool
+ @param dm_upload_allowed: If None, no effect. If True or False, only
+ return packages with that dm_upload_allowed setting
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of DBSource objects for the given name (may be empty)
+ """
+
+ q = session.query(DBSource).filter_by(source=source)
+
+ if version is not None:
+ q = q.filter_by(version=version)
+
+ if dm_upload_allowed is not None:
+ q = q.filter_by(dm_upload_allowed=dm_upload_allowed)
+
+ return q.all()
+
+__all__.append('get_sources_from_name')
+
+@session_wrapper
+def get_source_in_suite(source, suite, session=None):
+ """
+ Returns list of DBSource objects for a combination of C{source} and C{suite}.
+
+ - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc}
+ - B{suite} - a suite name, eg. I{unstable}
+
+ @type source: string
+ @param source: source package name
+
+ @type suite: string
+ @param suite: the suite name
+
+ @rtype: string
+ @return: the version for I{source} in I{suite}
+
+ """
+
+ q = session.query(SrcAssociation)
+ q = q.join('source').filter_by(source=source)
+ q = q.join('suite').filter_by(suite_name=suite)
+
+ try:
+ return q.one().source
+ except NoResultFound:
+ return None
+
+__all__.append('get_source_in_suite')
+
+################################################################################
+
+class SrcAssociation(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SrcAssociation %s (%s, %s)>' % (self.sa_id, self.source, self.suite)
+
+__all__.append('SrcAssociation')
+
+################################################################################
+
+class SrcFormat(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SrcFormat %s>' % (self.format_name)
+
+__all__.append('SrcFormat')
+
+################################################################################
+
+class SrcUploader(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SrcUploader %s>' % self.uploader_id
+
+__all__.append('SrcUploader')
+
+################################################################################
+
+SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
+ ('SuiteID', 'suite_id'),
+ ('Version', 'version'),
+ ('Origin', 'origin'),
+ ('Label', 'label'),
+ ('Description', 'description'),
+ ('Untouchable', 'untouchable'),
+ ('Announce', 'announce'),
+ ('Codename', 'codename'),
+ ('OverrideCodename', 'overridecodename'),
+ ('ValidTime', 'validtime'),
+ ('Priority', 'priority'),
+ ('NotAutomatic', 'notautomatic'),
+ ('CopyChanges', 'copychanges'),
+ ('CopyDotDak', 'copydotdak'),
+ ('CommentsDir', 'commentsdir'),
+ ('OverrideSuite', 'overridesuite'),
+ ('ChangelogBase', 'changelogbase')]
+
+
+class Suite(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<Suite %s>' % self.suite_name
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.suite_name == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.suite_name != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def details(self):
+ ret = []
+ for disp, field in SUITE_FIELDS:
+ val = getattr(self, field, None)
+ if val is not None:
+ ret.append("%s: %s" % (disp, val))
+
+ return "\n".join(ret)
+
+__all__.append('Suite')
+
+@session_wrapper
+def get_suite_architecture(suite, architecture, session=None):
+ """
+ Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it
+ doesn't exist
+
+ @type suite: str
+ @param suite: Suite name to search for
+
+ @type architecture: str
+ @param architecture: Architecture name to search for
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: SuiteArchitecture
+ @return: the SuiteArchitecture object or None
+ """
+
+ q = session.query(SuiteArchitecture)
+ q = q.join(Architecture).filter_by(arch_string=architecture)
+ q = q.join(Suite).filter_by(suite_name=suite)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_suite_architecture')
+
+@session_wrapper
+def get_suite(suite, session=None):
+ """
+ Returns Suite object for given C{suite name}.
+
+ @type suite: string
+ @param suite: The name of the suite
+
+ @type session: Session
+ @param session: Optional SQLA session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: Suite
+ @return: Suite object for the requested suite name (None if not present)
+ """
+
+ q = session.query(Suite).filter_by(suite_name=suite)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_suite')
+
+################################################################################
+
+class SuiteArchitecture(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SuiteArchitecture (%s, %s)>' % (self.suite_id, self.arch_id)
+
+__all__.append('SuiteArchitecture')
+
+@session_wrapper
+def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None):
+ """
+ Returns list of Architecture objects for given C{suite} name
+
+ @type source: str
+ @param source: Suite name to search for
+
+ @type skipsrc: boolean
+ @param skipsrc: Whether to skip returning the 'source' architecture entry
+ (Default False)
+
+ @type skipall: boolean
+ @param skipall: Whether to skip returning the 'all' architecture entry
+ (Default False)
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: list of Architecture objects for the given name (may be empty)
+ """
+
+ q = session.query(Architecture)
+ q = q.join(SuiteArchitecture)
+ q = q.join(Suite).filter_by(suite_name=suite)
+
+ if skipsrc:
+ q = q.filter(Architecture.arch_string != 'source')
+
+ if skipall:
+ q = q.filter(Architecture.arch_string != 'all')
+
+ q = q.order_by('arch_string')
+
+ return q.all()
+
+__all__.append('get_suite_architectures')
+
+################################################################################
+
+class SuiteSrcFormat(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __repr__(self):
+ return '<SuiteSrcFormat (%s, %s)>' % (self.suite_id, self.src_format_id)
+
+__all__.append('SuiteSrcFormat')
+
+@session_wrapper
+def get_suite_src_formats(suite, session=None):
+ """
+ Returns list of allowed SrcFormat for C{suite}.
+
+ @type suite: str
+ @param suite: Suite name to search for
+
+ @type session: Session
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied)
+
+ @rtype: list
+ @return: the list of allowed source formats for I{suite}
+ """
+
+ q = session.query(SrcFormat)
+ q = q.join(SuiteSrcFormat)
+ q = q.join(Suite).filter_by(suite_name=suite)
+ q = q.order_by('format_name')
+
+ return q.all()
+
+__all__.append('get_suite_src_formats')
+
+################################################################################
+
+class Uid(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __eq__(self, val):
+ if isinstance(val, str):
+ return (self.uid == val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __ne__(self, val):
+ if isinstance(val, str):
+ return (self.uid != val)
+ # This signals to use the normal comparison operator
+ return NotImplemented
+
+ def __repr__(self):
+ return '<Uid %s (%s)>' % (self.uid, self.name)
+
+__all__.append('Uid')
+
+@session_wrapper
+def add_database_user(uidname, session=None):
+ """
+ Adds a database user
+
+ @type uidname: string
+ @param uidname: The uid of the user to add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: Uid
+ @return: the uid object for the given uidname
+ """
+
+ session.execute("CREATE USER :uid", {'uid': uidname})
+ session.commit_or_flush()
+
+__all__.append('add_database_user')
+
+@session_wrapper
+def get_or_set_uid(uidname, session=None):
+ """
+ Returns uid object for given uidname.
+
+ If no matching uidname is found, a row is inserted.
+
+ @type uidname: string
+ @param uidname: The uid to add
+
+ @type session: SQLAlchemy
+ @param session: Optional SQL session object (a temporary one will be
+ generated if not supplied). If not passed, a commit will be performed at
+ the end of the function, otherwise the caller is responsible for commiting.
+
+ @rtype: Uid
+ @return: the uid object for the given uidname
+ """
+
+ q = session.query(Uid).filter_by(uid=uidname)
+
+ try:
+ ret = q.one()
+ except NoResultFound:
+ uid = Uid()
+ uid.uid = uidname
+ session.add(uid)
+ session.commit_or_flush()
+ ret = uid
+
+ return ret
+
+__all__.append('get_or_set_uid')
+
+@session_wrapper
+def get_uid_from_fingerprint(fpr, session=None):
+ q = session.query(Uid)
+ q = q.join(Fingerprint).filter_by(fingerprint=fpr)
+
+ try:
+ return q.one()
+ except NoResultFound:
+ return None
+
+__all__.append('get_uid_from_fingerprint')
+
+################################################################################
+
+class DBConn(Singleton):
+ """
+ database module init.
+ """
+ def __init__(self, *args, **kwargs):
+ super(DBConn, self).__init__(*args, **kwargs)
+
+ def _startup(self, *args, **kwargs):
+ self.debug = False
+ if kwargs.has_key('debug'):
+ self.debug = True
+ self.__createconn()
+
+ def __setuptables(self):
+ self.tbl_architecture = Table('architecture', self.db_meta, autoload=True)
+ self.tbl_archive = Table('archive', self.db_meta, autoload=True)
+ self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True)
+ self.tbl_binaries = Table('binaries', self.db_meta, autoload=True)
+ self.tbl_component = Table('component', self.db_meta, autoload=True)
+ self.tbl_config = Table('config', self.db_meta, autoload=True)
+ self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True)
+ self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True)
+ self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True)
+ self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True)
+ self.tbl_files = Table('files', self.db_meta, autoload=True)
+ self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
+ self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
+ self.tbl_location = Table('location', self.db_meta, autoload=True)
+ self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
+ self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
+ self.tbl_override = Table('override', self.db_meta, autoload=True)
+ self.tbl_override_type = Table('override_type', self.db_meta, autoload=True)
+ self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True)
+ self.tbl_priority = Table('priority', self.db_meta, autoload=True)
+ self.tbl_queue = Table('queue', self.db_meta, autoload=True)
+ self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True)
+ self.tbl_section = Table('section', self.db_meta, autoload=True)
+ self.tbl_source = Table('source', self.db_meta, autoload=True)
+ self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True)
+ self.tbl_src_format = Table('src_format', self.db_meta, autoload=True)
+ self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True)
+ self.tbl_suite = Table('suite', self.db_meta, autoload=True)
+ self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True)
+ self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True)
+ self.tbl_uid = Table('uid', self.db_meta, autoload=True)
+
+ def __setupmappers(self):
+ mapper(Architecture, self.tbl_architecture,
+ properties = dict(arch_id = self.tbl_architecture.c.id))
+
+ mapper(Archive, self.tbl_archive,
+ properties = dict(archive_id = self.tbl_archive.c.id,
+ archive_name = self.tbl_archive.c.name))
+
+ mapper(BinAssociation, self.tbl_bin_associations,
+ properties = dict(ba_id = self.tbl_bin_associations.c.id,
+ suite_id = self.tbl_bin_associations.c.suite,
+ suite = relation(Suite),
+ binary_id = self.tbl_bin_associations.c.bin,
+ binary = relation(DBBinary)))
+
+ mapper(DBBinary, self.tbl_binaries,
+ properties = dict(binary_id = self.tbl_binaries.c.id,
+ package = self.tbl_binaries.c.package,
+ version = self.tbl_binaries.c.version,
+ maintainer_id = self.tbl_binaries.c.maintainer,
+ maintainer = relation(Maintainer),
+ source_id = self.tbl_binaries.c.source,
+ source = relation(DBSource),
+ arch_id = self.tbl_binaries.c.architecture,
+ architecture = relation(Architecture),
+ poolfile_id = self.tbl_binaries.c.file,
+ poolfile = relation(PoolFile),
+ binarytype = self.tbl_binaries.c.type,
+ fingerprint_id = self.tbl_binaries.c.sig_fpr,
+ fingerprint = relation(Fingerprint),
+ install_date = self.tbl_binaries.c.install_date,
+ binassociations = relation(BinAssociation,
+ primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin))))
+
+ mapper(Component, self.tbl_component,
+ properties = dict(component_id = self.tbl_component.c.id,
+ component_name = self.tbl_component.c.name))
+
+ mapper(DBConfig, self.tbl_config,
+ properties = dict(config_id = self.tbl_config.c.id))
+
+ mapper(ContentAssociation, self.tbl_content_associations,
+ properties = dict(ca_id = self.tbl_content_associations.c.id,
+ filename_id = self.tbl_content_associations.c.filename,
+ filename = relation(ContentFilename),
+ filepath_id = self.tbl_content_associations.c.filepath,
+ filepath = relation(ContentFilepath),
+ binary_id = self.tbl_content_associations.c.binary_pkg,
+ binary = relation(DBBinary)))
+
+
+ mapper(ContentFilename, self.tbl_content_file_names,
+ properties = dict(cafilename_id = self.tbl_content_file_names.c.id,
+ filename = self.tbl_content_file_names.c.file))
+
+ mapper(ContentFilepath, self.tbl_content_file_paths,
+ properties = dict(cafilepath_id = self.tbl_content_file_paths.c.id,
+ filepath = self.tbl_content_file_paths.c.path))
+
+ mapper(DSCFile, self.tbl_dsc_files,
+ properties = dict(dscfile_id = self.tbl_dsc_files.c.id,
+ source_id = self.tbl_dsc_files.c.source,
+ source = relation(DBSource),
+ poolfile_id = self.tbl_dsc_files.c.file,
+ poolfile = relation(PoolFile)))
+
+ mapper(PoolFile, self.tbl_files,
+ properties = dict(file_id = self.tbl_files.c.id,
+ filesize = self.tbl_files.c.size,
+ location_id = self.tbl_files.c.location,
+ location = relation(Location)))
+
+ mapper(Fingerprint, self.tbl_fingerprint,
+ properties = dict(fingerprint_id = self.tbl_fingerprint.c.id,
+ uid_id = self.tbl_fingerprint.c.uid,
+ uid = relation(Uid),
+ keyring_id = self.tbl_fingerprint.c.keyring,
+ keyring = relation(Keyring)))
+
+ mapper(Keyring, self.tbl_keyrings,
+ properties = dict(keyring_name = self.tbl_keyrings.c.name,
+ keyring_id = self.tbl_keyrings.c.id))
+
+ mapper(Location, self.tbl_location,
+ properties = dict(location_id = self.tbl_location.c.id,
+ component_id = self.tbl_location.c.component,
+ component = relation(Component),
+ archive_id = self.tbl_location.c.archive,
+ archive = relation(Archive),
+ archive_type = self.tbl_location.c.type))
+
+ mapper(Maintainer, self.tbl_maintainer,
+ properties = dict(maintainer_id = self.tbl_maintainer.c.id))
+
+ mapper(NewComment, self.tbl_new_comments,
+ properties = dict(comment_id = self.tbl_new_comments.c.id))
+
+ mapper(Override, self.tbl_override,
+ properties = dict(suite_id = self.tbl_override.c.suite,
+ suite = relation(Suite),
+ component_id = self.tbl_override.c.component,
+ component = relation(Component),
+ priority_id = self.tbl_override.c.priority,
+ priority = relation(Priority),
+ section_id = self.tbl_override.c.section,
+ section = relation(Section),
+ overridetype_id = self.tbl_override.c.type,
+ overridetype = relation(OverrideType)))
+
+ mapper(OverrideType, self.tbl_override_type,
+ properties = dict(overridetype = self.tbl_override_type.c.type,
+ overridetype_id = self.tbl_override_type.c.id))
+
+ mapper(PendingContentAssociation, self.tbl_pending_content_associations,
+ properties = dict(pca_id = self.tbl_pending_content_associations.c.id,
+ filepath_id = self.tbl_pending_content_associations.c.filepath,
+ filepath = relation(ContentFilepath),
+ filename_id = self.tbl_pending_content_associations.c.filename,
+ filename = relation(ContentFilename)))
+
+ mapper(Priority, self.tbl_priority,
+ properties = dict(priority_id = self.tbl_priority.c.id))
+
+ mapper(Queue, self.tbl_queue,
+ properties = dict(queue_id = self.tbl_queue.c.id))
+
+ mapper(QueueBuild, self.tbl_queue_build,
+ properties = dict(suite_id = self.tbl_queue_build.c.suite,
+ queue_id = self.tbl_queue_build.c.queue,
+ queue = relation(Queue, backref='queuebuild')))
+
+ mapper(Section, self.tbl_section,
+ properties = dict(section_id = self.tbl_section.c.id))
+
+ mapper(DBSource, self.tbl_source,
+ properties = dict(source_id = self.tbl_source.c.id,
+ version = self.tbl_source.c.version,
+ maintainer_id = self.tbl_source.c.maintainer,
+ maintainer = relation(Maintainer,
+ primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)),
+ poolfile_id = self.tbl_source.c.file,
+ poolfile = relation(PoolFile),
+ fingerprint_id = self.tbl_source.c.sig_fpr,
+ fingerprint = relation(Fingerprint),
+ changedby_id = self.tbl_source.c.changedby,
+ changedby = relation(Maintainer,
+ primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)),
+ srcfiles = relation(DSCFile,
+ primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)),
+ srcassociations = relation(SrcAssociation,
+ primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source))))
+
+ mapper(SrcAssociation, self.tbl_src_associations,
+ properties = dict(sa_id = self.tbl_src_associations.c.id,
+ suite_id = self.tbl_src_associations.c.suite,
+ suite = relation(Suite),
+ source_id = self.tbl_src_associations.c.source,
+ source = relation(DBSource)))
+
+ mapper(SrcFormat, self.tbl_src_format,
+ properties = dict(src_format_id = self.tbl_src_format.c.id,
+ format_name = self.tbl_src_format.c.format_name))
+
+ mapper(SrcUploader, self.tbl_src_uploaders,
+ properties = dict(uploader_id = self.tbl_src_uploaders.c.id,
+ source_id = self.tbl_src_uploaders.c.source,
+ source = relation(DBSource,
+ primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)),
+ maintainer_id = self.tbl_src_uploaders.c.maintainer,
+ maintainer = relation(Maintainer,
+ primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
+
+ mapper(Suite, self.tbl_suite,
+ properties = dict(suite_id = self.tbl_suite.c.id))
+
+ mapper(SuiteArchitecture, self.tbl_suite_architectures,
+ properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
+ suite = relation(Suite, backref='suitearchitectures'),
+ arch_id = self.tbl_suite_architectures.c.architecture,
+ architecture = relation(Architecture)))
+
+ mapper(SuiteSrcFormat, self.tbl_suite_src_formats,
+ properties = dict(suite_id = self.tbl_suite_src_formats.c.suite,
+ suite = relation(Suite, backref='suitesrcformats'),
+ src_format_id = self.tbl_suite_src_formats.c.src_format,
+ src_format = relation(SrcFormat)))
+
+ mapper(Uid, self.tbl_uid,
+ properties = dict(uid_id = self.tbl_uid.c.id,
+ fingerprint = relation(Fingerprint)))
+
+ ## Connection functions
+ def __createconn(self):
+ from config import Config
+ cnf = Config()
+ if cnf["DB::Host"]:
+ # TCP/IP
+ connstr = "postgres://%s" % cnf["DB::Host"]
+ if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
+ connstr += ":%s" % cnf["DB::Port"]
+ connstr += "/%s" % cnf["DB::Name"]
+ else:
+ # Unix Socket
+ connstr = "postgres:///%s" % cnf["DB::Name"]
+ if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
+ connstr += "?port=%s" % cnf["DB::Port"]
+
+ self.db_pg = create_engine(connstr, echo=self.debug)
+ self.db_meta = MetaData()
+ self.db_meta.bind = self.db_pg
+ self.db_smaker = sessionmaker(bind=self.db_pg,
+ autoflush=True,
+ autocommit=False)
+
+ self.__setuptables()
+ self.__setupmappers()
+
+ def session(self):
+ return self.db_smaker()
+
+__all__.append('DBConn')
- c = self.db_con.cursor()
- c.execute("BEGIN WORK")
- try:
- arch_id = self.get_architecture_id(package['Architecture'])
-
- # Remove any already existing recorded files for this package
- c.execute("""DELETE FROM pending_content_associations
- WHERE package=%(Package)s
- AND version=%(Version)s
- AND architecture=%(ArchID)s""", {'Package': package['Package'],
- 'Version': package['Version'],
- 'ArchID': arch_id})
-
- for fullpath in fullpaths:
- (path, file) = os.path.split(fullpath)
-
- if path.startswith( "./" ):
- path = path[2:]
- # Get the necessary IDs ...
- file_id = self.get_or_set_contents_file_id(file)
- path_id = self.get_or_set_contents_path_id(path)
-
- c.execute("""INSERT INTO pending_content_associations
- (package, version, architecture, filepath, filename)
- VALUES (%%(Package)s, %%(Version)s, '%d', '%d', '%d')"""
- % (arch_id, path_id, file_id), package )
-
- c.execute("COMMIT")
- return True
- except:
- traceback.print_exc()
- c.execute("ROLLBACK")
- return False
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Simple singleton class for storing info about Holding directory
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+import os
+from errno import ENOENT, EEXIST, EACCES
+import shutil
+
+from singleton import Singleton
+from config import Config
+from utils import fubar
+
+###############################################################################
+
+class Holding(Singleton):
+ def __init__(self, *args, **kwargs):
+ super(Holding, self).__init__(*args, **kwargs)
+
+ def _startup(self):
+ self.in_holding = {}
+ self.holding_dir = Config()["Dir::Queue::Holding"]
+
+ def copy_to_holding(self, filename):
+ base_filename = os.path.basename(filename)
+
+ dest = os.path.join(self.holding_dir, base_filename)
+ try:
+ fd = os.open(dest, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0640)
+ os.close(fd)
+ except OSError, e:
+ # Shouldn't happen, but will if, for example, someone lists a
+ # file twice in the .changes.
+ if e.errno == EEXIST:
+ return "%s: already exists in holding area; can not overwrite." % (base_filename)
+
+ try:
+ shutil.copy(filename, dest)
+ except IOError, e:
+ # In either case (ENOENT or EACCES) we want to remove the
+ # O_CREAT | O_EXCLed ghost file, so add the file to the list
+ # of 'in holding' even if it's not the real file.
+ if e.errno == ENOENT:
+ os.unlink(dest)
+ return "%s: can not copy to holding area: file not found." % (base_filename)
+
+ elif e.errno == EACCES:
+ os.unlink(dest)
+ return "%s: can not copy to holding area: read permission denied." % (base_filename)
+
+ self.in_holding[base_filename] = ""
+
+ return None
+
+ def clean(self):
+ cwd = os.getcwd()
+ os.chdir(self.holding_dir)
+ for f in self.in_holding.keys():
+ # TODO: Sanitize path in a much better manner...
+ if os.path.exists(f):
+ if f.find('/') != -1:
+ fubar("WTF? clean_holding() got a file ('%s') with / in it!" % (f))
+ else:
+ os.unlink(f)
+ self.in_holding = {}
+ os.chdir(cwd)
+
+++ /dev/null
-#!/usr/bin/env python
-
-"""
-Logging functions
-
-@contact: Debian FTP Master <ftpmaster@debian.org>
-@copyright: 2001, 2002, 2006 James Troup <james@nocrew.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-################################################################################
-
-import os
-import pwd
-import time
-import sys
-import utils
-
-################################################################################
-
-class Logger:
- "Logger object"
- Cnf = None
- logfile = None
- program = None
-
- def __init__ (self, Cnf, program, debug=0):
- "Initialize a new Logger object"
- self.Cnf = Cnf
- self.program = program
- # Create the log directory if it doesn't exist
- logdir = Cnf["Dir::Log"]
- if not os.path.exists(logdir):
- umask = os.umask(00000)
- os.makedirs(logdir, 02775)
- os.umask(umask)
- # Open the logfile
- logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m"))
- logfile = None
- if debug:
- logfile = sys.stderr
- else:
- umask = os.umask(00002)
- logfile = utils.open_file(logfilename, 'a')
- os.umask(umask)
- self.logfile = logfile
- # Log the start of the program
- user = pwd.getpwuid(os.getuid())[0]
- self.log(["program start", user])
-
- def log (self, details):
- "Log an event"
- # Prepend the timestamp and program name
- details.insert(0, self.program)
- timestamp = time.strftime("%Y%m%d%H%M%S")
- details.insert(0, timestamp)
- # Force the contents of the list to be string.join-able
- details = [ str(i) for i in details ]
- # Write out the log in TSV
- self.logfile.write("|".join(details)+'\n')
- # Flush the output to enable tail-ing
- self.logfile.flush()
-
- def close (self):
- "Close a Logger object"
- self.log(["program end"])
- self.logfile.flush()
- self.logfile.close()
###############################################################################
-import cPickle
import errno
import os
import pg
import apt_inst
import apt_pkg
import utils
-import database
-from dak_exceptions import *
-from regexes import re_default_answer, re_fdnic, re_bin_only_nmu
-
+import commands
+import shutil
+import textwrap
+import tempfile
from types import *
+import yaml
+
+from dak_exceptions import *
+from changes import *
+from regexes import *
+from config import Config
+from holding import Holding
+from dbconn import *
+from summarystats import SummaryStats
+from utils import parse_changes, check_dsc_files
+from textutils import fix_maintainer
+from binary import Binary
+
###############################################################################
+def get_type(f, session):
+ """
+ Get the file type of C{f}
+
+ @type f: dict
+ @param f: file entry from Changes object
+
+ @type session: SQLA Session
+ @param session: SQL Alchemy session object
+
+ @rtype: string
+ @return: filetype
+
+ """
+ # Determine the type
+ if f.has_key("dbtype"):
+ file_type = f["dbtype"]
+ elif re_source_ext.match(f["type"]):
+ file_type = "dsc"
+ else:
+ utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
+
+ # Validate the override type
+ type_id = get_override_type(file_type, session)
+ if type_id is None:
+ utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
+
+ return file_type
+
+################################################################################
+
# Determine what parts in a .changes are NEW
-def determine_new(changes, files, projectB, warn=1):
+def determine_new(changes, files, warn=1):
"""
Determine what parts in a C{changes} file are NEW.
@type files: Upload.Pkg.files dict
@param files: Files dictionary
- @type projectB: pgobject
- @param projectB: DB handle
-
@type warn: bool
@param warn: Warn if overrides are added for (old)stable
"""
new = {}
+ session = DBConn().session()
+
# Build up a list of potentially new things
- for file_entry in files.keys():
- f = files[file_entry]
+ for name, f in files.items():
# Skip byhand elements
if f["type"] == "byhand":
continue
pkg = f["package"]
priority = f["priority"]
section = f["section"]
- file_type = get_type(f)
+ file_type = get_type(f, session)
component = f["component"]
if file_type == "dsc":
priority = "source"
+
if not new.has_key(pkg):
new[pkg] = {}
new[pkg]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["type"] = file_type
new[pkg]["component"] = component
- new[pkg]["files"].append(file_entry)
+
+ new[pkg]["files"].append(name)
+
if f.has_key("othercomponents"):
new[pkg]["othercomponents"] = f["othercomponents"]
for suite in changes["suite"].keys():
- suite_id = database.get_suite_id(suite)
for pkg in new.keys():
- component_id = database.get_component_id(new[pkg]["component"])
- type_id = database.get_override_type_id(new[pkg]["type"])
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
- ql = q.getresult()
- if ql:
+ ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
+ if len(ql) > 0:
for file_entry in new[pkg]["files"]:
if files[file_entry].has_key("new"):
del files[file_entry]["new"]
del new[pkg]
if warn:
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!"
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!"
+ for s in ['stable', 'oldstable']:
+ if changes["suite"].has_key(s):
+ print "WARNING: overrides will be added for %s!" % s
for pkg in new.keys():
if new[pkg].has_key("othercomponents"):
print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
- return new
-
-################################################################################
-
-def get_type(file):
- """
- Get the file type of C{file}
-
- @type file: dict
- @param file: file entry
-
- @rtype: string
- @return: filetype
-
- """
- # Determine the type
- if file.has_key("dbtype"):
- file_type = file["dbtype"]
- elif file["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
- file_type = "dsc"
- else:
- utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
-
- # Validate the override type
- type_id = database.get_override_type_id(file_type)
- if type_id == -1:
- utils.fubar("invalid type (%s) for new. Say wha?" % (file_type))
+ session.close()
- return file_type
+ return new
################################################################################
-
-
def check_valid(new):
"""
Check if section and priority for NEW packages exist in database.
"""
for pkg in new.keys():
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
+ section_name = new[pkg]["section"]
+ priority_name = new[pkg]["priority"]
file_type = new[pkg]["type"]
- new[pkg]["section id"] = database.get_section_id(section)
- new[pkg]["priority id"] = database.get_priority_id(new[pkg]["priority"])
+
+ section = get_section(section_name)
+ if section is None:
+ new[pkg]["section id"] = -1
+ else:
+ new[pkg]["section id"] = section.section_id
+
+ priority = get_priority(priority_name)
+ if priority is None:
+ new[pkg]["priority id"] = -1
+ else:
+ new[pkg]["priority id"] = priority.priority_id
+
# Sanity checks
- di = section.find("debian-installer") != -1
- if (di and file_type not in ("udeb", "dsc")) or (not di and file_type == "udeb"):
+ di = section_name.find("debian-installer") != -1
+
+ # If d-i, we must be udeb and vice-versa
+ if (di and file_type not in ("udeb", "dsc")) or \
+ (not di and file_type == "udeb"):
new[pkg]["section id"] = -1
+
+ # If dsc we need to be source and vice-versa
if (priority == "source" and file_type != "dsc") or \
(priority != "source" and file_type == "dsc"):
new[pkg]["priority id"] = -1
+###############################################################################
+
+def lookup_uid_from_fingerprint(fpr, session):
+ uid = None
+ uid_name = ""
+ # This is a stupid default, but see the comments below
+ is_dm = False
+
+ user = get_uid_from_fingerprint(fpr, session)
+
+ if user is not None:
+ uid = user.uid
+ if user.name is None:
+ uid_name = ''
+ else:
+ uid_name = user.name
+
+ # Check the relevant fingerprint (which we have to have)
+ for f in user.fingerprint:
+ if f.fingerprint == fpr:
+ is_dm = f.keyring.debian_maintainer
+ break
+
+ return (uid, uid_name, is_dm)
###############################################################################
-class Pkg:
- """ Convenience wrapper to carry around all the package information """
- def __init__(self, **kwds):
- self.__dict__.update(kwds)
+# Used by Upload.check_timestamps
+class TarTime(object):
+ def __init__(self, future_cutoff, past_cutoff):
+ self.reset()
+ self.future_cutoff = future_cutoff
+ self.past_cutoff = past_cutoff
+
+ def reset(self):
+ self.future_files = {}
+ self.ancient_files = {}
- def update(self, **kwds):
- self.__dict__.update(kwds)
+ def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
+ if MTime > self.future_cutoff:
+ self.future_files[Name] = MTime
+ if MTime < self.past_cutoff:
+ self.ancient_files[Name] = MTime
###############################################################################
-class Upload:
+class Upload(object):
"""
Everything that has to do with an upload processed.
"""
- def __init__(self, Cnf):
- """
- Initialize various variables and the global substitution template mappings.
- Also connect to the DB and initialize the Database module.
-
- """
- self.Cnf = Cnf
- self.accept_count = 0
- self.accept_bytes = 0L
- self.reject_message = ""
- self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {})
-
- # Initialize the substitution template mapping global
- Subst = self.Subst = {}
- Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
- Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]
- Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
- Subst["__DAK_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
-
- self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, self.projectB)
+ def __init__(self):
+ self.logger = None
+ self.pkg = Changes()
+ self.reset()
###########################################################################
- def init_vars (self):
- """ Reset a number of entries from our Pkg object. """
- self.pkg.changes.clear()
- self.pkg.dsc.clear()
- self.pkg.files.clear()
- self.pkg.dsc_files.clear()
- self.pkg.orig_tar_id = None
- self.pkg.orig_tar_location = ""
- self.pkg.orig_tar_gz = None
+ def reset (self):
+ """ Reset a number of internal variables."""
+
+ # Initialize the substitution template map
+ cnf = Config()
+ self.Subst = {}
+ self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
+ self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
+ self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+
+ self.rejects = []
+ self.warnings = []
+ self.notes = []
+
+ self.pkg.reset()
+
+ def package_info(self):
+ msg = ''
+
+ if len(self.rejects) > 0:
+ msg += "Reject Reasons:\n"
+ msg += "\n".join(self.rejects)
+
+ if len(self.warnings) > 0:
+ msg += "Warnings:\n"
+ msg += "\n".join(self.warnings)
+
+ if len(self.notes) > 0:
+ msg += "Notes:\n"
+ msg += "\n".join(self.notes)
+
+ return msg
###########################################################################
+ def update_subst(self):
+ """ Set up the per-package template substitution mappings """
+
+ cnf = Config()
+
+ # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
+ if not self.pkg.changes.has_key("architecture") or not \
+ isinstance(self.pkg.changes["architecture"], DictType):
+ self.pkg.changes["architecture"] = { "Unknown" : "" }
+
+ # and maintainer2047 may not exist.
+ if not self.pkg.changes.has_key("maintainer2047"):
+ self.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
+
+ self.Subst["__ARCHITECTURE__"] = " ".join(self.pkg.changes["architecture"].keys())
+ self.Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
+ self.Subst["__FILE_CONTENTS__"] = self.pkg.changes.get("filecontents", "")
+
+ # For source uploads the Changed-By field wins; otherwise Maintainer wins.
+ if self.pkg.changes["architecture"].has_key("source") and \
+ self.pkg.changes["changedby822"] != "" and \
+ (self.pkg.changes["changedby822"] != self.pkg.changes["maintainer822"]):
+
+ self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["changedby2047"]
+ self.Subst["__MAINTAINER_TO__"] = "%s, %s" % (self.pkg.changes["changedby2047"], self.pkg.changes["maintainer2047"])
+ self.Subst["__MAINTAINER__"] = self.pkg.changes.get("changed-by", "Unknown")
+ else:
+ self.Subst["__MAINTAINER_FROM__"] = self.pkg.changes["maintainer2047"]
+ self.Subst["__MAINTAINER_TO__"] = self.pkg.changes["maintainer2047"]
+ self.Subst["__MAINTAINER__"] = self.pkg.changes.get("maintainer", "Unknown")
+
+ if "sponsoremail" in self.pkg.changes:
+ self.Subst["__MAINTAINER_TO__"] += ", %s" % self.pkg.changes["sponsoremail"]
+
+ if cnf.has_key("Dinstall::TrackingServer") and self.pkg.changes.has_key("source"):
+ self.Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
+
+ # Apply any global override of the Maintainer field
+ if cnf.get("Dinstall::OverrideMaintainer"):
+ self.Subst["__MAINTAINER_TO__"] = cnf["Dinstall::OverrideMaintainer"]
+ self.Subst["__MAINTAINER_FROM__"] = cnf["Dinstall::OverrideMaintainer"]
+
+ self.Subst["__REJECT_MESSAGE__"] = self.package_info()
+ self.Subst["__SOURCE__"] = self.pkg.changes.get("source", "Unknown")
+ self.Subst["__VERSION__"] = self.pkg.changes.get("version", "Unknown")
- def update_vars (self):
+ ###########################################################################
+ def load_changes(self, filename):
"""
- Update our Pkg object by reading a previously created cPickle .dak dumpfile.
+ @rtype: boolean
+ @rvalue: whether the changes file was valid or not. We may want to
+ reject even if this is True (see what gets put in self.rejects).
+ This is simply to prevent us even trying things later which will
+ fail because we couldn't properly parse the file.
"""
- dump_filename = self.pkg.changes_file[:-8]+".dak"
- dump_file = utils.open_file(dump_filename)
- p = cPickle.Unpickler(dump_file)
+ Cnf = Config()
+ self.pkg.changes_file = filename
+
+ # Parse the .changes field into a dictionary
+ try:
+ self.pkg.changes.update(parse_changes(filename))
+ except CantOpenError:
+ self.rejects.append("%s: can't read file." % (filename))
+ return False
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
+ return False
+ except ChangesUnicodeError:
+ self.rejects.append("%s: changes file not proper utf-8" % (filename))
+ return False
+
+ # Parse the Files field from the .changes into another dictionary
+ try:
+ self.pkg.files.update(utils.build_file_list(self.pkg.changes))
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
+ return False
+ except UnknownFormatError, format:
+ self.rejects.append("%s: unknown format '%s'." % (filename, format))
+ return False
+
+ # Check for mandatory fields
+ for i in ("distribution", "source", "binary", "architecture",
+ "version", "maintainer", "files", "changes", "description"):
+ if not self.pkg.changes.has_key(i):
+ # Avoid undefined errors later
+ self.rejects.append("%s: Missing mandatory field `%s'." % (filename, i))
+ return False
+
+ # Strip a source version in brackets from the source field
+ if re_strip_srcver.search(self.pkg.changes["source"]):
+ self.pkg.changes["source"] = re_strip_srcver.sub('', self.pkg.changes["source"])
+
+ # Ensure the source field is a valid package name.
+ if not re_valid_pkg_name.match(self.pkg.changes["source"]):
+ self.rejects.append("%s: invalid source name '%s'." % (filename, self.pkg.changes["source"]))
+
+ # Split multi-value fields into a lower-level dictionary
+ for i in ("architecture", "distribution", "binary", "closes"):
+ o = self.pkg.changes.get(i, "")
+ if o != "":
+ del self.pkg.changes[i]
+
+ self.pkg.changes[i] = {}
+
+ for j in o.split():
+ self.pkg.changes[i][j] = 1
+
+ # Fix the Maintainer: field to be RFC822/2047 compatible
+ try:
+ (self.pkg.changes["maintainer822"],
+ self.pkg.changes["maintainer2047"],
+ self.pkg.changes["maintainername"],
+ self.pkg.changes["maintaineremail"]) = \
+ fix_maintainer (self.pkg.changes["maintainer"])
+ except ParseMaintError, msg:
+ self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
+ % (filename, changes["maintainer"], msg))
+
+ # ...likewise for the Changed-By: field if it exists.
+ try:
+ (self.pkg.changes["changedby822"],
+ self.pkg.changes["changedby2047"],
+ self.pkg.changes["changedbyname"],
+ self.pkg.changes["changedbyemail"]) = \
+ fix_maintainer (self.pkg.changes.get("changed-by", ""))
+ except ParseMaintError, msg:
+ self.pkg.changes["changedby822"] = ""
+ self.pkg.changes["changedby2047"] = ""
+ self.pkg.changes["changedbyname"] = ""
+ self.pkg.changes["changedbyemail"] = ""
+
+ self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
+ % (filename, changes["changed-by"], msg))
+
+ # Ensure all the values in Closes: are numbers
+ if self.pkg.changes.has_key("closes"):
+ for i in self.pkg.changes["closes"].keys():
+ if re_isanum.match (i) == None:
+ self.rejects.append(("%s: `%s' from Closes field isn't a number." % (filename, i)))
+
+ # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
+ self.pkg.changes["chopversion"] = re_no_epoch.sub('', self.pkg.changes["version"])
+ self.pkg.changes["chopversion2"] = re_no_revision.sub('', self.pkg.changes["chopversion"])
+
+ # Check there isn't already a changes file of the same name in one
+ # of the queue directories.
+ base_filename = os.path.basename(filename)
+ for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
+ if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
+ self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
+
+ # Check the .changes is non-empty
+ if not self.pkg.files:
+ self.rejects.append("%s: nothing to do (Files field is empty)." % (base_filename))
+ return False
+
+ # Changes was syntactically valid even if we'll reject
+ return True
- self.pkg.changes.update(p.load())
- self.pkg.dsc.update(p.load())
- self.pkg.files.update(p.load())
- self.pkg.dsc_files.update(p.load())
+ ###########################################################################
+
+ def check_distributions(self):
+ "Check and map the Distribution field"
+
+ Cnf = Config()
+
+ # Handle suite mappings
+ for m in Cnf.ValueList("SuiteMappings"):
+ args = m.split()
+ mtype = args[0]
+ if mtype == "map" or mtype == "silent-map":
+ (source, dest) = args[1:3]
+ if self.pkg.changes["distribution"].has_key(source):
+ del self.pkg.changes["distribution"][source]
+ self.pkg.changes["distribution"][dest] = 1
+ if mtype != "silent-map":
+ self.notes.append("Mapping %s to %s." % (source, dest))
+ if self.pkg.changes.has_key("distribution-version"):
+ if self.pkg.changes["distribution-version"].has_key(source):
+ self.pkg.changes["distribution-version"][source]=dest
+ elif mtype == "map-unreleased":
+ (source, dest) = args[1:3]
+ if self.pkg.changes["distribution"].has_key(source):
+ for arch in self.pkg.changes["architecture"].keys():
+ if arch not in [ a.arch_string for a in get_suite_architectures(source) ]:
+ self.notes.append("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch))
+ del self.pkg.changes["distribution"][source]
+ self.pkg.changes["distribution"][dest] = 1
+ break
+ elif mtype == "ignore":
+ suite = args[1]
+ if self.pkg.changes["distribution"].has_key(suite):
+ del self.pkg.changes["distribution"][suite]
+ self.warnings.append("Ignoring %s as a target suite." % (suite))
+ elif mtype == "reject":
+ suite = args[1]
+ if self.pkg.changes["distribution"].has_key(suite):
+ self.rejects.append("Uploads to %s are not accepted." % (suite))
+ elif mtype == "propup-version":
+ # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes"
+ #
+ # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'}
+ if self.pkg.changes["distribution"].has_key(args[1]):
+ self.pkg.changes.setdefault("distribution-version", {})
+ for suite in args[2:]:
+ self.pkg.changes["distribution-version"][suite] = suite
- self.pkg.orig_tar_id = p.load()
- self.pkg.orig_tar_location = p.load()
+ # Ensure there is (still) a target distribution
+ if len(self.pkg.changes["distribution"].keys()) < 1:
+ self.rejects.append("No valid distribution remaining.")
- dump_file.close()
+ # Ensure target distributions exist
+ for suite in self.pkg.changes["distribution"].keys():
+ if not Cnf.has_key("Suite::%s" % (suite)):
+ self.rejects.append("Unknown distribution `%s'." % (suite))
###########################################################################
+ def binary_file_checks(self, f, session):
+ cnf = Config()
+ entry = self.pkg.files[f]
- def dump_vars(self, dest_dir):
- """
- Dump our Pkg object into a cPickle file.
+ # Extract package control information
+ deb_file = utils.open_file(f)
+ try:
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))
+ except:
+ self.rejects.append("%s: debExtractControl() raised %s." % (f, sys.exc_type))
+ deb_file.close()
+ # Can't continue, none of the checks on control would work.
+ return
+
+ # Check for mandantory "Description:"
+ deb_file.seek(0)
+ try:
+ apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file))["Description"] + '\n'
+ except:
+ self.rejects.append("%s: Missing Description in binary package" % (f))
+ return
+
+ deb_file.close()
+
+ # Check for mandatory fields
+ for field in [ "Package", "Architecture", "Version" ]:
+ if control.Find(field) == None:
+ # Can't continue
+ self.rejects.append("%s: No %s field in control." % (f, field))
+ return
+
+ # Ensure the package name matches the one give in the .changes
+ if not self.pkg.changes["binary"].has_key(control.Find("Package", "")):
+ self.rejects.append("%s: control file lists name as `%s', which isn't in changes file." % (f, control.Find("Package", "")))
+
+ # Validate the package field
+ package = control.Find("Package")
+ if not re_valid_pkg_name.match(package):
+ self.rejects.append("%s: invalid package name '%s'." % (f, package))
+
+ # Validate the version field
+ version = control.Find("Version")
+ if not re_valid_version.match(version):
+ self.rejects.append("%s: invalid version number '%s'." % (f, version))
+
+ # Ensure the architecture of the .deb is one we know about.
+ default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
+ architecture = control.Find("Architecture")
+ upload_suite = self.pkg.changes["distribution"].keys()[0]
+
+ if architecture not in [a.arch_string for a in get_suite_architectures(default_suite, session)] \
+ and architecture not in [a.arch_string for a in get_suite_architectures(upload_suite, session)]:
+ self.rejects.append("Unknown architecture '%s'." % (architecture))
+
+ # Ensure the architecture of the .deb is one of the ones
+ # listed in the .changes.
+ if not self.pkg.changes["architecture"].has_key(architecture):
+ self.rejects.append("%s: control file lists arch as `%s', which isn't in changes file." % (f, architecture))
+
+ # Sanity-check the Depends field
+ depends = control.Find("Depends")
+ if depends == '':
+ self.rejects.append("%s: Depends field is empty." % (f))
+
+ # Sanity-check the Provides field
+ provides = control.Find("Provides")
+ if provides:
+ provide = re_spacestrip.sub('', provides)
+ if provide == '':
+ self.rejects.append("%s: Provides field is empty." % (f))
+ prov_list = provide.split(",")
+ for prov in prov_list:
+ if not re_valid_pkg_name.match(prov):
+ self.rejects.append("%s: Invalid Provides field content %s." % (f, prov))
+
+ # Check the section & priority match those given in the .changes (non-fatal)
+ if control.Find("Section") and entry["section"] != "" \
+ and entry["section"] != control.Find("Section"):
+ self.warnings.append("%s control file lists section as `%s', but changes file has `%s'." % \
+ (f, control.Find("Section", ""), entry["section"]))
+ if control.Find("Priority") and entry["priority"] != "" \
+ and entry["priority"] != control.Find("Priority"):
+ self.warnings.append("%s control file lists priority as `%s', but changes file has `%s'." % \
+ (f, control.Find("Priority", ""), entry["priority"]))
+
+ entry["package"] = package
+ entry["architecture"] = architecture
+ entry["version"] = version
+ entry["maintainer"] = control.Find("Maintainer", "")
+
+ if f.endswith(".udeb"):
+ self.pkg.files[f]["dbtype"] = "udeb"
+ elif f.endswith(".deb"):
+ self.pkg.files[f]["dbtype"] = "deb"
+ else:
+ self.rejects.append("%s is neither a .deb or a .udeb." % (f))
+
+ entry["source"] = control.Find("Source", entry["package"])
+
+ # Get the source version
+ source = entry["source"]
+ source_version = ""
+
+ if source.find("(") != -1:
+ m = re_extract_src_version.match(source)
+ source = m.group(1)
+ source_version = m.group(2)
+
+ if not source_version:
+ source_version = self.pkg.files[f]["version"]
+
+ entry["source package"] = source
+ entry["source version"] = source_version
+
+ # Ensure the filename matches the contents of the .deb
+ m = re_isadeb.match(f)
+
+ # package name
+ file_package = m.group(1)
+ if entry["package"] != file_package:
+ self.rejects.append("%s: package part of filename (%s) does not match package name in the %s (%s)." % \
+ (f, file_package, entry["dbtype"], entry["package"]))
+ epochless_version = re_no_epoch.sub('', control.Find("Version"))
+
+ # version
+ file_version = m.group(2)
+ if epochless_version != file_version:
+ self.rejects.append("%s: version part of filename (%s) does not match package version in the %s (%s)." % \
+ (f, file_version, entry["dbtype"], epochless_version))
+
+ # architecture
+ file_architecture = m.group(3)
+ if entry["architecture"] != file_architecture:
+ self.rejects.append("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % \
+ (f, file_architecture, entry["dbtype"], entry["architecture"]))
+
+ # Check for existent source
+ source_version = entry["source version"]
+ source_package = entry["source package"]
+ if self.pkg.changes["architecture"].has_key("source"):
+ if source_version != self.pkg.changes["version"]:
+ self.rejects.append("source version (%s) for %s doesn't match changes version %s." % \
+ (source_version, f, self.pkg.changes["version"]))
+ else:
+ # Check in the SQL database
+ if not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys(), session):
+ # Check in one of the other directories
+ source_epochless_version = re_no_epoch.sub('', source_version)
+ dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
+ if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
+ entry["byhand"] = 1
+ elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
+ entry["new"] = 1
+ else:
+ dsc_file_exists = False
+ for myq in ["Accepted", "Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
+ if cnf.has_key("Dir::Queue::%s" % (myq)):
+ if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
+ dsc_file_exists = True
+ break
+
+ if not dsc_file_exists:
+ self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))
+
+ # Check the version and for file overwrites
+ self.check_binary_against_db(f, session)
+
+ # Temporarily disable contents generation until we change the table storage layout
+ #b = Binary(f)
+ #b.scan_package()
+ #if len(b.rejects) > 0:
+ # for j in b.rejects:
+ # self.rejects.append(j)
+
+ def source_file_checks(self, f, session):
+ entry = self.pkg.files[f]
+
+ m = re_issource.match(f)
+ if not m:
+ return
+
+ entry["package"] = m.group(1)
+ entry["version"] = m.group(2)
+ entry["type"] = m.group(3)
+
+ # Ensure the source package name matches the Source filed in the .changes
+ if self.pkg.changes["source"] != entry["package"]:
+ self.rejects.append("%s: changes file doesn't say %s for Source" % (f, entry["package"]))
+
+ # Ensure the source version matches the version in the .changes file
+ if re_is_orig_source.match(f):
+ changes_version = self.pkg.changes["chopversion2"]
+ else:
+ changes_version = self.pkg.changes["chopversion"]
- @type dest_dir: string
- @param dest_dir: Path where the dumpfile should be stored
+ if changes_version != entry["version"]:
+ self.rejects.append("%s: should be %s according to changes file." % (f, changes_version))
+
+ # Ensure the .changes lists source in the Architecture field
+ if not self.pkg.changes["architecture"].has_key("source"):
+ self.rejects.append("%s: changes file doesn't list `source' in Architecture field." % (f))
+
+ # Check the signature of a .dsc file
+ if entry["type"] == "dsc":
+ # check_signature returns either:
+ # (None, [list, of, rejects]) or (signature, [])
+ (self.pkg.dsc["fingerprint"], rejects) = utils.check_signature(f)
+ for j in rejects:
+ self.rejects.append(j)
+
+ entry["architecture"] = "source"
+
+ def per_suite_file_checks(self, f, suite, session):
+ cnf = Config()
+ entry = self.pkg.files[f]
+ archive = utils.where_am_i()
+
+ # Skip byhand
+ if entry.has_key("byhand"):
+ return
+
+ # Check we have fields we need to do these checks
+ oktogo = True
+ for m in ['component', 'package', 'priority', 'size', 'md5sum']:
+ if not entry.has_key(m):
+ self.rejects.append("file '%s' does not have field %s set" % (f, m))
+ oktogo = False
+
+ if not oktogo:
+ return
+
+ # Handle component mappings
+ for m in cnf.ValueList("ComponentMappings"):
+ (source, dest) = m.split()
+ if entry["component"] == source:
+ entry["original component"] = source
+ entry["component"] = dest
+
+ # Ensure the component is valid for the target suite
+ if cnf.has_key("Suite:%s::Components" % (suite)) and \
+ entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
+ self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
+ return
+
+ # Validate the component
+ if not get_component(entry["component"], session):
+ self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
+ return
+
+ # See if the package is NEW
+ if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), f, session):
+ entry["new"] = 1
+
+ # Validate the priority
+ if entry["priority"].find('/') != -1:
+ self.rejects.append("file '%s' has invalid priority '%s' [contains '/']." % (f, entry["priority"]))
+
+ # Determine the location
+ location = cnf["Dir::Pool"]
+ l = get_location(location, entry["component"], archive, session)
+ if l is None:
+ self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
+ entry["location id"] = -1
+ else:
+ entry["location id"] = l.location_id
+
+ # Check the md5sum & size against existing files (if any)
+ entry["pool name"] = utils.poolify(self.pkg.changes["source"], entry["component"])
+
+ found, poolfile = check_poolfile(os.path.join(entry["pool name"], f),
+ entry["size"], entry["md5sum"], entry["location id"])
+
+ if found is None:
+ self.rejects.append("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (f))
+ elif found is False and poolfile is not None:
+ self.rejects.append("md5sum and/or size mismatch on existing copy of %s." % (f))
+ else:
+ if poolfile is None:
+ entry["files id"] = None
+ else:
+ entry["files id"] = poolfile.file_id
+
+ # Check for packages that have moved from one component to another
+ entry['suite'] = suite
+ res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
+ if res.rowcount > 0:
+ entry["othercomponents"] = res.fetchone()[0]
+
+ def check_files(self, action=True):
+ archive = utils.where_am_i()
+ file_keys = self.pkg.files.keys()
+ holding = Holding()
+ cnf = Config()
+
+ # XXX: As far as I can tell, this can no longer happen - see
+ # comments by AJ in old revisions - mhy
+ # if reprocess is 2 we've already done this and we're checking
+ # things again for the new .orig.tar.gz.
+ # [Yes, I'm fully aware of how disgusting this is]
+ if action and self.reprocess < 2:
+ cwd = os.getcwd()
+ os.chdir(self.pkg.directory)
+ for f in file_keys:
+ ret = holding.copy_to_holding(f)
+ if ret is not None:
+ # XXX: Should we bail out here or try and continue?
+ self.rejects.append(ret)
+
+ os.chdir(cwd)
+
+ # Check there isn't already a .changes or .dak file of the same name in
+ # the proposed-updates "CopyChanges" or "CopyDotDak" storage directories.
+ # [NB: this check must be done post-suite mapping]
+ base_filename = os.path.basename(self.pkg.changes_file)
+ dot_dak_filename = base_filename[:-8] + ".dak"
+
+ for suite in self.pkg.changes["distribution"].keys():
+ copychanges = "Suite::%s::CopyChanges" % (suite)
+ if cnf.has_key(copychanges) and \
+ os.path.exists(os.path.join(cnf[copychanges], base_filename)):
+ self.rejects.append("%s: a file with this name already exists in %s" \
+ % (base_filename, cnf[copychanges]))
+
+ copy_dot_dak = "Suite::%s::CopyDotDak" % (suite)
+ if cnf.has_key(copy_dot_dak) and \
+ os.path.exists(os.path.join(cnf[copy_dot_dak], dot_dak_filename)):
+ self.rejects.append("%s: a file with this name already exists in %s" \
+ % (dot_dak_filename, Cnf[copy_dot_dak]))
+
+ self.reprocess = 0
+ has_binaries = False
+ has_source = False
+
+ session = DBConn().session()
+
+ for f, entry in self.pkg.files.items():
+ # Ensure the file does not already exist in one of the accepted directories
+ for d in [ "Accepted", "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
+ if not cnf.has_key("Dir::Queue::%s" % (d)): continue
+ if os.path.exists(cnf["Dir::Queue::%s" % (d) ] + '/' + f):
+ self.rejects.append("%s file already exists in the %s directory." % (f, d))
+
+ if not re_taint_free.match(f):
+ self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
+
+ # Check the file is readable
+ if os.access(f, os.R_OK) == 0:
+ # When running in -n, copy_to_holding() won't have
+ # generated the reject_message, so we need to.
+ if action:
+ if os.path.exists(f):
+ self.rejects.append("Can't read `%s'. [permission denied]" % (f))
+ else:
+ self.rejects.append("Can't read `%s'. [file not found]" % (f))
+ entry["type"] = "unreadable"
+ continue
- @note: This could just dump the dictionaries as is, but I'd like to avoid this so
- there's some idea of what process-accepted & process-new use from
- process-unchecked. (JT)
+ # If it's byhand skip remaining checks
+ if entry["section"] == "byhand" or entry["section"][:4] == "raw-":
+ entry["byhand"] = 1
+ entry["type"] = "byhand"
+ # Checks for a binary package...
+ elif re_isadeb.match(f):
+ has_binaries = True
+ entry["type"] = "deb"
+
+ # This routine appends to self.rejects/warnings as appropriate
+ self.binary_file_checks(f, session)
+
+ # Checks for a source package...
+ elif re_issource.match(f):
+ has_source = True
+
+ # This routine appends to self.rejects/warnings as appropriate
+ self.source_file_checks(f, session)
+
+ # Not a binary or source package? Assume byhand...
+ else:
+ entry["byhand"] = 1
+ entry["type"] = "byhand"
+
+ # Per-suite file checks
+ entry["oldfiles"] = {}
+ for suite in self.pkg.changes["distribution"].keys():
+ self.per_suite_file_checks(f, suite, session)
+
+ session.close()
+
+ # If the .changes file says it has source, it must have source.
+ if self.pkg.changes["architecture"].has_key("source"):
+ if not has_source:
+ self.rejects.append("no source found and Architecture line in changes mention source.")
+
+ if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
+ self.rejects.append("source only uploads are not supported.")
+
+ ###########################################################################
+ def check_dsc(self, action=True, session=None):
+ """Returns bool indicating whether or not the source changes are valid"""
+ # Ensure there is source to check
+ if not self.pkg.changes["architecture"].has_key("source"):
+ return True
+
+ # Find the .dsc
+ dsc_filename = None
+ for f, entry in self.pkg.files.items():
+ if entry["type"] == "dsc":
+ if dsc_filename:
+ self.rejects.append("can not process a .changes file with multiple .dsc's.")
+ return False
+ else:
+ dsc_filename = f
+
+ # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
+ if not dsc_filename:
+ self.rejects.append("source uploads must contain a dsc file")
+ return False
+
+ # Parse the .dsc file
+ try:
+ self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1))
+ except CantOpenError:
+ # if not -n copy_to_holding() will have done this for us...
+ if not action:
+ self.rejects.append("%s: can't read file." % (dsc_filename))
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
+ except InvalidDscError, line:
+ self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
+ except ChangesUnicodeError:
+ self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
+
+ # Build up the file list of files mentioned by the .dsc
+ try:
+ self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
+ except NoFilesFieldError:
+ self.rejects.append("%s: no Files: field." % (dsc_filename))
+ return False
+ except UnknownFormatError, format:
+ self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
+ return False
+ except ParseChangesError, line:
+ self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
+ return False
+
+ # Enforce mandatory fields
+ for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
+ if not self.pkg.dsc.has_key(i):
+ self.rejects.append("%s: missing mandatory field `%s'." % (dsc_filename, i))
+ return False
+
+ # Validate the source and version fields
+ if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
+ self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
+ if not re_valid_version.match(self.pkg.dsc["version"]):
+ self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
+
+ # Only a limited list of source formats are allowed in each suite
+ for dist in self.pkg.changes["distribution"].keys():
+ allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
+ if self.pkg.dsc["format"] not in allowed:
+ self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
+
+ # Validate the Maintainer field
+ try:
+ # We ignore the return value
+ fix_maintainer(self.pkg.dsc["maintainer"])
+ except ParseMaintError, msg:
+ self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
+ % (dsc_filename, self.pkg.dsc["maintainer"], msg))
+
+ # Validate the build-depends field(s)
+ for field_name in [ "build-depends", "build-depends-indep" ]:
+ field = self.pkg.dsc.get(field_name)
+ if field:
+ # Have apt try to parse them...
+ try:
+ apt_pkg.ParseSrcDepends(field)
+ except:
+ self.rejects.append("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title()))
+
+ # Ensure the version number in the .dsc matches the version number in the .changes
+ epochless_dsc_version = re_no_epoch.sub('', self.pkg.dsc["version"])
+ changes_version = self.pkg.files[dsc_filename]["version"]
+
+ if epochless_dsc_version != self.pkg.files[dsc_filename]["version"]:
+ self.rejects.append("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version))
+
+ # Ensure the Files field contain only what's expected
+ self.rejects.extend(check_dsc_files(dsc_filename, self.pkg.dsc, self.pkg.dsc_files))
+
+ # Ensure source is newer than existing source in target suites
+ session = DBConn().session()
+ self.check_source_against_db(dsc_filename, session)
+ self.check_dsc_against_db(dsc_filename, session)
+ session.close()
+
+ return True
+
+ ###########################################################################
+
+ def ensure_all_source_exists(self, dest_dir=None):
+ """
+ Ensure that dest_dir contains all the orig tarballs for the specified
+ changes. If it does not, symlink them into place.
+
+ If dest_dir is None, populate the current directory.
"""
- changes = self.pkg.changes
- dsc = self.pkg.dsc
- files = self.pkg.files
- dsc_files = self.pkg.dsc_files
- orig_tar_id = self.pkg.orig_tar_id
- orig_tar_location = self.pkg.orig_tar_location
+ if dest_dir is None:
+ dest_dir = os.getcwd()
+
+ # Create a symlink mirror of the source files in our temporary directory
+ for f in self.pkg.files.keys():
+ m = re_issource.match(f)
+ if m:
+ src = os.path.join(source_dir, f)
+ # If a file is missing for whatever reason, give up.
+ if not os.path.exists(src):
+ return
+ ftype = m.group(3)
+ if re_is_orig_source.match(f) and pkg.orig_files.has_key(f) and \
+ pkg.orig_files[f].has_key("path"):
+ continue
+ dest = os.path.join(os.getcwd(), f)
+ os.symlink(src, dest)
+
+ # If the orig files are not a part of the upload, create symlinks to the
+ # existing copies.
+ for orig_file in self.pkg.orig_files.keys():
+ if not self.pkg.orig_files[orig_file].has_key("path"):
+ continue
+ dest = os.path.join(os.getcwd(), os.path.basename(orig_file))
+ os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
+
+ ###########################################################################
+
+ def get_changelog_versions(self, source_dir):
+ """Extracts a the source package and (optionally) grabs the
+ version history out of debian/changelog for the BTS."""
+
+ cnf = Config()
+
+ # Find the .dsc (again)
+ dsc_filename = None
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f]["type"] == "dsc":
+ dsc_filename = f
+
+ # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
+ if not dsc_filename:
+ return
+
+ self.ensure_all_source_exists()
+
+ # Extract the source
+ cmd = "dpkg-source -sn -x %s" % (dsc_filename)
+ (result, output) = commands.getstatusoutput(cmd)
+ if (result != 0):
+ self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
+ self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "))
+ return
+
+ if not cnf.Find("Dir::Queue::BTSVersionTrack"):
+ return
+
+ # Get the upstream version
+ upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
+ if re_strip_revision.search(upstr_version):
+ upstr_version = re_strip_revision.sub('', upstr_version)
+
+ # Ensure the changelog file exists
+ changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
+ if not os.path.exists(changelog_filename):
+ self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
+ return
+
+ # Parse the changelog
+ self.pkg.dsc["bts changelog"] = ""
+ changelog_file = utils.open_file(changelog_filename)
+ for line in changelog_file.readlines():
+ m = re_changelog_versions.match(line)
+ if m:
+ self.pkg.dsc["bts changelog"] += line
+ changelog_file.close()
+
+ # Check we found at least one revision in the changelog
+ if not self.pkg.dsc["bts changelog"]:
+ self.rejects.append("%s: changelog format not recognised (empty version tree)." % (dsc_filename))
+
+ def check_source(self):
+ # XXX: I'm fairly sure reprocess == 2 can never happen
+ # AJT disabled the is_incoming check years ago - mhy
+ # We should probably scrap or rethink the whole reprocess thing
+ # Bail out if:
+ # a) there's no source
+ # or b) reprocess is 2 - we will do this check next time when orig
+ # tarball is in 'files'
+ # or c) the orig files are MIA
+ if not self.pkg.changes["architecture"].has_key("source") or self.reprocess == 2 \
+ or len(self.pkg.orig_files) == 0:
+ return
+
+ tmpdir = utils.temp_dirname()
+
+ # Move into the temporary directory
+ cwd = os.getcwd()
+ os.chdir(tmpdir)
+
+ # Get the changelog version history
+ self.get_changelog_versions(cwd)
+
+ # Move back and cleanup the temporary tree
+ os.chdir(cwd)
- dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".dak")
- dump_file = utils.open_file(dump_filename, 'w')
try:
- os.chmod(dump_filename, 0664)
+ shutil.rmtree(tmpdir)
except OSError, e:
- # chmod may fail when the dumpfile is not owned by the user
- # invoking dak (like e.g. when NEW is processed by a member
- # of ftpteam)
- if errno.errorcode[e.errno] == 'EPERM':
- perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE])
- # security precaution, should never happen unless a weird
- # umask is set anywhere
- if perms & stat.S_IWOTH:
- utils.fubar("%s is world writable and chmod failed." % \
- (dump_filename,))
- # ignore the failed chmod otherwise as the file should
- # already have the right privileges and is just, at worst,
- # unreadable for world
+ if e.errno != errno.EACCES:
+ print "foobar"
+ utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
+
+ self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
+ # We probably have u-r or u-w directories so chmod everything
+ # and try again.
+ cmd = "chmod -R u+rwx %s" % (tmpdir)
+ result = os.system(cmd)
+ if result != 0:
+ utils.fubar("'%s' failed with result %s." % (cmd, result))
+ shutil.rmtree(tmpdir)
+ except Exception, e:
+ print "foobar2 (%s)" % e
+ utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
+
+ ###########################################################################
+ def ensure_hashes(self):
+ # Make sure we recognise the format of the Files: field in the .changes
+ format = self.pkg.changes.get("format", "0.0").split(".", 1)
+ if len(format) == 2:
+ format = int(format[0]), int(format[1])
+ else:
+ format = int(float(format[0])), 0
+
+ # We need to deal with the original changes blob, as the fields we need
+ # might not be in the changes dict serialised into the .dak anymore.
+ orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
+
+ # Copy the checksums over to the current changes dict. This will keep
+ # the existing modifications to it intact.
+ for field in orig_changes:
+ if field.startswith('checksums-'):
+ self.pkg.changes[field] = orig_changes[field]
+
+ # Check for unsupported hashes
+ for j in utils.check_hash_fields(".changes", self.pkg.changes):
+ self.rejects.append(j)
+
+ for j in utils.check_hash_fields(".dsc", self.pkg.dsc):
+ self.rejects.append(j)
+
+ # We have to calculate the hash if we have an earlier changes version than
+ # the hash appears in rather than require it exist in the changes file
+ for hashname, hashfunc, version in utils.known_hashes:
+ # TODO: Move _ensure_changes_hash into this class
+ for j in utils._ensure_changes_hash(self.pkg.changes, format, version, self.pkg.files, hashname, hashfunc):
+ self.rejects.append(j)
+ if "source" in self.pkg.changes["architecture"]:
+ # TODO: Move _ensure_dsc_hash into this class
+ for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
+ self.rejects.append(j)
+
+ def check_hashes(self):
+ for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
+ self.rejects.append(m)
+
+ for m in utils.check_size(".changes", self.pkg.files):
+ self.rejects.append(m)
+
+ for m in utils.check_hash(".dsc", self.pkg.dsc_files, "md5", apt_pkg.md5sum):
+ self.rejects.append(m)
+
+ for m in utils.check_size(".dsc", self.pkg.dsc_files):
+ self.rejects.append(m)
+
+ self.ensure_hashes()
+
+ ###########################################################################
+ def check_lintian(self):
+ # Only check some distributions
+ valid_dist = False
+ for dist in ('unstable', 'experimental'):
+ if dist in self.pkg.changes['distribution']:
+ valid_dist = True
+ break
+
+ if not valid_dist:
+ return
+
+ self.ensure_all_source_exists()
+
+ cnf = Config()
+ tagfile = cnf.get("Dinstall::LintianTags")
+ if tagfile is None:
+ # We don't have a tagfile, so just don't do anything.
+ return
+ # Parse the yaml file
+ sourcefile = file(tagfile, 'r')
+ sourcecontent = sourcefile.read()
+ sourcefile.close()
+ try:
+ lintiantags = yaml.load(sourcecontent)['lintian']
+ except yaml.YAMLError, msg:
+ utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
+ return
+
+ # Now setup the input file for lintian. lintian wants "one tag per line" only,
+ # so put it together like it. We put all types of tags in one file and then sort
+ # through lintians output later to see if its a fatal tag we detected, or not.
+ # So we only run lintian once on all tags, even if we might reject on some, but not
+ # reject on others.
+ # Additionally build up a set of tags
+ tags = set()
+ (fd, temp_filename) = utils.temp_filename()
+ temptagfile = os.fdopen(fd, 'w')
+ for tagtype in lintiantags:
+ for tag in lintiantags[tagtype]:
+ temptagfile.write("%s\n" % tag)
+ tags.add(tag)
+ temptagfile.close()
+
+ # So now we should look at running lintian at the .changes file, capturing output
+ # to then parse it.
+ command = "lintian --show-overrides --tags-from-file %s %s" % (temp_filename, self.pkg.changes_file)
+ (result, output) = commands.getstatusoutput(command)
+ # We are done with lintian, remove our tempfile
+ os.unlink(temp_filename)
+ if (result == 2):
+ utils.warn("lintian failed for %s [return code: %s]." % (self.pkg.changes_file, result))
+ utils.warn(utils.prefix_multi_line_string(output, " [possible output:] "))
+
+ if len(output) == 0:
+ return
+
+ # We have output of lintian, this package isn't clean. Lets parse it and see if we
+ # are having a victim for a reject.
+ # W: tzdata: binary-without-manpage usr/sbin/tzconfig
+ for line in output.split('\n'):
+ m = re_parse_lintian.match(line)
+ if m is None:
+ continue
+
+ etype = m.group(1)
+ epackage = m.group(2)
+ etag = m.group(3)
+ etext = m.group(4)
+
+ # So lets check if we know the tag at all.
+ if etag not in tags:
+ continue
+
+ if etype == 'O':
+ # We know it and it is overriden. Check that override is allowed.
+ if etag in lintiantags['warning']:
+ # The tag is overriden, and it is allowed to be overriden.
+ # Don't add a reject message.
+ pass
+ elif etag in lintiantags['error']:
+ # The tag is overriden - but is not allowed to be
+ self.rejects.append("%s: Overriden tag %s found, but this tag may not be overwritten." % (epackage, etag))
else:
- raise
-
- p = cPickle.Pickler(dump_file, 1)
- d_changes = {}
- d_dsc = {}
- d_files = {}
- d_dsc_files = {}
-
- ## files
- for file_entry in files.keys():
- d_files[file_entry] = {}
- for i in [ "package", "version", "architecture", "type", "size",
- "md5sum", "sha1sum", "sha256sum", "component",
- "location id", "source package", "source version",
- "maintainer", "dbtype", "files id", "new",
- "section", "priority", "othercomponents",
- "pool name", "original component" ]:
- if files[file_entry].has_key(i):
- d_files[file_entry][i] = files[file_entry][i]
- ## changes
- # Mandatory changes fields
- for i in [ "distribution", "source", "architecture", "version",
- "maintainer", "urgency", "fingerprint", "changedby822",
- "changedby2047", "changedbyname", "maintainer822",
- "maintainer2047", "maintainername", "maintaineremail",
- "closes", "changes" ]:
- d_changes[i] = changes[i]
- # Optional changes fields
- for i in [ "changed-by", "filecontents", "format", "process-new note", "adv id", "distribution-version",
- "sponsoremail" ]:
- if changes.has_key(i):
- d_changes[i] = changes[i]
- ## dsc
- for i in [ "source", "version", "maintainer", "fingerprint",
- "uploaders", "bts changelog", "dm-upload-allowed" ]:
- if dsc.has_key(i):
- d_dsc[i] = dsc[i]
- ## dsc_files
- for file_entry in dsc_files.keys():
- d_dsc_files[file_entry] = {}
- # Mandatory dsc_files fields
- for i in [ "size", "md5sum" ]:
- d_dsc_files[file_entry][i] = dsc_files[file_entry][i]
- # Optional dsc_files fields
- for i in [ "files id" ]:
- if dsc_files[file_entry].has_key(i):
- d_dsc_files[file_entry][i] = dsc_files[file_entry][i]
-
- for i in [ d_changes, d_dsc, d_files, d_dsc_files,
- orig_tar_id, orig_tar_location ]:
- p.dump(i)
- dump_file.close()
+ # Tag is known, it is not overriden, direct reject.
+ self.rejects.append("%s: Found lintian output: '%s %s', automatically rejected package." % (epackage, etag, etext))
+ # Now tell if they *might* override it.
+ if etag in lintiantags['warning']:
+ self.rejects.append("%s: If you have a good reason, you may override this lintian tag." % (epackage))
###########################################################################
+ def check_urgency(self):
+ cnf = Config()
+ if self.pkg.changes["architecture"].has_key("source"):
+ if not self.pkg.changes.has_key("urgency"):
+ self.pkg.changes["urgency"] = cnf["Urgency::Default"]
+ self.pkg.changes["urgency"] = self.pkg.changes["urgency"].lower()
+ if self.pkg.changes["urgency"] not in cnf.ValueList("Urgency::Valid"):
+ self.warnings.append("%s is not a valid urgency; it will be treated as %s by testing." % \
+ (self.pkg.changes["urgency"], cnf["Urgency::Default"]))
+ self.pkg.changes["urgency"] = cnf["Urgency::Default"]
- # Set up the per-package template substitution mappings
+ ###########################################################################
- def update_subst (self, reject_message = ""):
- """ Set up the per-package template substitution mappings """
+ # Sanity check the time stamps of files inside debs.
+ # [Files in the near future cause ugly warnings and extreme time
+ # travel can cause errors on extraction]
- Subst = self.Subst
- changes = self.pkg.changes
- # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
- if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
- changes["architecture"] = { "Unknown" : "" }
- # and maintainer2047 may not exist.
- if not changes.has_key("maintainer2047"):
- changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"]
+ def check_timestamps(self):
+ Cnf = Config()
- Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys())
- Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file)
- Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "")
+ future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
+ past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
+ tar = TarTime(future_cutoff, past_cutoff)
- # For source uploads the Changed-By field wins; otherwise Maintainer wins.
- if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
- Subst["__MAINTAINER_FROM__"] = changes["changedby2047"]
- Subst["__MAINTAINER_TO__"] = "%s, %s" % (changes["changedby2047"],
- changes["maintainer2047"])
- Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown")
- else:
- Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"]
- Subst["__MAINTAINER_TO__"] = changes["maintainer2047"]
- Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown")
+ for filename, entry in self.pkg.files.items():
+ if entry["type"] == "deb":
+ tar.reset()
+ try:
+ deb_file = utils.open_file(filename)
+ apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
+ deb_file.seek(0)
+ try:
+ apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
+ except SystemError, e:
+ # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
+ if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
+ raise
+ deb_file.seek(0)
+ apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
+
+ deb_file.close()
+
+ future_files = tar.future_files.keys()
+ if future_files:
+ num_future_files = len(future_files)
+ future_file = future_files[0]
+ future_date = tar.future_files[future_file]
+ self.rejects.append("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])."
+ % (filename, num_future_files, future_file, time.ctime(future_date)))
+
+ ancient_files = tar.ancient_files.keys()
+ if ancient_files:
+ num_ancient_files = len(ancient_files)
+ ancient_file = ancient_files[0]
+ ancient_date = tar.ancient_files[ancient_file]
+ self.rejects.append("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])."
+ % (filename, num_ancient_files, ancient_file, time.ctime(ancient_date)))
+ except:
+ self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
- if "sponsoremail" in changes:
- Subst["__MAINTAINER_TO__"] += ", %s"%changes["sponsoremail"]
+ ###########################################################################
+ def check_transition(self, session):
+ cnf = Config()
- if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
- Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
+ sourcepkg = self.pkg.changes["source"]
- # Apply any global override of the Maintainer field
- if self.Cnf.get("Dinstall::OverrideMaintainer"):
- Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"]
- Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"]
+ # No sourceful upload -> no need to do anything else, direct return
+ # We also work with unstable uploads, not experimental or those going to some
+ # proposed-updates queue
+ if "source" not in self.pkg.changes["architecture"] or \
+ "unstable" not in self.pkg.changes["distribution"]:
+ return
+
+ # Also only check if there is a file defined (and existant) with
+ # checks.
+ transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
+ if transpath == "" or not os.path.exists(transpath):
+ return
+
+ # Parse the yaml file
+ sourcefile = file(transpath, 'r')
+ sourcecontent = sourcefile.read()
+ try:
+ transitions = yaml.load(sourcecontent)
+ except yaml.YAMLError, msg:
+ # This shouldn't happen, there is a wrapper to edit the file which
+ # checks it, but we prefer to be safe than ending up rejecting
+ # everything.
+ utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
+ return
+
+ # Now look through all defined transitions
+ for trans in transitions:
+ t = transitions[trans]
+ source = t["source"]
+ expected = t["new"]
+
+ # Will be None if nothing is in testing.
+ current = get_source_in_suite(source, "testing", session)
+ if current is not None:
+ compare = apt_pkg.VersionCompare(current.version, expected)
+
+ if current is None or compare < 0:
+ # This is still valid, the current version in testing is older than
+ # the new version we wait for, or there is none in testing yet
+
+ # Check if the source we look at is affected by this.
+ if sourcepkg in t['packages']:
+ # The source is affected, lets reject it.
+
+ rejectmsg = "%s: part of the %s transition.\n\n" % (
+ sourcepkg, trans)
+
+ if current is not None:
+ currentlymsg = "at version %s" % (current.version)
+ else:
+ currentlymsg = "not present in testing"
- Subst["__REJECT_MESSAGE__"] = reject_message
- Subst["__SOURCE__"] = changes.get("source", "Unknown")
- Subst["__VERSION__"] = changes.get("version", "Unknown")
+ rejectmsg += "Transition description: %s\n\n" % (t["reason"])
+
+ rejectmsg += "\n".join(textwrap.wrap("""Your package
+is part of a testing transition designed to get %s migrated (it is
+currently %s, we need version %s). This transition is managed by the
+Release Team, and %s is the Release-Team member responsible for it.
+Please mail debian-release@lists.debian.org or contact %s directly if you
+need further assistance. You might want to upload to experimental until this
+transition is done."""
+ % (source, currentlymsg, expected,t["rm"], t["rm"])))
+
+ self.rejects.append(rejectmsg)
+ return
###########################################################################
+ def check_signed_by_key(self):
+ """Ensure the .changes is signed by an authorized uploader."""
+ session = DBConn().session()
+
+ self.check_transition(session)
+
+ (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(self.pkg.changes["fingerprint"], session=session)
+
+ # match claimed name with actual name:
+ if uid is None:
+ # This is fundamentally broken but need us to refactor how we get
+ # the UIDs/Fingerprints in order for us to fix it properly
+ uid, uid_email = self.pkg.changes["fingerprint"], uid
+ may_nmu, may_sponsor = 1, 1
+ # XXX by default new dds don't have a fingerprint/uid in the db atm,
+ # and can't get one in there if we don't allow nmu/sponsorship
+ elif is_dm is False:
+ # If is_dm is False, we allow full upload rights
+ uid_email = "%s@debian.org" % (uid)
+ may_nmu, may_sponsor = 1, 1
+ else:
+ # Assume limited upload rights unless we've discovered otherwise
+ uid_email = uid
+ may_nmu, may_sponsor = 0, 0
+
+ if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
+ sponsored = 0
+ elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
+ sponsored = 0
+ if uid_name == "": sponsored = 1
+ else:
+ sponsored = 1
+ if ("source" in self.pkg.changes["architecture"] and
+ uid_email and utils.is_email_alias(uid_email)):
+ sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
+ if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
+ self.pkg.changes["changedbyemail"] not in sponsor_addresses):
+ self.pkg.changes["sponsoremail"] = uid_email
+
+ if sponsored and not may_sponsor:
+ self.rejects.append("%s is not authorised to sponsor uploads" % (uid))
+
+ if not sponsored and not may_nmu:
+ should_reject = True
+ highest_sid, highest_version = None, None
+
+ # XXX: This reimplements in SQLA what existed before but it's fundamentally fucked
+ # It ignores higher versions with the dm_upload_allowed flag set to false
+ # I'm keeping the existing behaviour for now until I've gone back and
+ # checked exactly what the GR says - mhy
+ for si in get_sources_from_name(source=self.pkg.changes['source'], dm_upload_allowed=True, session=session):
+ if highest_version is None or apt_pkg.VersionCompare(si.version, highest_version) == 1:
+ highest_sid = si.source_id
+ highest_version = si.version
+
+ if highest_sid is None:
+ self.rejects.append("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % self.pkg.changes["source"])
+ else:
+ for sup in session.query(SrcUploader).join(DBSource).filter_by(source_id=highest_sid):
+ (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
+ if email == uid_email or name == uid_name:
+ should_reject = False
+ break
+
+ if should_reject is True:
+ self.rejects.append("%s is not in Maintainer or Uploaders of source package %s" % (uid, self.pkg.changes["source"]))
+ for b in self.pkg.changes["binary"].keys():
+ for suite in self.pkg.changes["distribution"].keys():
+ q = session.query(DBSource)
+ q = q.join(DBBinary).filter_by(package=b)
+ q = q.join(BinAssociation).join(Suite).filter_by(suite_name=suite)
+
+ for s in q.all():
+ if s.source != self.pkg.changes["source"]:
+ self.rejects.append("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
+
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f].has_key("byhand"):
+ self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
+ if self.pkg.files[f].has_key("new"):
+ self.rejects.append("%s may not upload NEW file %s" % (uid, f))
+
+ session.close()
+
+ ###########################################################################
def build_summaries(self):
""" Build a summary of changes the upload introduces. """
- changes = self.pkg.changes
- files = self.pkg.files
-
- byhand = summary = new = ""
-
- # changes["distribution"] may not exist in corner cases
- # (e.g. unreadable changes files)
- if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
- changes["distribution"] = {}
-
- override_summary =""
- file_keys = files.keys()
- file_keys.sort()
- for file_entry in file_keys:
- if files[file_entry].has_key("byhand"):
- byhand = 1
- summary += file_entry + " byhand\n"
- elif files[file_entry].has_key("new"):
- new = 1
- summary += "(new) %s %s %s\n" % (file_entry, files[file_entry]["priority"], files[file_entry]["section"])
- if files[file_entry].has_key("othercomponents"):
- summary += "WARNING: Already present in %s distribution.\n" % (files[file_entry]["othercomponents"])
- if files[file_entry]["type"] == "deb":
- deb_fh = utils.open_file(file_entry)
- summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n'
- deb_fh.close()
- else:
- files[file_entry]["pool name"] = utils.poolify (changes.get("source",""), files[file_entry]["component"])
- destination = self.Cnf["Dir::PoolRoot"] + files[file_entry]["pool name"] + file_entry
- summary += file_entry + "\n to " + destination + "\n"
- if not files[file_entry].has_key("type"):
- files[file_entry]["type"] = "unknown"
- if files[file_entry]["type"] in ["deb", "udeb", "dsc"]:
- # (queue/unchecked), there we have override entries already, use them
- # (process-new), there we dont have override entries, use the newly generated ones.
- override_prio = files[file_entry].get("override priority", files[file_entry]["priority"])
- override_sect = files[file_entry].get("override section", files[file_entry]["section"])
- override_summary += "%s - %s %s\n" % (file_entry, override_prio, override_sect)
+
+ (byhand, new, summary, override_summary) = self.pkg.file_summary()
short_summary = summary
# This is for direport's benefit...
- f = re_fdnic.sub("\n .\n", changes.get("changes",""))
+ f = re_fdnic.sub("\n .\n", self.pkg.changes.get("changes", ""))
if byhand or new:
summary += "Changes: " + f
###########################################################################
- def close_bugs (self, summary, action):
+ def close_bugs(self, summary, action):
"""
Send mail to close bugs as instructed by the closes field in the changes file.
Also add a line to summary if any work was done.
@return: summary. If action was taken, extended by the list of closed bugs.
"""
- changes = self.pkg.changes
- Subst = self.Subst
- Cnf = self.Cnf
- bugs = changes["closes"].keys()
+ template = os.path.join(Config()["Dir::Templates"], 'process-unchecked.bug-close')
+
+ bugs = self.pkg.changes["closes"].keys()
if not bugs:
return summary
for bug in bugs:
summary += "%s " % (bug)
if action:
- Subst["__BUG_NUMBER__"] = bug
- if changes["distribution"].has_key("stable"):
- Subst["__STABLE_WARNING__"] = """
+ self.update_subst()
+ self.Subst["__BUG_NUMBER__"] = bug
+ if self.pkg.changes["distribution"].has_key("stable"):
+ self.Subst["__STABLE_WARNING__"] = """
Note that this package is not part of the released stable Debian
distribution. It may have dependencies on other unreleased software,
or other instabilities. Please take care if you wish to install it.
The update will eventually make its way into the next released Debian
distribution."""
else:
- Subst["__STABLE_WARNING__"] = ""
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.bug-close")
- utils.send_mail (mail_message)
- if action:
- self.Logger.log(["closing bugs"]+bugs)
+ self.Subst["__STABLE_WARNING__"] = ""
+ mail_message = utils.TemplateSubst(self.Subst, template)
+ utils.send_mail(mail_message)
+
+ # Clear up after ourselves
+ del self.Subst["__BUG_NUMBER__"]
+ del self.Subst["__STABLE_WARNING__"]
+
+ if action and self.logger:
+ self.logger.log(["closing bugs"] + bugs)
+
summary += "\n"
return summary
###########################################################################
- def announce (self, short_summary, action):
+ def announce(self, short_summary, action):
"""
Send an announce mail about a new upload.
@return: Textstring about action taken.
"""
- Subst = self.Subst
- Cnf = self.Cnf
- changes = self.pkg.changes
+
+ cnf = Config()
+ announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
# Only do announcements for source uploads with a recent dpkg-dev installed
- if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
+ if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
+ self.pkg.changes["architecture"].has_key("source"):
return ""
lists_done = {}
summary = ""
- Subst["__SHORT_SUMMARY__"] = short_summary
- for dist in changes["distribution"].keys():
- announce_list = Cnf.Find("Suite::%s::Announce" % (dist))
+ self.Subst["__SHORT_SUMMARY__"] = short_summary
+
+ for dist in self.pkg.changes["distribution"].keys():
+ announce_list = cnf.Find("Suite::%s::Announce" % (dist))
if announce_list == "" or lists_done.has_key(announce_list):
continue
+
lists_done[announce_list] = 1
summary += "Announcing to %s\n" % (announce_list)
if action:
- Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
- if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
- Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"])
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.announce")
- utils.send_mail (mail_message)
+ self.update_subst()
+ self.Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list
+ if cnf.get("Dinstall::TrackingServer") and \
+ self.pkg.changes["architecture"].has_key("source"):
+ trackingsendto = "Bcc: %s@%s" % (self.pkg.changes["source"], cnf["Dinstall::TrackingServer"])
+ self.Subst["__ANNOUNCE_LIST_ADDRESS__"] += "\n" + trackingsendto
+
+ mail_message = utils.TemplateSubst(self.Subst, announcetemplate)
+ utils.send_mail(mail_message)
- if Cnf.FindB("Dinstall::CloseBugs"):
+ del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
+
+ if cnf.FindB("Dinstall::CloseBugs"):
summary = self.close_bugs(summary, action)
+ del self.Subst["__SHORT_SUMMARY__"]
+
return summary
###########################################################################
"""
- Cnf = self.Cnf
- Subst = self.Subst
- files = self.pkg.files
- changes = self.pkg.changes
- changes_file = self.pkg.changes_file
- dsc = self.pkg.dsc
+ cnf = Config()
+ stats = SummaryStats()
+
+ accepttemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.accepted')
if targetdir is None:
- targetdir = Cnf["Dir::Queue::Accepted"]
+ targetdir = cnf["Dir::Queue::Accepted"]
print "Accepting."
- self.Logger.log(["Accepting changes",changes_file])
+ if self.logger:
+ self.logger.log(["Accepting changes", self.pkg.changes_file])
- self.dump_vars(targetdir)
+ self.pkg.write_dot_dak(targetdir)
# Move all the files into the accepted directory
- utils.move(changes_file, targetdir)
- file_keys = files.keys()
- for file_entry in file_keys:
- utils.move(file_entry, targetdir)
- self.accept_bytes += float(files[file_entry]["size"])
- self.accept_count += 1
+ utils.move(self.pkg.changes_file, targetdir)
+
+ for name, entry in sorted(self.pkg.files.items()):
+ utils.move(name, targetdir)
+ stats.accept_bytes += float(entry["size"])
+
+ stats.accept_count += 1
# Send accept mail, announce to lists, close bugs and check for
# override disparities
- if not Cnf["Dinstall::Options::No-Mail"]:
- Subst["__SUITE__"] = ""
- Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.accepted")
+ if not cnf["Dinstall::Options::No-Mail"]:
+ self.update_subst()
+ self.Subst["__SUITE__"] = ""
+ self.Subst["__SUMMARY__"] = summary
+ mail_message = utils.TemplateSubst(self.Subst, accepttemplate)
utils.send_mail(mail_message)
self.announce(short_summary, 1)
-
## Helper stuff for DebBugs Version Tracking
- if Cnf.Find("Dir::Queue::BTSVersionTrack"):
+ if cnf.Find("Dir::Queue::BTSVersionTrack"):
# ??? once queue/* is cleared on *.d.o and/or reprocessed
# the conditionalization on dsc["bts changelog"] should be
# dropped.
# Write out the version history from the changelog
- if changes["architecture"].has_key("source") and \
- dsc.has_key("bts changelog"):
+ if self.pkg.changes["architecture"].has_key("source") and \
+ self.pkg.dsc.has_key("bts changelog"):
- (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
version_history = os.fdopen(fd, 'w')
- version_history.write(dsc["bts changelog"])
+ version_history.write(self.pkg.dsc["bts changelog"])
version_history.close()
- filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
- changes_file[:-8]+".versions")
+ filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+ self.pkg.changes_file[:-8]+".versions")
os.rename(temp_filename, filename)
os.chmod(filename, 0644)
# Write out the binary -> source mapping.
- (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
debinfo = os.fdopen(fd, 'w')
- for file_entry in file_keys:
- f = files[file_entry]
- if f["type"] == "deb":
- line = " ".join([f["package"], f["version"],
- f["architecture"], f["source package"],
- f["source version"]])
+ for name, entry in sorted(self.pkg.files.items()):
+ if entry["type"] == "deb":
+ line = " ".join([entry["package"], entry["version"],
+ entry["architecture"], entry["source package"],
+ entry["source version"]])
debinfo.write(line+"\n")
debinfo.close()
- filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
- changes_file[:-8]+".debinfo")
+ filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+ self.pkg.changes_file[:-8]+".debinfo")
os.rename(temp_filename, filename)
os.chmod(filename, 0644)
# <Ganneff> so it will work out, as unchecked move it over
# <mhy> that's all completely sick
# <Ganneff> yes
- self.queue_build("accepted", Cnf["Dir::Queue::Accepted"])
- ###########################################################################
-
- def queue_build (self, queue, path):
- """
- Prepare queue_build database table used for incoming autobuild support.
+ # This routine returns None on success or an error on failure
+ res = get_or_set_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
+ if res:
+ utils.fubar(res)
- @type queue: string
- @param queue: queue name
- @type path: string
- @param path: path for the queue file entries/link destinations
- """
-
- Cnf = self.Cnf
- Subst = self.Subst
- files = self.pkg.files
- changes = self.pkg.changes
- changes_file = self.pkg.changes_file
- dsc = self.pkg.dsc
- file_keys = files.keys()
-
- ## Special support to enable clean auto-building of queued packages
- queue_id = database.get_or_set_queue_id(queue)
-
- self.projectB.query("BEGIN WORK")
- for suite in changes["distribution"].keys():
- if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
- continue
- suite_id = database.get_suite_id(suite)
- dest_dir = Cnf["Dir::QueueBuild"]
- if Cnf.FindB("Dinstall::SecurityQueueBuild"):
- dest_dir = os.path.join(dest_dir, suite)
- for file_entry in file_keys:
- src = os.path.join(path, file_entry)
- dest = os.path.join(dest_dir, file_entry)
- if Cnf.FindB("Dinstall::SecurityQueueBuild"):
- # Copy it since the original won't be readable by www-data
- utils.copy(src, dest)
- else:
- # Create a symlink to it
- os.symlink(src, dest)
- # Add it to the list of packages for later processing by apt-ftparchive
- self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest))
- # If the .orig.tar.gz is in the pool, create a symlink to
- # it (if one doesn't already exist)
- if self.pkg.orig_tar_id:
- # Determine the .orig.tar.gz file name
- for dsc_file in self.pkg.dsc_files.keys():
- if dsc_file.endswith(".orig.tar.gz"):
- filename = dsc_file
- dest = os.path.join(dest_dir, filename)
- # If it doesn't exist, create a symlink
- if not os.path.exists(dest):
- # Find the .orig.tar.gz in the pool
- q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id))
- ql = q.getresult()
- if not ql:
- utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id))
- src = os.path.join(ql[0][0], ql[0][1])
- os.symlink(src, dest)
- # Add it to the list of packages for later processing by apt-ftparchive
- self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest))
- # if it does, update things to ensure it's not removed prematurely
- else:
- self.projectB.query("UPDATE queue_build SET in_queue = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id))
-
- self.projectB.query("COMMIT WORK")
-
- ###########################################################################
-
- def check_override (self):
+ def check_override(self):
"""
Checks override entries for validity. Mails "Override disparity" warnings,
if that feature is enabled.
Abandons the check if
- - this is a non-sourceful upload
- override disparity checks are disabled
- mail sending is disabled
-
"""
- Subst = self.Subst
- changes = self.pkg.changes
- files = self.pkg.files
- Cnf = self.Cnf
+
+ cnf = Config()
# Abandon the check if:
- # a) it's a non-sourceful upload
- # b) override disparity checks have been disabled
- # c) we're not sending mail
- if not changes["architecture"].has_key("source") or \
- not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
- Cnf["Dinstall::Options::No-Mail"]:
+ # a) override disparity checks have been disabled
+ # b) we're not sending mail
+ if not cnf.FindB("Dinstall::OverrideDisparityCheck") or \
+ cnf["Dinstall::Options::No-Mail"]:
return
- summary = ""
- file_keys = files.keys()
- file_keys.sort()
- for file_entry in file_keys:
- if not files[file_entry].has_key("new") and files[file_entry]["type"] == "deb":
- section = files[file_entry]["section"]
- override_section = files[file_entry]["override section"]
- if section.lower() != override_section.lower() and section != "-":
- summary += "%s: package says section is %s, override says %s.\n" % (file_entry, section, override_section)
- priority = files[file_entry]["priority"]
- override_priority = files[file_entry]["override priority"]
- if priority != override_priority and priority != "-":
- summary += "%s: package says priority is %s, override says %s.\n" % (file_entry, priority, override_priority)
+ summary = self.pkg.check_override()
if summary == "":
return
- Subst["__SUMMARY__"] = summary
- mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/process-unchecked.override-disparity")
+ overridetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.override-disparity')
+
+ self.update_subst()
+ self.Subst["__SUMMARY__"] = summary
+ mail_message = utils.TemplateSubst(self.Subst, overridetemplate)
utils.send_mail(mail_message)
+ del self.Subst["__SUMMARY__"]
+
+ ###########################################################################
+
+ def remove(self, dir=None):
+ """
+ Used (for instance) in p-u to remove the package from unchecked
+ """
+ if dir is None:
+ os.chdir(self.pkg.directory)
+ else:
+ os.chdir(dir)
+
+ for f in self.pkg.files.keys():
+ os.unlink(f)
+ os.unlink(self.pkg.changes_file)
+
+ ###########################################################################
+
+ def move_to_dir (self, dest, perms=0660, changesperms=0664):
+ """
+ Move files to dest with certain perms/changesperms
+ """
+ utils.move(self.pkg.changes_file, dest, perms=changesperms)
+ for f in self.pkg.files.keys():
+ utils.move(f, dest, perms=perms)
###########################################################################
- def force_reject (self, files):
+ def force_reject(self, reject_files):
"""
Forcefully move files from the current directory to the
reject directory. If any file already exists in the reject
"""
- Cnf = self.Cnf
+ cnf = Config()
- for file_entry in files:
+ for file_entry in reject_files:
# Skip any files which don't exist or which we don't have permission to copy.
- if os.access(file_entry,os.R_OK) == 0:
+ if os.access(file_entry, os.R_OK) == 0:
continue
- dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file_entry)
+
+ dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
+
try:
- dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
except OSError, e:
# File exists? Let's try and move it to the morgue
- if errno.errorcode[e.errno] == 'EEXIST':
- morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file_entry)
+ if e.errno == errno.EEXIST:
+ morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
try:
morgue_file = utils.find_next_free(morgue_file)
except NoFreeFilenameError:
os.close(dest_fd)
###########################################################################
-
- def do_reject (self, manual = 0, reject_message = "", note = ""):
+ def do_reject (self, manual=0, reject_message="", note=""):
"""
Reject an upload. If called without a reject message or C{manual} is
true, spawn an editor so the user can write one.
print "Rejecting.\n"
- Cnf = self.Cnf
- Subst = self.Subst
- pkg = self.pkg
+ cnf = Config()
- reason_filename = pkg.changes_file[:-8] + ".reason"
- reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename
+ reason_filename = self.pkg.changes_file[:-8] + ".reason"
+ reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
# Move all the files into the reject directory
- reject_files = pkg.files.keys() + [pkg.changes_file]
+ reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
self.force_reject(reject_files)
# If we fail here someone is probably trying to exploit the race
os.unlink(reason_filename)
reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ rej_template = os.path.join(cnf["Dir::Templates"], "queue.rejected")
+
+ self.update_subst()
if not manual:
- Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
- Subst["__MANUAL_REJECT_MESSAGE__"] = ""
- Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)\nX-Katie-Rejection: automatic (moo)"
+ self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__MANUAL_REJECT_MESSAGE__"] = ""
+ self.Subst["__CC__"] = "X-DAK-Rejection: automatic (moo)"
os.write(reason_fd, reject_message)
- reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/queue.rejected")
+ reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
else:
# Build up the rejection email
- user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"])
-
- Subst["__REJECTOR_ADDRESS__"] = user_email_address
- Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
- Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
- reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/queue.rejected")
+ user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
+ self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
+ self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
+ self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
+ reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
# Write the rejection email out as the <foo>.reason file
os.write(reason_fd, reject_mail_message)
+ del self.Subst["__REJECTOR_ADDRESS__"]
+ del self.Subst["__MANUAL_REJECT_MESSAGE__"]
+ del self.Subst["__CC__"]
+
os.close(reason_fd)
# Send the rejection mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
+ if not cnf["Dinstall::Options::No-Mail"]:
utils.send_mail(reject_mail_message)
- self.Logger.log(["rejected", pkg.changes_file])
- return 0
-
- ################################################################################
-
- def source_exists (self, package, source_version, suites = ["any"]):
- """
- Ensure that source exists somewhere in the archive for the binary
- upload being processed.
- 1. exact match => 1.0-3
- 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
-
- @type package: string
- @param package: package source name
-
- @type source_version: string
- @param source_version: expected source version
+ if self.logger:
+ self.logger.log(["rejected", self.pkg.changes_file])
- @type suites: list
- @param suites: list of suites to check in, default I{any}
-
- @rtype: int
- @return: returns 1 if a source with expected version is found, otherwise 0
-
- """
- okay = 1
- for suite in suites:
- if suite == "any":
- que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
- (package)
- else:
- # source must exist in suite X, or in some other suite that's
- # mapped to X, recursively... silent-maps are counted too,
- # unreleased-maps aren't.
- maps = self.Cnf.ValueList("SuiteMappings")[:]
- maps.reverse()
- maps = [ m.split() for m in maps ]
- maps = [ (x[1], x[2]) for x in maps
- if x[0] == "map" or x[0] == "silent-map" ]
- s = [suite]
- for x in maps:
- if x[1] in s and x[0] not in s:
- s.append(x[0])
-
- que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, " OR ".join(["su.suite_name = '%s'" % a for a in s]))
- q = self.projectB.query(que)
-
- # Reduce the query results to a list of version numbers
- ql = [ i[0] for i in q.getresult() ]
-
- # Try (1)
- if source_version in ql:
- continue
-
- # Try (2)
- orig_source_version = re_bin_only_nmu.sub('', source_version)
- if orig_source_version in ql:
- continue
-
- # No source found...
- okay = 0
- break
- return okay
+ return 0
################################################################################
-
- def in_override_p (self, package, component, suite, binary_type, file):
+ def in_override_p(self, package, component, suite, binary_type, file, session):
"""
Check if a package already has override entries in the DB
@param package: package name
@type component: string
- @param component: database id of the component, as returned by L{database.get_component_id}
+ @param component: database id of the component
@type suite: int
- @param suite: database id of the suite, as returned by L{database.get_suite_id}
+ @param suite: database id of the suite
@type binary_type: string
@param binary_type: type of the package
@return: the database result. But noone cares anyway.
"""
- files = self.pkg.files
+
+ cnf = Config()
if binary_type == "": # must be source
file_type = "dsc"
file_type = binary_type
# Override suite name; used for example with proposed-updates
- if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
- suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)]
-
- # Avoid <undef> on unknown distributions
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
- return None
- component_id = database.get_component_id(component)
- type_id = database.get_override_type_id(file_type)
-
- q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
- % (package, suite_id, component_id, type_id))
- result = q.getresult()
+ if cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
+ suite = cnf["Suite::%s::OverrideSuite" % (suite)]
+
+ result = get_override(package, suite, component, file_type, session)
+
# If checking for a source package fall back on the binary override type
- if file_type == "dsc" and not result:
- deb_type_id = database.get_override_type_id("deb")
- udeb_type_id = database.get_override_type_id("udeb")
- q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
- % (package, suite_id, component_id, deb_type_id, udeb_type_id))
- result = q.getresult()
+ if file_type == "dsc" and len(result) < 1:
+ result = get_override(package, suite, component, ['deb', 'udeb'], session)
# Remember the section and priority so we can check them later if appropriate
- if result:
- files[file]["override section"] = result[0][0]
- files[file]["override priority"] = result[0][1]
+ if len(result) > 0:
+ result = result[0]
+ self.pkg.files[file]["override section"] = result.section.section
+ self.pkg.files[file]["override priority"] = result.priority.priority
+ return result
- return result
+ return None
################################################################################
-
- def reject (self, str, prefix="Rejected: "):
+ def get_anyversion(self, sv_list, suite):
"""
- Add C{str} to reject_message. Adds C{prefix}, by default "Rejected: "
-
- @type str: string
- @param str: Reject text
+ @type sv_list: list
+ @param sv_list: list of (suite, version) tuples to check
- @type prefix: string
- @param prefix: Prefix text, default Rejected:
+ @type suite: string
+ @param suite: suite name
+ Description: TODO
"""
- if str:
- # Unlike other rejects we add new lines first to avoid trailing
- # new lines when this message is passed back up to a caller.
- if self.reject_message:
- self.reject_message += "\n"
- self.reject_message += prefix + str
-
- ################################################################################
-
- def get_anyversion(self, query_result, suite):
- """ """
- anyversion=None
- anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
- for (v, s) in query_result:
+ Cnf = Config()
+ anyversion = None
+ anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
+ for (s, v) in sv_list:
if s in [ x.lower() for x in anysuite ]:
if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
- anyversion=v
+ anyversion = v
+
return anyversion
################################################################################
- def cross_suite_version_check(self, query_result, file, new_version,
- sourceful=False):
+ def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
"""
+ @type sv_list: list
+ @param sv_list: list of (suite, version) tuples to check
+
+ @type file: string
+ @param file: XXX
+
+ @type new_version: string
+ @param new_version: XXX
+
Ensure versions are newer than existing packages in target
suites and that cross-suite version checking rules as
set out in the conf file are satisfied.
-
"""
+ cnf = Config()
+
# Check versions for each target suite
for target_suite in self.pkg.changes["distribution"].keys():
- must_be_newer_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
- must_be_older_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+ must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
+ must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+
# Enforce "must be newer than target suite" even if conffile omits it
if target_suite not in must_be_newer_than:
must_be_newer_than.append(target_suite)
- for entry in query_result:
- existent_version = entry[0]
- suite = entry[1]
- if suite in must_be_newer_than and sourceful and \
- apt_pkg.VersionCompare(new_version, existent_version) < 1:
- self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
- if suite in must_be_older_than and \
- apt_pkg.VersionCompare(new_version, existent_version) > -1:
- ch = self.pkg.changes
+
+ for (suite, existent_version) in sv_list:
+ vercmp = apt_pkg.VersionCompare(new_version, existent_version)
+
+ if suite in must_be_newer_than and sourceful and vercmp < 1:
+ self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+
+ if suite in must_be_older_than and vercmp > -1:
cansave = 0
- if ch.get('distribution-version', {}).has_key(suite):
- # we really use the other suite, ignoring the conflicting one ...
- addsuite = ch["distribution-version"][suite]
- add_version = self.get_anyversion(query_result, addsuite)
- target_version = self.get_anyversion(query_result, target_suite)
+ if self.pkg.changes.get('distribution-version', {}).has_key(suite):
+ # we really use the other suite, ignoring the conflicting one ...
+ addsuite = self.pkg.changes["distribution-version"][suite]
+
+ add_version = self.get_anyversion(sv_list, addsuite)
+ target_version = self.get_anyversion(sv_list, target_suite)
if not add_version:
# not add_version can only happen if we map to a suite
# than complaining. either way, this isn't a REJECT issue
#
# And - we really should complain to the dorks who configured dak
- self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ")
+ self.warnings.append("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite))
self.pkg.changes.setdefault("propdistribution", {})
self.pkg.changes["propdistribution"][addsuite] = 1
cansave = 1
# not targets_version is true when the package is NEW
# we could just stick with the "...old version..." REJECT
# for this, I think.
- self.reject("Won't propogate NEW packages.")
+ self.rejects.append("Won't propogate NEW packages.")
elif apt_pkg.VersionCompare(new_version, add_version) < 0:
# propogation would be redundant. no need to reject though.
- self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ")
+ self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
cansave = 1
elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
apt_pkg.VersionCompare(add_version, target_version) >= 0:
# propogate!!
- self.reject("Propogating upload to %s" % (addsuite), "Warning: ")
+ self.warnings.append("Propogating upload to %s" % (addsuite))
self.pkg.changes.setdefault("propdistribution", {})
self.pkg.changes["propdistribution"][addsuite] = 1
cansave = 1
if not cansave:
- self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
################################################################################
-
- def check_binary_against_db(self, file):
- """
-
- """
- self.reject_message = ""
- files = self.pkg.files
-
+ def check_binary_against_db(self, file, session):
# Ensure version is sane
- q = self.projectB.query("""
-SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
- architecture a
- WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
- AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
- % (files[file]["package"],
- files[file]["architecture"]))
- self.cross_suite_version_check(q.getresult(), file,
- files[file]["version"], sourceful=False)
+ q = session.query(BinAssociation)
+ q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
+ q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
+
+ self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
+ file, self.pkg.files[file]["version"], sourceful=False)
# Check for any existing copies of the file
- q = self.projectB.query("""
-SELECT b.id FROM binaries b, architecture a
- WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
- AND a.id = b.architecture"""
- % (files[file]["package"],
- files[file]["version"],
- files[file]["architecture"]))
- if q.getresult():
- self.reject("%s: can not overwrite existing copy already in the archive." % (file))
-
- return self.reject_message
+ q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
+ q = q.filter_by(version=self.pkg.files[file]["version"])
+ q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
+
+ if q.count() > 0:
+ self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
################################################################################
- def check_source_against_db(self, file):
+ def check_source_against_db(self, file, session):
"""
"""
- self.reject_message = ""
- dsc = self.pkg.dsc
+ source = self.pkg.dsc.get("source")
+ version = self.pkg.dsc.get("version")
# Ensure version is sane
- q = self.projectB.query("""
-SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
- WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")))
- self.cross_suite_version_check(q.getresult(), file, dsc.get("version"),
- sourceful=True)
+ q = session.query(SrcAssociation)
+ q = q.join(DBSource).filter(DBSource.source==source)
- return self.reject_message
+ self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
+ file, version, sourceful=True)
################################################################################
-
-
- def check_dsc_against_db(self, file):
+ def check_dsc_against_db(self, file, session):
"""
@warning: NB: this function can remove entries from the 'files' index [if
- the .orig.tar.gz is a duplicate of the one in the archive]; if
+ the orig tarball is a duplicate of the one in the archive]; if
you're iterating over 'files' and call this function as part of
the loop, be sure to add a check to the top of the loop to
ensure you haven't just tried to dereference the deleted entry.
"""
- self.reject_message = ""
- files = self.pkg.files
- dsc_files = self.pkg.dsc_files
- self.pkg.orig_tar_gz = None
+
+ Cnf = Config()
+ self.pkg.orig_files = {} # XXX: do we need to clear it?
+ orig_files = self.pkg.orig_files
# Try and find all files mentioned in the .dsc. This has
# to work harder to cope with the multiple possible
# locations of an .orig.tar.gz.
# The ordering on the select is needed to pick the newest orig
# when it exists in multiple places.
- for dsc_file in dsc_files.keys():
+ for dsc_name, dsc_entry in self.pkg.dsc_files.items():
found = None
- if files.has_key(dsc_file):
- actual_md5 = files[dsc_file]["md5sum"]
- actual_size = int(files[dsc_file]["size"])
- found = "%s in incoming" % (dsc_file)
+ if self.pkg.files.has_key(dsc_name):
+ actual_md5 = self.pkg.files[dsc_name]["md5sum"]
+ actual_size = int(self.pkg.files[dsc_name]["size"])
+ found = "%s in incoming" % (dsc_name)
+
# Check the file does not already exist in the archive
- q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location ORDER BY f.id DESC" % (dsc_file))
- ql = q.getresult()
+ ql = get_poolfile_like_name(dsc_name, session)
+
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
- if i[3] != dsc_file and i[3][-(len(dsc_file)+1):] != '/'+dsc_file:
+ if not i.filename.endswith(dsc_name):
ql.remove(i)
# "[dak] has not broken them. [dak] has fixed a
# the same name and version.)"
# -- ajk@ on d-devel@l.d.o
- if ql:
+ if len(ql) > 0:
# Ignore exact matches for .orig.tar.gz
match = 0
- if dsc_file.endswith(".orig.tar.gz"):
+ if re_is_orig_source.match(dsc_name):
for i in ql:
- if files.has_key(dsc_file) and \
- int(files[dsc_file]["size"]) == int(i[0]) and \
- files[dsc_file]["md5sum"] == i[1]:
- self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ")
- del files[dsc_file]
- self.pkg.orig_tar_gz = i[2] + i[3]
+ if self.pkg.files.has_key(dsc_name) and \
+ int(self.pkg.files[dsc_name]["size"]) == int(i.filesize) and \
+ self.pkg.files[dsc_name]["md5sum"] == i.md5sum:
+ self.warnings.append("ignoring %s, since it's already in the archive." % (dsc_name))
+ # TODO: Don't delete the entry, just mark it as not needed
+ # This would fix the stupidity of changing something we often iterate over
+ # whilst we're doing it
+ del self.pkg.files[dsc_name]
+ if not orig_files.has_key(dsc_name):
+ orig_files[dsc_name] = {}
+ orig_files[dsc_name]["path"] = os.path.join(i.location.path, i.filename)
match = 1
if not match:
- self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file))
- elif dsc_file.endswith(".orig.tar.gz"):
+ self.rejects.append("can not overwrite existing copy of '%s' already in the archive." % (dsc_name))
+
+ elif re_is_orig_source.match(dsc_name):
# Check in the pool
- q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file))
- ql = q.getresult()
+ ql = get_poolfile_like_name(dsc_name, session)
+
# Strip out anything that isn't '%s' or '/%s$'
+ # TODO: Shouldn't we just search for things which end with our string explicitly in the SQL?
for i in ql:
- if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
+ if not i.filename.endswith(dsc_name):
ql.remove(i)
- if ql:
+ if len(ql) > 0:
# Unfortunately, we may get more than one match here if,
# for example, the package was in potato but had an -sa
# upload in woody. So we need to choose the right one.
if len(ql) > 1:
for i in ql:
- old_file = i[0] + i[1]
+ old_file = os.path.join(i.location.path, i.filename)
old_file_fh = utils.open_file(old_file)
actual_md5 = apt_pkg.md5sum(old_file_fh)
old_file_fh.close()
actual_size = os.stat(old_file)[stat.ST_SIZE]
- if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
+ if actual_md5 == dsc_entry["md5sum"] and actual_size == int(dsc_entry["size"]):
x = i
- old_file = x[0] + x[1]
+ old_file = os.path.join(i.location.path, i.filename)
old_file_fh = utils.open_file(old_file)
actual_md5 = apt_pkg.md5sum(old_file_fh)
old_file_fh.close()
actual_size = os.stat(old_file)[stat.ST_SIZE]
found = old_file
- suite_type = x[2]
+ suite_type = x.location.archive_type
# need this for updating dsc_files in install()
- dsc_files[dsc_file]["files id"] = x[3]
+ dsc_entry["files id"] = x.file_id
# See install() in process-accepted...
- self.pkg.orig_tar_id = x[3]
- self.pkg.orig_tar_gz = old_file
- self.pkg.orig_tar_location = x[4]
+ if not orig_files.has_key(dsc_name):
+ orig_files[dsc_name] = {}
+ orig_files[dsc_name]["id"] = x.file_id
+ orig_files[dsc_name]["path"] = old_file
+ orig_files[dsc_name]["location"] = x.location.location_id
else:
+ # TODO: Record the queues and info in the DB so we don't hardcode all this crap
# Not there? Check the queue directories...
-
- in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file)
- # See process_it() in 'dak process-unchecked' for explanation of this
- # in_unchecked check dropped by ajt 2007-08-28, how did that
- # ever make sense?
- if os.path.exists(in_unchecked) and False:
- return (self.reject_message, in_unchecked)
- else:
- for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
- in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (directory)],dsc_file)
- if os.path.exists(in_otherdir):
- in_otherdir_fh = utils.open_file(in_otherdir)
- actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
- in_otherdir_fh.close()
- actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
- found = in_otherdir
- self.pkg.orig_tar_gz = in_otherdir
+ for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
+ if not Cnf.has_key("Dir::Queue::%s" % (directory)):
+ continue
+ in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
+ if os.path.exists(in_otherdir):
+ in_otherdir_fh = utils.open_file(in_otherdir)
+ actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
+ in_otherdir_fh.close()
+ actual_size = os.stat(in_otherdir)[stat.ST_SIZE]
+ found = in_otherdir
+ if not orig_files.has_key(dsc_name):
+ orig_files[dsc_name] = {}
+ orig_files[dsc_name]["path"] = in_otherdir
if not found:
- self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file))
- self.pkg.orig_tar_gz = -1
+ self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
continue
else:
- self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file))
+ self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
continue
- if actual_md5 != dsc_files[dsc_file]["md5sum"]:
- self.reject("md5sum for %s doesn't match %s." % (found, file))
- if actual_size != int(dsc_files[dsc_file]["size"]):
- self.reject("size for %s doesn't match %s." % (found, file))
+ if actual_md5 != dsc_entry["md5sum"]:
+ self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
+ if actual_size != int(dsc_entry["size"]):
+ self.rejects.append("size for %s doesn't match %s." % (found, file))
- return (self.reject_message, None)
+ ################################################################################
+ def accepted_checks(self, overwrite_checks, session):
+ # Recheck anything that relies on the database; since that's not
+ # frozen between accept and our run time when called from p-a.
- def do_query(self, query):
- """
- Executes a database query. Writes statistics / timing to stderr.
+ # overwrite_checks is set to False when installing to stable/oldstable
- @type query: string
- @param query: database query string, passed unmodified
+ propogate={}
+ nopropogate={}
- @return: db result
+ # Find the .dsc (again)
+ dsc_filename = None
+ for f in self.pkg.files.keys():
+ if self.pkg.files[f]["type"] == "dsc":
+ dsc_filename = f
- @warning: The query is passed B{unmodified}, so be careful what you use this for.
- """
- sys.stderr.write("query: \"%s\" ... " % (query))
- before = time.time()
- r = self.projectB.query(query)
- time_diff = time.time()-before
- sys.stderr.write("took %.3f seconds.\n" % (time_diff))
- return r
+ for checkfile in self.pkg.files.keys():
+ # The .orig.tar.gz can disappear out from under us is it's a
+ # duplicate of one in the archive.
+ if not self.pkg.files.has_key(checkfile):
+ continue
+
+ entry = self.pkg.files[checkfile]
+
+ # Check that the source still exists
+ if entry["type"] == "deb":
+ source_version = entry["source version"]
+ source_package = entry["source package"]
+ if not self.pkg.changes["architecture"].has_key("source") \
+ and not source_exists(source_package, source_version, self.pkg.changes["distribution"].keys()):
+ self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, checkfile))
+
+ # Version and file overwrite checks
+ if overwrite_checks:
+ if entry["type"] == "deb":
+ self.check_binary_against_db(checkfile, session)
+ elif entry["type"] == "dsc":
+ self.check_source_against_db(checkfile, session)
+ self.check_dsc_against_db(dsc_filename, session)
+
+ # propogate in the case it is in the override tables:
+ for suite in self.pkg.changes.get("propdistribution", {}).keys():
+ if self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
+ propogate[suite] = 1
+ else:
+ nopropogate[suite] = 1
+
+ for suite in propogate.keys():
+ if suite in nopropogate:
+ continue
+ self.pkg.changes["distribution"][suite] = 1
+
+ for checkfile in self.pkg.files.keys():
+ # Check the package is still in the override tables
+ for suite in self.pkg.changes["distribution"].keys():
+ if not self.in_override_p(entry["package"], entry["component"], suite, entry.get("dbtype",""), checkfile, session):
+ self.rejects.append("%s is NEW for %s." % (checkfile, suite))
+
+ ################################################################################
+ # This is not really a reject, but an unaccept, but since a) the code for
+ # that is non-trivial (reopen bugs, unannounce etc.), b) this should be
+ # extremely rare, for now we'll go with whining at our admin folks...
+
+ def do_unaccept(self):
+ cnf = Config()
+
+ self.update_subst()
+ self.Subst["__REJECTOR_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__REJECT_MESSAGE__"] = self.package_info()
+ self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
+ self.Subst["__BCC__"] = "X-DAK: dak process-accepted"
+ if cnf.has_key("Dinstall::Bcc"):
+ self.Subst["__BCC__"] += "\nBcc: %s" % (cnf["Dinstall::Bcc"])
+
+ template = os.path.join(cnf["Dir::Templates"], "process-accepted.unaccept")
+
+ reject_mail_message = utils.TemplateSubst(self.Subst, template)
+
+ # Write the rejection email out as the <foo>.reason file
+ reason_filename = os.path.basename(self.pkg.changes_file[:-8]) + ".reason"
+ reject_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
+
+ # If we fail here someone is probably trying to exploit the race
+ # so let's just raise an exception ...
+ if os.path.exists(reject_filename):
+ os.unlink(reject_filename)
+
+ fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
+ os.write(fd, reject_mail_message)
+ os.close(fd)
+
+ utils.send_mail(reject_mail_message)
+
+ del self.Subst["__REJECTOR_ADDRESS__"]
+ del self.Subst["__REJECT_MESSAGE__"]
+ del self.Subst["__CC__"]
+
+ ################################################################################
+ # If any file of an upload has a recent mtime then chances are good
+ # the file is still being uploaded.
+
+ def upload_too_new(self):
+ cnf = Config()
+ too_new = False
+ # Move back to the original directory to get accurate time stamps
+ cwd = os.getcwd()
+ os.chdir(self.pkg.directory)
+ file_list = self.pkg.files.keys()
+ file_list.extend(self.pkg.dsc_files.keys())
+ file_list.append(self.pkg.changes_file)
+ for f in file_list:
+ try:
+ last_modified = time.time()-os.path.getmtime(f)
+ if last_modified < int(cnf["Dinstall::SkipTime"]):
+ too_new = True
+ break
+ except:
+ pass
+
+ os.chdir(cwd)
+ return too_new
re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$")
-re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$")
+orig_source_ext_re = r"orig(?:-.+)?\.tar\.(?:gz|bz2)"
+re_orig_source_ext = re.compile(orig_source_ext_re + "$")
+re_source_ext = re.compile("(" + orig_source_ext_re + r"|debian\.tar\.(?:gz|bz2)|diff\.gz|tar\.(?:gz|bz2)|dsc)$")
+re_issource = re.compile(r"(.+)_(.+?)\." + re_source_ext.pattern)
+re_is_orig_source = re.compile (r"(.+)_(.+?)\.orig(?:-.+)?\.tar\.(?:gz|bz2)$")
re_single_line_field = re.compile(r"^(\S*?)\s*:\s*(.*)")
re_multi_line_field = re.compile(r"^\s(.*)")
re_user_name = re.compile(r"^pub:.*:(.*)<.*$", re.MULTILINE);
re_re_mark = re.compile(r'^RE:')
+re_parse_lintian = re.compile(r"^(W|E|O): (.*?): ([^ ]*) ?(.*)$")
--- /dev/null
+import re
+
+from regexes import re_verwithext
+from dak_exceptions import UnknownFormatError
+
+srcformats = []
+
+def get_format_from_string(txt):
+ """
+ Returns the SourceFormat class that corresponds to the specified .changes
+ Format value. If the string does not match any class, UnknownFormatError
+ is raised.
+ """
+
+ for format in srcformats:
+ if format.re_format.match(txt):
+ return format
+
+ raise UnknownFormatError, "Unknown format %r" % txt
+
+def parse_format(txt):
+ """
+ Parse a .changes Format string into a tuple representation for easy
+ comparison.
+
+ >>> parse_format('1.0')
+ (1, 0)
+ >>> parse_format('8.4 (hardy)')
+ (8, 4, 'hardy')
+
+ If the format doesn't match these forms, raises UnknownFormatError.
+ """
+
+ format = re_verwithext.search(txt)
+
+ if format is None:
+ raise UnknownFormatError, txt
+
+ format = format.groups()
+
+ if format[1] is None:
+ format = int(float(format[0])), 0, format[2]
+ else:
+ format = int(format[0]), int(format[1]), format[2]
+
+ if format[2] is None:
+ format = format[:2]
+
+ return format
+
+class SourceFormat(type):
+ def __new__(cls, name, bases, attrs):
+ klass = super(SourceFormat, cls).__new__(cls, name, bases, attrs)
+ srcformats.append(klass)
+
+ assert str(klass.name)
+ assert iter(klass.requires)
+ assert iter(klass.disallowed)
+
+ klass.re_format = re.compile(klass.format)
+
+ return klass
+
+ @classmethod
+ def reject_msgs(cls, has):
+ if len(cls.requires) != len([x for x in cls.requires if has[x]]):
+ yield "lack of required files for format %s" % cls.name
+
+ for key in cls.disallowed:
+ if has[key]:
+ yield "contains source files not allowed in format %s" % cls.name
+
+ @classmethod
+ def validate_format(cls, format, is_a_dsc=False, field='files'):
+ """
+ Raises UnknownFormatError if the specified format tuple is not valid for
+ this format (for example, the format (1, 0) is not valid for the
+ "3.0 (quilt)" format). Return value is undefined in all other cases.
+ """
+ pass
+
+class FormatOne(SourceFormat):
+ __metaclass__ = SourceFormat
+
+ name = '1.0'
+ format = r'1.0'
+
+ requires = ()
+ disallowed = ('debian_tar', 'more_orig_tar')
+
+ @classmethod
+ def reject_msgs(cls, has):
+ if not (has['native_tar_gz'] or (has['orig_tar_gz'] and has['debian_diff'])):
+ yield "no .tar.gz or .orig.tar.gz+.diff.gz in 'Files' field."
+ if has['native_tar_gz'] and has['debian_diff']:
+ yield "native package with diff makes no sense"
+ if (has['orig_tar_gz'] != has['orig_tar']) or \
+ (has['native_tar_gz'] != has['native_tar']):
+ yield "contains source files not allowed in format %s" % cls.name
+
+ for msg in super(FormatOne, cls).reject_msgs(has):
+ yield msg
+
+ @classmethod
+ def validate_format(cls, format, is_a_dsc=False, field='files'):
+ msg = "Invalid format %s definition: %r" % (cls.name, format)
+
+ if is_a_dsc:
+ if format != (1, 0):
+ raise UnknownFormatError, msg
+ else:
+ if (format < (1,5) or format > (1,8)):
+ raise UnknownFormatError, msg
+ if field != "files" and format < (1,8):
+ raise UnknownFormatError, msg
+
+class FormatThree(SourceFormat):
+ __metaclass__ = SourceFormat
+
+ name = '3.x (native)'
+ format = r'3\.\d+ \(native\)'
+
+ requires = ('native_tar',)
+ disallowed = ('orig_tar', 'debian_diff', 'debian_tar', 'more_orig_tar')
+
+ @classmethod
+ def validate_format(cls, format, **kwargs):
+ if format != (3, 0, 'native'):
+ raise UnknownFormatError, "Invalid format %s definition: %r" % \
+ (cls.name, format)
+
+class FormatThreeQuilt(SourceFormat):
+ __metaclass__ = SourceFormat
+
+ name = '3.x (quilt)'
+ format = r'3\.\d+ \(quilt\)'
+
+ requires = ('orig_tar', 'debian_tar')
+ disallowed = ('debian_diff', 'native_tar')
+
+ @classmethod
+ def validate_format(cls, format, **kwargs):
+ if format != (3, 0, 'quilt'):
+ raise UnknownFormatError, "Invalid format %s definition: %r" % \
+ (cls.name, format)
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Simple summary class for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+from singleton import Singleton
+
+###############################################################################
+
+class SummaryStats(Singleton):
+ def __init__(self, *args, **kwargs):
+ super(SummaryStats, self).__init__(*args, **kwargs)
+
+ def _startup(self):
+ self.reset_accept()
+
+ def reset_accept(self):
+ self.accept_count = 0
+ self.accept_bytes = 0
+
--- /dev/null
+#!/usr/bin/env python
+# vim:set et ts=4 sw=4:
+
+"""Text utility functions
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import email.Header
+
+from dak_exceptions import *
+from regexes import re_parse_maintainer
+
+################################################################################
+
+def force_to_utf8(s):
+ """
+ Forces a string to UTF-8. If the string isn't already UTF-8,
+ it's assumed to be ISO-8859-1.
+ """
+ try:
+ unicode(s, 'utf-8')
+ return s
+ except UnicodeError:
+ latin1_s = unicode(s,'iso8859-1')
+ return latin1_s.encode('utf-8')
+
+def rfc2047_encode(s):
+ """
+ Encodes a (header) string per RFC2047 if necessary. If the
+ string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.
+ """
+ for enc in ['ascii', 'utf-8', 'iso-8859-1']:
+ try:
+ h = email.Header.Header(s, enc, 998)
+ return str(h)
+ except UnicodeError:
+ pass
+
+ # If we get here, we're boned beyond belief
+ return ''
+
+################################################################################
+
+# <Culus> 'The standard sucks, but my tool is supposed to interoperate
+# with it. I know - I'll fix the suckage and make things
+# incompatible!'
+
+def fix_maintainer(maintainer):
+ """
+ Parses a Maintainer or Changed-By field and returns:
+ 1. an RFC822 compatible version,
+ 2. an RFC2047 compatible version,
+ 3. the name
+ 4. the email
+
+ The name is forced to UTF-8 for both 1. and 3.. If the name field
+ contains '.' or ',' (as allowed by Debian policy), 1. and 2. are
+ switched to 'email (name)' format.
+
+ """
+ maintainer = maintainer.strip()
+ if not maintainer:
+ return ('', '', '', '')
+
+ if maintainer.find("<") == -1:
+ email = maintainer
+ name = ""
+ elif (maintainer[0] == "<" and maintainer[-1:] == ">"):
+ email = maintainer[1:-1]
+ name = ""
+ else:
+ m = re_parse_maintainer.match(maintainer)
+ if not m:
+ raise ParseMaintError, "Doesn't parse as a valid Maintainer field."
+ name = m.group(1)
+ email = m.group(2)
+
+ # Get an RFC2047 compliant version of the name
+ rfc2047_name = rfc2047_encode(name)
+
+ # Force the name to be UTF-8
+ name = force_to_utf8(name)
+
+ if name.find(',') != -1 or name.find('.') != -1:
+ rfc822_maint = "%s (%s)" % (email, name)
+ rfc2047_maint = "%s (%s)" % (email, rfc2047_name)
+ else:
+ rfc822_maint = "%s <%s>" % (name, email)
+ rfc2047_maint = "%s <%s>" % (rfc2047_name, email)
+
+ if email.find("@") == -1 and email.find("buildd_") != 0:
+ raise ParseMaintError, "No @ found in email address part."
+
+ return (rfc822_maint, rfc2047_maint, name, email)
+
+################################################################################
--- /dev/null
+#!/usr/bin/env python
+# vim:set et sw=4:
+
+"""
+Urgency Logger class for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+###############################################################################
+
+import os
+import time
+
+from singleton import Singleton
+from config import Config
+from utils import warn, open_file, move
+
+###############################################################################
+
+class UrgencyLog(Singleton):
+ "Urgency Logger object"
+ def __init__(self, *args, **kwargs):
+ super(UrgencyLog, self).__init__(*args, **kwargs)
+
+ def _startup(self):
+ "Initialize a new Urgency Logger object"
+
+ self.timestamp = time.strftime("%Y%m%d%H%M%S")
+
+ # Create the log directory if it doesn't exist
+ self.log_dir = Config()["Dir::UrgencyLog"]
+
+ if not os.path.exists(self.log_dir) or not os.access(self.log_dir, os.W_OK):
+ warn("UrgencyLog directory %s does not exist or is not writeable, using /srv/ftp.debian.org/tmp/ instead" % (self.log_dir))
+ self.log_dir = '/srv/ftp.debian.org/tmp/'
+
+ # Open the logfile
+ self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp)
+ self.log_file = open_file(self.log_filename, 'w')
+ self.writes = 0
+
+ def log(self, source, version, urgency):
+ "Log an event"
+ self.log_file.write(" ".join([source, version, urgency])+'\n')
+ self.log_file.flush()
+ self.writes += 1
+
+ def close(self):
+ "Close a Logger object"
+ self.log_file.flush()
+ self.log_file.close()
+
+ if self.writes:
+ new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp)
+ move(self.log_filename, new_filename)
+ else:
+ os.unlink(self.log_filename)
+
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-import codecs
import commands
import email.Header
import os
import traceback
import stat
import apt_pkg
-import database
import time
import re
import string
import email as modemail
+import subprocess
+
+from dbconn import DBConn, get_architecture, get_component, get_suite
from dak_exceptions import *
+from textutils import fix_maintainer
from regexes import re_html_escaping, html_escaping, re_single_line_field, \
- re_multi_line_field, re_srchasver, re_verwithext, \
- re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark, \
- re_whitespace_comment
+ re_multi_line_field, re_srchasver, re_taint_free, \
+ re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \
+ re_is_orig_source
+
+from srcformats import get_format_from_string
+from collections import defaultdict
################################################################################
known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
+# Monkeypatch commands.getstatusoutput as it returns a "0" exit code in
+# all situations under lenny's Python.
+import commands
+def dak_getstatusoutput(cmd):
+ pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+ output = "".join(pipe.stdout.readlines())
+
+ ret = pipe.wait()
+ if ret is None:
+ ret = 0
+
+ return ret, output
+commands.getstatusoutput = dak_getstatusoutput
+
################################################################################
def html_escape(s):
################################################################################
+def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
+ """
+ Verify that the files listed in the Files field of the .dsc are
+ those expected given the announced Format.
+
+ @type dsc_filename: string
+ @param dsc_filename: path of .dsc file
+
+ @type dsc: dict
+ @param dsc: the content of the .dsc parsed by C{parse_changes()}
+
+ @type dsc_files: dict
+ @param dsc_files: the file list returned by C{build_file_list()}
+
+ @rtype: list
+ @return: all errors detected
+ """
+ rejmsg = []
+
+ # Parse the file if needed
+ if dsc is None:
+ dsc = parse_changes(dsc_filename, signing_rules=1);
+
+ if dsc_files is None:
+ dsc_files = build_file_list(dsc, is_a_dsc=1)
+
+ # Ensure .dsc lists proper set of source files according to the format
+ # announced
+ has = defaultdict(lambda: 0)
+
+ ftype_lookup = (
+ (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')),
+ (r'diff.gz', ('debian_diff',)),
+ (r'tar.gz', ('native_tar_gz', 'native_tar')),
+ (r'debian\.tar\.(gz|bz2)', ('debian_tar',)),
+ (r'orig\.tar\.(gz|bz2)', ('orig_tar',)),
+ (r'tar\.(gz|bz2)', ('native_tar',)),
+ (r'orig-.+\.tar\.(gz|bz2)', ('more_orig_tar',)),
+ )
+
+ for f in dsc_files.keys():
+ m = re_issource.match(f)
+ if not m:
+ rejmsg.append("%s: %s in Files field not recognised as source."
+ % (dsc_filename, f))
+ continue
+
+ # Populate 'has' dictionary by resolving keys in lookup table
+ matched = False
+ for regex, keys in ftype_lookup:
+ if re.match(regex, m.group(3)):
+ matched = True
+ for key in keys:
+ has[key] += 1
+ break
+
+ # File does not match anything in lookup table; reject
+ if not matched:
+ reject("%s: unexpected source file '%s'" % (dsc_filename, f))
+
+ # Check for multiple files
+ for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'):
+ if has[file_type] > 1:
+ rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type))
+
+ # Source format specific tests
+ try:
+ format = get_format_from_string(dsc['format'])
+ rejmsg.extend([
+ '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has)
+ ])
+
+ except UnknownFormatError:
+ # Not an error here for now
+ pass
+
+ return rejmsg
+
+################################################################################
+
def check_hash_fields(what, manifest):
"""
check_hash_fields ensures that there are no checksum fields in the
################################################################################
-def ensure_hashes(changes, dsc, files, dsc_files):
- rejmsg = []
-
- # Make sure we recognise the format of the Files: field in the .changes
- format = changes.get("format", "0.0").split(".", 1)
- if len(format) == 2:
- format = int(format[0]), int(format[1])
- else:
- format = int(float(format[0])), 0
-
- # We need to deal with the original changes blob, as the fields we need
- # might not be in the changes dict serialised into the .dak anymore.
- orig_changes = parse_deb822(changes['filecontents'])
-
- # Copy the checksums over to the current changes dict. This will keep
- # the existing modifications to it intact.
- for field in orig_changes:
- if field.startswith('checksums-'):
- changes[field] = orig_changes[field]
-
- # Check for unsupported hashes
- rejmsg.extend(check_hash_fields(".changes", changes))
- rejmsg.extend(check_hash_fields(".dsc", dsc))
-
- # We have to calculate the hash if we have an earlier changes version than
- # the hash appears in rather than require it exist in the changes file
- for hashname, hashfunc, version in known_hashes:
- rejmsg.extend(_ensure_changes_hash(changes, format, version, files,
- hashname, hashfunc))
- if "source" in changes["architecture"]:
- rejmsg.extend(_ensure_dsc_hash(dsc, dsc_files, hashname,
- hashfunc))
-
- return rejmsg
-
def parse_checksums(where, files, manifest, hashname):
rejmsg = []
field = 'checksums-%s' % hashname
for line in manifest[field].split('\n'):
if not line:
break
- checksum, size, checkfile = line.strip().split(' ')
+ clist = line.strip().split(' ')
+ if len(clist) == 3:
+ checksum, size, checkfile = clist
+ else:
+ rejmsg.append("Cannot parse checksum line [%s]" % (line))
+ continue
if not files.has_key(checkfile):
# TODO: check for the file's entry in the original files dict, not
# the one modified by (auto)byhand and other weird stuff
if not changes.has_key(field):
raise NoFilesFieldError
- # Make sure we recognise the format of the Files: field
- format = re_verwithext.search(changes.get("format", "0.0"))
- if not format:
- raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
-
- format = format.groups()
- if format[1] == None:
- format = int(float(format[0])), 0, format[2]
- else:
- format = int(format[0]), int(format[1]), format[2]
- if format[2] == None:
- format = format[:2]
-
- if is_a_dsc:
- # format = (1,0) are the only formats we currently accept,
- # format = (0,0) are missing format headers of which we still
- # have some in the archive.
- if format != (1,0) and format != (0,0):
- raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
- else:
- if (format < (1,5) or format > (1,8)):
- raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
- if field != "files" and format < (1,8):
- raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+ # Get SourceFormat object for this Format and validate it
+ format = get_format_from_string(changes['format'])
+ format.validate_format(is_a_dsc=is_a_dsc, field=field)
includes_section = (not is_a_dsc) and field == "files"
################################################################################
-def force_to_utf8(s):
- """
- Forces a string to UTF-8. If the string isn't already UTF-8,
- it's assumed to be ISO-8859-1.
- """
- try:
- unicode(s, 'utf-8')
- return s
- except UnicodeError:
- latin1_s = unicode(s,'iso8859-1')
- return latin1_s.encode('utf-8')
-
-def rfc2047_encode(s):
- """
- Encodes a (header) string per RFC2047 if necessary. If the
- string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.
- """
- try:
- codecs.lookup('ascii')[1](s)
- return s
- except UnicodeError:
- pass
- try:
- codecs.lookup('utf-8')[1](s)
- h = email.Header.Header(s, 'utf-8', 998)
- return str(h)
- except UnicodeError:
- h = email.Header.Header(s, 'iso-8859-1', 998)
- return str(h)
-
-################################################################################
-
-# <Culus> 'The standard sucks, but my tool is supposed to interoperate
-# with it. I know - I'll fix the suckage and make things
-# incompatible!'
-
-def fix_maintainer (maintainer):
- """
- Parses a Maintainer or Changed-By field and returns:
- 1. an RFC822 compatible version,
- 2. an RFC2047 compatible version,
- 3. the name
- 4. the email
-
- The name is forced to UTF-8 for both 1. and 3.. If the name field
- contains '.' or ',' (as allowed by Debian policy), 1. and 2. are
- switched to 'email (name)' format.
-
- """
- maintainer = maintainer.strip()
- if not maintainer:
- return ('', '', '', '')
-
- if maintainer.find("<") == -1:
- email = maintainer
- name = ""
- elif (maintainer[0] == "<" and maintainer[-1:] == ">"):
- email = maintainer[1:-1]
- name = ""
- else:
- m = re_parse_maintainer.match(maintainer)
- if not m:
- raise ParseMaintError, "Doesn't parse as a valid Maintainer field."
- name = m.group(1)
- email = m.group(2)
-
- # Get an RFC2047 compliant version of the name
- rfc2047_name = rfc2047_encode(name)
-
- # Force the name to be UTF-8
- name = force_to_utf8(name)
-
- if name.find(',') != -1 or name.find('.') != -1:
- rfc822_maint = "%s (%s)" % (email, name)
- rfc2047_maint = "%s (%s)" % (email, rfc2047_name)
- else:
- rfc822_maint = "%s <%s>" % (name, email)
- rfc2047_maint = "%s <%s>" % (rfc2047_name, email)
-
- if email.find("@") == -1 and email.find("buildd_") != 0:
- raise ParseMaintError, "No @ found in email address part."
-
- return (rfc822_maint, rfc2047_maint, name, email)
-
-################################################################################
-
def send_mail (message, filename=""):
"""sendmail wrapper, takes _either_ a message string or a file as arguments"""
################################################################################
-# Escape characters which have meaning to SQL's regex comparison operator ('~')
-# (woefully incomplete)
-
-def regex_safe (s):
- s = s.replace('+', '\\\\+')
- s = s.replace('.', '\\\\.')
- return s
-
-################################################################################
-
def TemplateSubst(map, filename):
""" Perform a substition of template """
templatefile = open_file(filename)
template = templatefile.read()
for x in map.keys():
- template = template.replace(x,map[x])
+ template = template.replace(x, str(map[x]))
templatefile.close()
return template
def parse_args(Options):
""" Handle -a, -c and -s arguments; returns them as SQL constraints """
+ # XXX: This should go away and everything which calls it be converted
+ # to use SQLA properly. For now, we'll just fix it not to use
+ # the old Pg interface though
+ session = DBConn().session()
# Process suite
if Options["Suite"]:
suite_ids_list = []
- for suite in split_args(Options["Suite"]):
- suite_id = database.get_suite_id(suite)
- if suite_id == -1:
- warn("suite '%s' not recognised." % (suite))
+ for suitename in split_args(Options["Suite"]):
+ suite = get_suite(suitename, session=session)
+ if suite.suite_id is None:
+ warn("suite '%s' not recognised." % (suite.suite_name))
else:
- suite_ids_list.append(suite_id)
+ suite_ids_list.append(suite.suite_id)
if suite_ids_list:
con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
else:
# Process component
if Options["Component"]:
component_ids_list = []
- for component in split_args(Options["Component"]):
- component_id = database.get_component_id(component)
- if component_id == -1:
- warn("component '%s' not recognised." % (component))
+ for componentname in split_args(Options["Component"]):
+ component = get_component(componentname, session=session)
+ if component is None:
+ warn("component '%s' not recognised." % (componentname))
else:
- component_ids_list.append(component_id)
+ component_ids_list.append(component.component_id)
if component_ids_list:
con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
else:
# Process architecture
con_architectures = ""
+ check_source = 0
if Options["Architecture"]:
arch_ids_list = []
- check_source = 0
- for architecture in split_args(Options["Architecture"]):
- if architecture == "source":
+ for archname in split_args(Options["Architecture"]):
+ if archname == "source":
check_source = 1
else:
- architecture_id = database.get_architecture_id(architecture)
- if architecture_id == -1:
- warn("architecture '%s' not recognised." % (architecture))
+ arch = get_architecture(archname, session=session)
+ if arch is None:
+ warn("architecture '%s' not recognised." % (archname))
else:
- arch_ids_list.append(architecture_id)
+ arch_ids_list.append(arch.arch_id)
if arch_ids_list:
con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
else:
################################################################################
-def check_signature (sig_filename, reject, data_filename="", keyrings=None, autofetch=None):
+def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None):
"""
Check the signature of a file and return the fingerprint if the
signature is valid or 'None' if it's not. The first argument is the
used.
"""
+ rejects = []
+
# Ensure the filename contains no shell meta-characters or other badness
if not re_taint_free.match(sig_filename):
- reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
- return None
+ rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
+ return (None, rejects)
if data_filename and not re_taint_free.match(data_filename):
- reject("!!WARNING!! tainted data filename: '%s'." % (data_filename))
- return None
+ rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
+ return (None, rejects)
if not keyrings:
keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
if autofetch:
error_msg = retrieve_key(sig_filename)
if error_msg:
- reject(error_msg)
- return None
+ rejects.append(error_msg)
+ return (None, rejects)
# Build the command line
status_read, status_write = os.pipe()
# If we failed to parse the status-fd output, let's just whine and bail now
if internal_error:
- reject("internal error while performing signature check on %s." % (sig_filename))
- reject(internal_error, "")
- reject("Please report the above errors to the Archive maintainers by replying to this mail.", "")
- return None
+ rejects.append("internal error while performing signature check on %s." % (sig_filename))
+ rejects.append(internal_error, "")
+ rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
+ return (None, rejects)
- bad = ""
# Now check for obviously bad things in the processed output
if keywords.has_key("KEYREVOKED"):
- reject("The key used to sign %s has been revoked." % (sig_filename))
- bad = 1
+ rejects.append("The key used to sign %s has been revoked." % (sig_filename))
if keywords.has_key("BADSIG"):
- reject("bad signature on %s." % (sig_filename))
- bad = 1
+ rejects.append("bad signature on %s." % (sig_filename))
if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
- reject("failed to check signature on %s." % (sig_filename))
- bad = 1
+ rejects.append("failed to check signature on %s." % (sig_filename))
if keywords.has_key("NO_PUBKEY"):
args = keywords["NO_PUBKEY"]
if len(args) >= 1:
key = args[0]
- reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
- bad = 1
+ rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
if keywords.has_key("BADARMOR"):
- reject("ASCII armour of signature was corrupt in %s." % (sig_filename))
- bad = 1
+ rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))
if keywords.has_key("NODATA"):
- reject("no signature found in %s." % (sig_filename))
- bad = 1
+ rejects.append("no signature found in %s." % (sig_filename))
if keywords.has_key("EXPKEYSIG"):
args = keywords["EXPKEYSIG"]
if len(args) >= 1:
key = args[0]
- reject("Signature made by expired key 0x%s" % (key))
- bad = 1
+ rejects.append("Signature made by expired key 0x%s" % (key))
if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
args = keywords["KEYEXPIRED"]
expiredate=""
expiredate = "unknown (%s)" % (timestamp)
else:
expiredate = timestamp
- reject("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
- bad = 1
+ rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
- if bad:
- return None
+ if len(rejects) > 0:
+ return (None, rejects)
# Next check gpgv exited with a zero return code
if exit_status:
- reject("gpgv failed while checking %s." % (sig_filename))
+ rejects.append("gpgv failed while checking %s." % (sig_filename))
if status.strip():
- reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
+ rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
else:
- reject(prefix_multi_line_string(output, " [GPG output:] "), "")
- return None
+ rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "")
+ return (None, rejects)
# Sanity check the good stuff we expect
if not keywords.has_key("VALIDSIG"):
- reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
- bad = 1
+ rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
else:
args = keywords["VALIDSIG"]
if len(args) < 1:
- reject("internal error while checking signature on %s." % (sig_filename))
- bad = 1
+ rejects.append("internal error while checking signature on %s." % (sig_filename))
else:
fingerprint = args[0]
if not keywords.has_key("GOODSIG"):
- reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
- bad = 1
+ rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
if not keywords.has_key("SIG_ID"):
- reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
- bad = 1
+ rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
# Finally ensure there's not something we don't recognise
known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
for keyword in keywords.keys():
if not known_keywords.has_key(keyword):
- reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
- bad = 1
+ rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
- if bad:
- return None
+ if len(rejects) > 0:
+ return (None, rejects)
else:
- return fingerprint
+ return (fingerprint, [])
################################################################################
# apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
###############################################################################
+
+def ensure_orig_files(changes, dest_dir, session):
+ """
+ Ensure that dest_dir contains all the orig tarballs for the specified
+ changes. If it does not, symlink them into place.
+
+ Returns a 2-tuple (already_exists, symlinked) containing a list of files
+ that were already there and a list of files that were symlinked into place.
+ """
+
+ exists, symlinked = [], []
+
+ for dsc_file in changes.dsc_files:
+
+ # Skip all files that are not orig tarballs
+ if not re_is_orig_source.match(dsc_file):
+ continue
+
+ # Skip orig files not identified in the pool
+ if not (dsc_file in changes.orig_files and
+ 'id' in changes.orig_files[dsc_file]):
+ continue
+
+ dest = os.path.join(dest_dir, dsc_file)
+
+ if os.path.exists(dest):
+ exists.append(dest)
+ continue
+
+ orig_file_id = changes.orig_files[dsc_file]['id']
+
+ c = session.execute(
+ 'SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id',
+ {'id': orig_file_id}
+ )
+
+ res = c.fetchone()
+ if not res:
+ return "[INTERNAL ERROR] Couldn't find id %s in files table." % orig_file_id
+
+ src = os.path.join(res[0], res[1])
+ os.symlink(src, dest)
+ symlinked.append(dest)
+
+ return (exists, symlinked)
Various
-------
-* Lintian based automated rejects
- - Have a set of lintian tags each package *must* not have. If it does
- -> reject.
- - If a tag is overriden by the maintainer, do not reject, but put it
- into NEW. If the override is ok note that in a table and dont act on
- it for any future uploads of this package anymore.
- - possibly have two classes of tags. one for "shouldnt happen by
- accident" and one "shouldnt happen". the first gets ignored from us
- if overwritten in the package, the second only us can overwrite.
- - its a suite option in dak, not active for all at once.
+* Implement autosigning, see ftpmaster_autosigning on ftp-master host in text/.
* Throw away all DD uploaded .debs. (Depend on "Lintian based automated
rejects")
- its a suite option, not active for all at once.
- should have all buildd machines under dsa control
-* Implement autosigning, see ftpmaster_autosigning on ftp-master host in text/.
-
* Check TODO.old and move still-valid/useful entries over here.
* need a testsuite _badly_
- needs updateX.py written and then the rest of the code changed to deal
with it.
-* Checkout SQL Alchemy and probably use that for our database layer.
-
-* reject on > or < in a version constraint
-
* use pythonX.Y-tarfile to check orig.tar.gz timestamps too.
* the .dak stuff is fundamentally braindamaged for various reasons, it
+#!/usr/bin/env python
+
import unittest
import os, sys
self.assertEqual(self.MATCH(': ::').groups(), ('', '::'))
self.assertEqual(self.MATCH('Foo::bar').groups(), ('Foo', ':bar'))
self.assertEqual(self.MATCH('Foo: :bar').groups(), ('Foo', ':bar'))
+
+class re_parse_lintian(unittest.TestCase):
+ MATCH = regexes.re_parse_lintian.match
+
+ def testBinary(self):
+ self.assertEqual(
+ self.MATCH('W: pkgname: some-tag path/to/file').groups(),
+ ('W', 'pkgname', 'some-tag', 'path/to/file')
+ )
+
+ def testBinaryNoDescription(self):
+ self.assertEqual(
+ self.MATCH('W: pkgname: some-tag').groups(),
+ ('W', 'pkgname', 'some-tag', '')
+ )
+
+ def testSource(self):
+ self.assertEqual(
+ self.MATCH('W: pkgname source: some-tag').groups(),
+ ('W', 'pkgname source', 'some-tag', '')
+ )
+
+ def testSourceNoDescription(self):
+ self.assertEqual(
+ self.MATCH('W: pkgname source: some-tag path/to/file').groups(),
+ ('W', 'pkgname source', 'some-tag', 'path/to/file')
+ )
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+#!/usr/bin/env python
+
+import unittest
+
+import os, sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from collections import defaultdict
+
+from daklib import srcformats
+from daklib.dak_exceptions import UnknownFormatError
+
+class SourceFormatTestCase(unittest.TestCase):
+ def get_rejects(self, has_vars):
+ has = defaultdict(lambda: 0)
+ has.update(has_vars)
+ return list(self.fmt.reject_msgs(has))
+
+ def assertAccepted(self, has):
+ self.assertEqual(self.get_rejects(has), [])
+
+ def assertRejected(self, has):
+ self.assertNotEqual(self.get_rejects(has), [])
+
+class FormatOneTestCase(SourceFormatTestCase):
+ fmt = srcformats.FormatOne
+
+ def testEmpty(self):
+ self.assertRejected({})
+
+ def testNative(self):
+ self.assertAccepted({'native_tar': 1, 'native_tar_gz': 1})
+
+ def testStandard(self):
+ self.assertAccepted({
+ 'orig_tar': 1,
+ 'orig_tar_gz': 1,
+ 'debian_diff': 1,
+ })
+
+ def testDisallowed(self):
+ self.assertRejected({
+ 'native_tar': 1,
+ 'native_tar_gz': 1,
+ 'debian_tar': 1,
+ })
+ self.assertRejected({
+ 'orig_tar': 1,
+ 'orig_tar_gz': 1,
+ 'debian_diff': 0,
+ })
+ self.assertRejected({
+ 'native_tar': 1,
+ 'native_tar_gz': 1,
+ 'more_orig_tar': 1,
+ })
+ self.assertRejected({
+ 'native_tar': 1,
+ 'native_tar_gz': 1,
+ 'debian_diff': 1,
+ })
+
+class FormatTreeTestCase(SourceFormatTestCase):
+ fmt = srcformats.FormatThree
+
+ def testEmpty(self):
+ self.assertRejected({})
+
+ def testSimple(self):
+ self.assertAccepted({'native_tar': 1})
+
+ def testDisallowed(self):
+ self.assertRejected({'native_tar': 1, 'orig_tar': 1})
+ self.assertRejected({'native_tar': 1, 'debian_diff': 1})
+ self.assertRejected({'native_tar': 1, 'debian_tar': 1})
+ self.assertRejected({'native_tar': 1, 'more_orig_tar': 1})
+
+class FormatTreeQuiltTestCase(SourceFormatTestCase):
+ fmt = srcformats.FormatThreeQuilt
+
+ def testEmpty(self):
+ self.assertRejected({})
+
+ def testSimple(self):
+ self.assertAccepted({'orig_tar': 1, 'debian_tar': 1})
+
+ def testMultipleTarballs(self):
+ self.assertAccepted({
+ 'orig_tar': 1,
+ 'debian_tar': 1,
+ 'more_orig_tar': 42,
+ })
+
+ def testDisallowed(self):
+ self.assertRejected({
+ 'orig_tar': 1,
+ 'debian_tar': 1,
+ 'debian_diff': 1
+ })
+ self.assertRejected({
+ 'orig_tar': 1,
+ 'debian_tar': 1,
+ 'native_tar': 1,
+ })
+
+##
+
+class ParseFormatTestCase(unittest.TestCase):
+ def assertParse(self, format, expected):
+ self.assertEqual(srcformats.parse_format(format), expected)
+
+ def assertParseFail(self, format):
+ self.assertRaises(
+ UnknownFormatError,
+ lambda: srcformats.parse_format(format)
+ )
+
+ def testParse(self):
+ self.assertParse('1.0', (1, 0))
+
+ def testEmpty(self):
+ self.assertParseFail('')
+ self.assertParseFail(' ')
+ self.assertParseFail(' ')
+
+ def textText(self):
+ self.assertParse('1.2 (three)', (1, 2, 'three'))
+ self.assertParseFail('0.0 ()')
+
+class ValidateFormatTestCase(unittest.TestCase):
+ def assertValid(self, format, **kwargs):
+ kwargs['is_a_dsc'] = kwargs.get('is_a_dsc', True)
+ self.fmt.validate_format(format, **kwargs)
+
+ def assertInvalid(self, *args, **kwargs):
+ self.assertRaises(
+ UnknownFormatError,
+ lambda: self.assertValid(*args, **kwargs),
+ )
+
+class ValidateFormatOneTestCase(ValidateFormatTestCase):
+ fmt = srcformats.FormatOne
+
+ def testValid(self):
+ self.assertValid((1, 0))
+
+ def testInvalid(self):
+ self.assertInvalid((0, 1))
+ self.assertInvalid((3, 0, 'quilt'))
+
+ ##
+
+ def testBinary(self):
+ self.assertValid((1, 5), is_a_dsc=False)
+ self.assertInvalid((1, 0), is_a_dsc=False)
+
+ def testRange(self):
+ self.assertInvalid((1, 3), is_a_dsc=False)
+ self.assertValid((1, 5), is_a_dsc=False)
+ self.assertValid((1, 8), is_a_dsc=False)
+ self.assertInvalid((1, 9), is_a_dsc=False)
+
+ def testFilesField(self):
+ self.assertInvalid((1, 7), is_a_dsc=False, field='notfiles')
+ self.assertValid((1, 8), is_a_dsc=False, field='notfiles')
+
+class ValidateFormatThreeTestCase(ValidateFormatTestCase):
+ fmt = srcformats.FormatThree
+
+ def testValid(self):
+ self.assertValid((3, 0, 'native'))
+
+ def testInvalid(self):
+ self.assertInvalid((1, 0))
+ self.assertInvalid((0, 0))
+ self.assertInvalid((3, 0, 'quilt'))
+
+class ValidateFormatThreeQuiltTestCase(ValidateFormatTestCase):
+ fmt = srcformats.FormatThreeQuilt
+
+ def testValid(self):
+ self.assertValid((3, 0, 'quilt'))
+
+ def testInvalid(self):
+ self.assertInvalid((1, 0))
+ self.assertInvalid((0, 0))
+ self.assertInvalid((3, 0, 'native'))
+
+class FormatFromStringTestCase(unittest.TestCase):
+ def assertFormat(self, txt, klass):
+ self.assertEqual(srcformats.get_format_from_string(txt), klass)
+
+ def assertInvalid(self, txt):
+ self.assertRaises(
+ UnknownFormatError,
+ lambda: srcformats.get_format_from_string(txt),
+ )
+
+ def testFormats(self):
+ self.assertFormat('1.0', srcformats.FormatOne)
+ self.assertFormat('3.0 (native)', srcformats.FormatThree)
+ self.assertFormat('3.0 (quilt)', srcformats.FormatThreeQuilt)
+
+ def testInvalidFormats(self):
+ self.assertInvalid('')
+ self.assertInvalid('.')
+ self.assertInvalid('3.0 (cvs)')
+ self.assertInvalid(' 1.0 ')
+ self.assertInvalid('8.4 (hardy)')
+
+if __name__ == '__main__':
+ unittest.main()
$selecteddelayed = $1;
s,^DELAYED/[0-9]+-day/,,;
}
- if ( $origword eq "--searchdirs" ) {
+ if (m,(^|/)\*,) {
+ msg("mail,log", "$_: filename component cannot start with a wildcard\n");
+ } elsif ( $origword eq "--searchdirs" ) {
$selecteddelayed = -2;
} elsif (m,/,) {
msg(
projects / dak.git / commitdiff