# Stolen from userdir-ldap
# Compute a random password using /dev/urandom.
def GenPass():
- # Generate a 10 character random string
- SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/."
- Rand = open("/dev/urandom")
- Password = ""
- for i in range(0,15):
- Password = Password + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
- return Password
+ # Generate a 10 character random string
+ SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/."
+ Rand = open("/dev/urandom")
+ Password = ""
+ for i in range(0,15):
+ Password = Password + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
+ return Password
# Compute the MD5 crypted version of the given password
def HashPass(Password):
- import crypt
- # Hash it telling glibc to use the MD5 algorithm - if you dont have
- # glibc then just change Salt = "$1$" to Salt = ""
- SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/."
- Salt = "$1$"
- Rand = open("/dev/urandom")
- for x in range(0,10):
- Salt = Salt + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
- Pass = crypt.crypt(Password,Salt)
- if len(Pass) < 14:
- raise "Password Error", "MD5 password hashing failed, not changing the password!"
- return Pass
+ import crypt
+ # Hash it telling glibc to use the MD5 algorithm - if you dont have
+ # glibc then just change Salt = "$1$" to Salt = ""
+ SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/."
+ Salt = "$1$"
+ Rand = open("/dev/urandom")
+ for x in range(0,10):
+ Salt = Salt + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
+ Pass = crypt.crypt(Password,Salt)
+ if len(Pass) < 14:
+ raise "Password Error", "MD5 password hashing failed, not changing the password!"
+ return Pass
################################################################################
]
for i in [ "help", "create" ]:
- if not Cnf.has_key("Add-User::Options::%s" % (i)):
- Cnf["Add-User::Options::%s" % (i)] = ""
+ if not Cnf.has_key("Add-User::Options::%s" % (i)):
+ Cnf["Add-User::Options::%s" % (i)] = ""
apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
(result, output) = commands.getstatusoutput(cmd)
m = re_gpg_fingerprint.search(output)
if not m:
- print output
+ print output
utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s" \
- % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, \
- " [GPG output:] ")))
+ % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, \
+ " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ","")
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
-# Create an account for the user?
- summary = ""
- if Cnf.FindB("Add-User::CreateAccount") or Cnf["Add-User::Options::Create"]:
- password = GenPass()
- pwcrypt = HashPass(password)
- if Cnf.has_key("Add-User::GID"):
- cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' -G %s %s" \
- % (pwcrypt, name, Cnf["Add-User::GID"], uid)
- else:
- cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' %s" \
- % (pwcrypt, name, uid)
- (result, output) = commands.getstatusoutput(cmd)
- if (result != 0):
- utils.fubar("Invocation of '%s' failed:\n%s\n" % (cmd, output), result)
- try:
- summary+=createMail(uid, password, Cnf["Add-User::Options::Key"], Cnf["Dinstall::GPGKeyring"])
- except:
- summary=""
- utils.warn("Could not prepare password information for mail, not sending password.")
-
-# Now add user to the database.
- # Note that we provide a session, so we're responsible for committing
- uidobj = get_or_set_uid(uid, session=session)
- uid_id = uidobj.uid_id
- add_database_user(uid)
- session.commit()
-# The following two are kicked out in rhona, so we don't set them. kelly adds
-# them as soon as she installs a package with unknown ones, so no problems to expect here.
-# Just leave the comment in, to not think about "Why the hell aren't they added" in
-# a year, if we ever touch uma again.
-# maint_id = database.get_or_set_maintainer_id(name)
-# session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)",
-# {'fingerprint': primary_key, 'uid': uid_id})
-
-# Lets add user to the email-whitelist file if its configured.
- if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "":
- file = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
- for mail in emails:
- file.write(mail+'\n')
- file.close()
-
- print "Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, \
- name, primary_key)
-
-# Should we send mail to the newly added user?
- if Cnf.FindB("Add-User::SendEmail"):
- mail = name + "<" + emails[0] +">"
- Subst = {}
- Subst["__NEW_MAINTAINER__"] = mail
- Subst["__UID__"] = uid
- Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
- Subst["__PRIMARY_KEY__"] = primary_key
- Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
- Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
- Subst["__SUMMARY__"] = summary
- new_add_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/add-user.added")
- utils.send_mail(new_add_message)
+ # Create an account for the user?
+ summary = ""
+ if Cnf.FindB("Add-User::CreateAccount") or Cnf["Add-User::Options::Create"]:
+ password = GenPass()
+ pwcrypt = HashPass(password)
+ if Cnf.has_key("Add-User::GID"):
+ cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' -G %s %s" \
+ % (pwcrypt, name, Cnf["Add-User::GID"], uid)
+ else:
+ cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' %s" \
+ % (pwcrypt, name, uid)
+ (result, output) = commands.getstatusoutput(cmd)
+ if (result != 0):
+ utils.fubar("Invocation of '%s' failed:\n%s\n" % (cmd, output), result)
+ try:
+ summary+=createMail(uid, password, Cnf["Add-User::Options::Key"], Cnf["Dinstall::GPGKeyring"])
+ except:
+ summary=""
+ utils.warn("Could not prepare password information for mail, not sending password.")
+
+ # Now add user to the database.
+ # Note that we provide a session, so we're responsible for committing
+ uidobj = get_or_set_uid(uid, session=session)
+ uid_id = uidobj.uid_id
+ add_database_user(uid)
+ session.commit()
+
+ # The following two are kicked out in rhona, so we don't set them. kelly adds
+ # them as soon as she installs a package with unknown ones, so no problems to expect here.
+ # Just leave the comment in, to not think about "Why the hell aren't they added" in
+ # a year, if we ever touch uma again.
+ # maint_id = database.get_or_set_maintainer_id(name)
+ # session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)",
+ # {'fingerprint': primary_key, 'uid': uid_id})
+
+ # Lets add user to the email-whitelist file if its configured.
+ if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "":
+ f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
+ for mail in emails:
+ f.write(mail+'\n')
+ f.close()
+
+ print "Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, \
+ name, primary_key)
+
+ # Should we send mail to the newly added user?
+ if Cnf.FindB("Add-User::SendEmail"):
+ mail = name + "<" + emails[0] +">"
+ Subst = {}
+ Subst["__NEW_MAINTAINER__"] = mail
+ Subst["__UID__"] = uid
+ Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
+ Subst["__PRIMARY_KEY__"] = primary_key
+ Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
+ Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
+ Subst["__SUMMARY__"] = summary
+ new_add_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/add-user.added")
+ utils.send_mail(new_add_message)
else:
- uid = None
-
+ uid = None
#######################################################################################
if __name__ == '__main__':
main()
-
print suite_name
print "-" * len(suite_name)
print
- suite = get_suite(suite)
- q = s.execute("""
+ suite = get_suite(suite_name)
+ q = session.execute("""
SELECT DISTINCT b.package FROM binaries b, bin_associations ba
WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS
(SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)"""
for j in q.fetchall():
print j[0]
- q = s.execute("""
+ q = session.execute("""
SELECT DISTINCT s.source FROM source s, src_associations sa
WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS
(SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)"""
"""
for suite in [ "stable", "testing", "unstable" ]:
for component in Cnf.ValueList("Suite::%s::Components" % (suite)):
- architectures = database.get_suite_architectures(suite)
- for arch in [ i.lower() for i in architectures ]:
+ architectures = get_suite_architectures(suite)
+ for arch in [ i.arch_string.lower() for i in architectures ]:
if arch == "source":
validate_sources(suite, component)
elif arch == "all":
def check_build_depends():
""" Validate build-dependencies of .dsc files in the archive """
+ cnf = Config()
os.path.walk(cnf["Dir::Root"], chk_bd_process_dir, None)
################################################################################
import logging
import gzip
import threading
+import traceback
import Queue
import apt_pkg
from daklib import utils
WHERE ba.suite = :suiteid AND ba.bin = b.id
AND b.architecture = a.id AND b.package = :package""",
{'suiteid': suite_id, 'package': binary})
+ ql = q.fetchall()
versions = []
- for i in q.fetchall():
+ for i in ql:
arch = i[0]
version = i[1]
if architectures.has_key(arch):
"Generate statistics"),
("bts-categorize",
"Categorize uncategorized bugs filed against ftp.debian.org"),
+ ("import-known-changes",
+ "import old changes files into known_changes table"),
("add-user",
"Add a user to the archive"),
]
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Add policy queue handling support
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+import os
+import datetime
+import traceback
+
+from daklib.dak_exceptions import DBUpdateError
+from daklib.config import Config
+
+################################################################################
+
+def do_update(self):
+ print "Updating use of queue table"
+
+ try:
+ c = self.db.cursor()
+
+ cnf = Config()
+
+ print "Adding path to queue table"
+ c.execute("ALTER TABLE queue ADD COLUMN path TEXT")
+ c.execute("SELECT * FROM queue")
+ rows = c.fetchall()
+ seenqueues = {}
+ for row in rows:
+ dir = cnf["Dir::Queue::%s" % row[1]].rstrip('/')
+ seenqueues[row[1].lower()] = 1
+ print "Setting %s queue to use path %s" % (row[1], dir)
+ c.execute("UPDATE queue SET path = %s WHERE id = %s", (dir, row[0]))
+
+ print "Adding missing queues to the queue table"
+ for q in cnf.SubTree("Dir::Queue").keys():
+ qname = q.lower()
+ if qname in seenqueues.keys():
+ continue
+ if qname in ["done", "holding", "reject", "newstage", "btsversiontrack"]:
+ print "Skipping queue %s" % qname
+ continue
+ pth = cnf["Dir::Queue::%s" % qname].rstrip('/')
+ if not os.path.exists(pth):
+ print "Skipping %s as %s does not exist" % (qname, pth)
+ continue
+
+ print "Adding %s queue with path %s" % (qname, pth)
+ c.execute("INSERT INTO queue (queue_name, path) VALUES (%s, %s)", (qname, pth))
+ seenqueues[qname] = 1
+
+ print "Adding queue and approved_for columns to known_changes"
+ c.execute("ALTER TABLE known_changes ADD COLUMN in_queue INT4 REFERENCES queue(id) DEFAULT NULL")
+ c.execute("ALTER TABLE known_changes ADD COLUMN approved_for INT4 REFERENCES queue(id) DEFAULT NULL")
+
+ print "Adding policy queue column to suite table"
+ c.execute("ALTER TABLE suite DROP COLUMN policy_engine")
+ c.execute("ALTER TABLE suite ADD COLUMN policy_queue_id INT4 REFERENCES queue(id) DEFAULT NULL")
+ # Handle some of our common cases automatically
+ if seenqueues.has_key('proposedupdates'):
+ c.execute("""UPDATE suite SET policy_queue_id = (SELECT id FROM queue WHERE queue_name = 'proposedupdates')
+ WHERE suite_name = 'proposed-updates'""")
+
+ if seenqueues.has_key('oldproposedupdates'):
+ c.execute("""UPDATE suite SET policy_queue_id = (SELECT id FROM queue WHERE queue_name = 'oldproposedupdates')
+ WHERE suite_name = 'oldstable-proposed-updates'""")
+
+ print "Committing"
+ c.execute("UPDATE config SET value = '20' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.InternalError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply debversion update 20, rollback issued. Error message : %s" % (str(msg))
#!/usr/bin/env python
-""" Create all the Release files """
-
-# Copyright (C) 2001, 2002, 2006 Anthony Towns <ajt@debian.org>
+""" Create all the Release files
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@Copyright: 2001, 2002, 2006 Anthony Towns <ajt@debian.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
################################################################################
-import sys, os, stat, time
-import gzip, bz2
+import sys
+import os
+import stat
+import time
+import gzip
+import bz2
import apt_pkg
from daklib import utils
dest = Cnf["Dir::Root"] + tree + "/Release.gpg"
if os.path.exists(dest):
os.unlink(dest)
+ inlinedest = Cnf["Dir::Root"] + tree + "/InRelease"
+ if os.path.exists(inlinedest):
+ os.unlink(inlinedest)
for keyid in signkeyids:
- if keyid != "": defkeyid = "--default-key %s" % keyid
- else: defkeyid = ""
+ if keyid != "":
+ defkeyid = "--default-key %s" % keyid
+ else:
+ defkeyid = ""
os.system("gpg %s %s %s --detach-sign <%s >>%s" %
(keyring, defkeyid, arguments,
Cnf["Dir::Root"] + tree + "/Release", dest))
+ os.system("gpg %s %s %s --clearsign <%s >>%s" %
+ (keyring, defkeyid, arguments,
+ Cnf["Dir::Root"] + tree + "/Release", inlinedest))
#######################################################################################
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Import known_changes files
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Mike O'Connor <stew@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import sys
+import os
+import logging
+import threading
+from daklib.dbconn import DBConn,get_knownchange
+from daklib.config import Config
+import apt_pkg
+from daklib.dak_exceptions import DBUpdateError, InvalidDscError, ChangesUnicodeError
+from daklib.changes import Changes
+from daklib.utils import parse_changes, warn, gpgv_get_status_output, process_gpgv_output
+import traceback
+
+# where in dak.conf all of our configuration will be stowed
+options_prefix = "KnownChanges"
+options_prefix = "%s::Options" % options_prefix
+
+log = logging.getLogger()
+
+################################################################################
+
+
+def usage (exit_code=0):
+ print """Usage: dak import-known-changes [options]
+
+OPTIONS
+ -j n
+ run with n threads concurrently
+
+ -v, --verbose
+ show verbose information messages
+
+ -q, --quiet
+ supress all output but errors
+
+"""
+ sys.exit(exit_code)
+
+def check_signature (sig_filename, data_filename=""):
+ fingerprint = None
+
+ keyrings = [
+ "/home/joerg/keyring/keyrings/debian-keyring.gpg",
+ "/home/joerg/keyring/keyrings/debian-keyring.pgp",
+ "/home/joerg/keyring/keyrings/debian-maintainers.gpg",
+ "/home/joerg/keyring/keyrings/debian-role-keys.gpg",
+ "/home/joerg/keyring/keyrings/emeritus-keyring.pgp",
+ "/home/joerg/keyring/keyrings/emeritus-keyring.gpg",
+ "/home/joerg/keyring/keyrings/removed-keys.gpg",
+ "/home/joerg/keyring/keyrings/removed-keys.pgp"
+ ]
+
+ keyringargs = " ".join(["--keyring %s" % x for x in keyrings ])
+
+ # Build the command line
+ status_read, status_write = os.pipe()
+ cmd = "gpgv --status-fd %s %s %s" % (status_write, keyringargs, sig_filename)
+
+ # Invoke gpgv on the file
+ (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
+
+ # Process the status-fd output
+ (keywords, internal_error) = process_gpgv_output(status)
+
+ # If we failed to parse the status-fd output, let's just whine and bail now
+ if internal_error:
+ warn("Couldn't parse signature")
+ return None
+
+ # usually one would check for bad things here. We, however, do not care.
+
+ # Next check gpgv exited with a zero return code
+ if exit_status:
+ warn("Couldn't parse signature")
+ return None
+
+ # Sanity check the good stuff we expect
+ if not keywords.has_key("VALIDSIG"):
+ warn("Couldn't parse signature")
+ else:
+ args = keywords["VALIDSIG"]
+ if len(args) < 1:
+ warn("Couldn't parse signature")
+ else:
+ fingerprint = args[0]
+
+ return fingerprint
+
+
+class EndOfChanges(object):
+ """something enqueued to signify the last change"""
+ pass
+
+
+class OneAtATime(object):
+ """
+ a one space queue which sits between multiple possible producers
+ and multiple possible consumers
+ """
+ def __init__(self):
+ self.next_in_line = None
+ self.read_lock = threading.Condition()
+ self.write_lock = threading.Condition()
+ self.die = False
+
+ def plsDie(self):
+ self.die = True
+ self.write_lock.acquire()
+ self.write_lock.notifyAll()
+ self.write_lock.release()
+
+ self.read_lock.acquire()
+ self.read_lock.notifyAll()
+ self.read_lock.release()
+
+ def enqueue(self, next):
+ self.write_lock.acquire()
+ while self.next_in_line:
+ if self.die:
+ return
+ self.write_lock.wait()
+
+ assert( not self.next_in_line )
+ self.next_in_line = next
+ self.write_lock.release()
+ self.read_lock.acquire()
+ self.read_lock.notify()
+ self.read_lock.release()
+
+ def dequeue(self):
+ self.read_lock.acquire()
+ while not self.next_in_line:
+ if self.die:
+ return
+ self.read_lock.wait()
+
+ result = self.next_in_line
+
+ self.next_in_line = None
+ self.read_lock.release()
+ self.write_lock.acquire()
+ self.write_lock.notify()
+ self.write_lock.release()
+
+ if isinstance(result, EndOfChanges):
+ return None
+
+ return result
+
+class ChangesToImport(object):
+ """A changes file to be enqueued to be processed"""
+ def __init__(self, checkdir, changesfile, count):
+ self.dirpath = checkdir
+ self.changesfile = changesfile
+ self.count = count
+
+ def __str__(self):
+ return "#%d: %s in %s" % (self.count, self.changesfile, self.dirpath)
+
+class ChangesGenerator(threading.Thread):
+ """enqueues changes files to be imported"""
+ def __init__(self, parent, queue):
+ threading.Thread.__init__(self)
+ self.queue = queue
+ self.session = DBConn().session()
+ self.parent = parent
+ self.die = False
+
+ def plsDie(self):
+ self.die = True
+
+ def run(self):
+ cnf = Config()
+ count = 1
+ for directory in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
+ checkdir = cnf["Dir::Queue::%s" % (directory) ]
+ if os.path.exists(checkdir):
+ print "Looking into %s" % (checkdir)
+
+ for dirpath, dirnames, filenames in os.walk(checkdir, topdown=True):
+ if not filenames:
+ # Empty directory (or only subdirectories), next
+ continue
+
+ for changesfile in filenames:
+ try:
+ if not changesfile.endswith(".changes"):
+ # Only interested in changes files.
+ continue
+ count += 1
+
+ if not get_knownchange(changesfile, self.session):
+ to_import = ChangesToImport(dirpath, changesfile, count)
+ if self.die:
+ return
+ self.queue.enqueue(to_import)
+ except KeyboardInterrupt:
+ print("got Ctrl-c in enqueue thread. terminating")
+ self.parent.plsDie()
+ sys.exit(1)
+
+ self.queue.enqueue(EndOfChanges())
+
+class ImportThread(threading.Thread):
+ def __init__(self, parent, queue):
+ threading.Thread.__init__(self)
+ self.queue = queue
+ self.session = DBConn().session()
+ self.parent = parent
+ self.die = False
+
+ def plsDie(self):
+ self.die = True
+
+ def run(self):
+ while True:
+ try:
+ if self.die:
+ return
+ to_import = self.queue.dequeue()
+ if not to_import:
+ return
+
+ print( "Directory %s, file %7d, (%s)" % (to_import.dirpath[-10:], to_import.count, to_import.changesfile) )
+
+ changes = Changes()
+ changes.changes_file = to_import.changesfile
+ changesfile = os.path.join(to_import.dirpath, to_import.changesfile)
+ changes.changes = parse_changes(changesfile, signing_rules=-1)
+ changes.changes["fingerprint"] = check_signature(changesfile)
+ changes.add_known_changes(to_import.dirpath, self.session)
+ self.session.commit()
+
+ except InvalidDscError, line:
+ warn("syntax error in .dsc file '%s', line %s." % (f, line))
+
+ except ChangesUnicodeError:
+ warn("found invalid changes file, not properly utf-8 encoded")
+
+ except KeyboardInterrupt:
+ print("Caught C-c; on ImportThread. terminating.")
+ self.parent.plsDie()
+ sys.exit(1)
+
+ except:
+ self.parent.plsDie()
+ sys.exit(1)
+
+class ImportKnownChanges(object):
+ def __init__(self,num_threads):
+ self.queue = OneAtATime()
+ self.threads = [ ChangesGenerator(self,self.queue) ]
+
+ for i in range(num_threads):
+ self.threads.append( ImportThread(self,self.queue) )
+
+ try:
+ for thread in self.threads:
+ thread.start()
+
+ except KeyboardInterrupt:
+ print("Caught C-c; terminating.")
+ utils.warn("Caught C-c; terminating.")
+ self.plsDie()
+
+ def plsDie(self):
+ traceback.print_stack90
+ for thread in self.threads:
+ print( "STU: before ask %s to die" % thread )
+ thread.plsDie()
+ print( "STU: after ask %s to die" % thread )
+
+ self.threads=[]
+ sys.exit(1)
+
+
+def main():
+ cnf = Config()
+
+ arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
+ ('j',"concurrency", "%s::%s" % (options_prefix,"Concurrency"),"HasArg"),
+ ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
+ ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
+ ]
+
+ args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments,sys.argv)
+
+ num_threads = 1
+
+ if len(args) > 0:
+ usage()
+
+ if cnf.has_key("%s::%s" % (options_prefix,"Help")):
+ usage()
+
+ level=logging.INFO
+ if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
+ level=logging.ERROR
+
+ elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
+ level=logging.DEBUG
+
+
+ logging.basicConfig( level=level,
+ format='%(asctime)s %(levelname)s %(message)s',
+ stream = sys.stderr )
+
+ if Config().has_key( "%s::%s" %(options_prefix,"Concurrency")):
+ num_threads = int(Config()[ "%s::%s" %(options_prefix,"Concurrency")])
+
+ ImportKnownChanges(num_threads)
+
+
+
+
+if __name__ == '__main__':
+ main()
################################################################################
-def Dict(**dict): return dict
-
-################################################################################
-
def usage (exit_code=0):
print """Usage: dak make-suite-file-list [OPTION]
Write out file lists suitable for use with apt-ftparchive.
################################################################################
def do_da_do_da():
+ cnf = Config()
+
# If we're only doing a subset of suites, ensure we do enough to
# be able to do arch: all mapping.
if Options["Suite"]:
# 'id' comes from either 'binaries' or 'source', so it's not unique
unique_id += 1
- packages[unique_id] = Dict(sourceid=sourceid, pkg=pkg, arch=arch, version=version,
+ packages[unique_id] = dict(sourceid=sourceid, pkg=pkg, arch=arch, version=version,
path=path, filename=filename,
component=component, file_id=file_id,
suite=suite, filetype = filetype)
svs = srcverarches.keys()
svs.sort()
for sv in svs:
- as = srcverarches[sv].keys()
- as.sort()
- print " %s (%s)" % (sv, ", ".join(as))
+ as_ = srcverarches[sv].keys()
+ as_.sort()
+ print " %s (%s)" % (sv, ", ".join(as_))
def prompt(opts, default):
p = ""
<th align="center">Closes</th>
</tr>
"""
- return res
def table_footer():
return '</table><br/><p>non-NEW uploads are <a href="/deferred/">available</a>, see the <a href="ftp://ftp-master.debian.org/pub/UploadQueue/README">UploadQueue-README</a> for more information.</p></center><br/>\n'
################################################################################
Cnf = None
-required_database_schema = 19
+required_database_schema = 20
################################################################################
except:
print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
+ result = True
+
except:
traceback.print_exc()
result = False
os.chdir(cwd)
+ return result
+
__all__.append('Binary')
def copy_temporary_contents(package, version, archname, deb, reject, session=None):
import os
import stat
-import time
import datetime
from cPickle import Unpickler, Pickler
session.commit()
session.close()
- def add_known_changes(self, queue, session=None):
- cnf = Config()
+ def mark_missing_fields(self):
+ """add "missing" in fields which we will require for the known_changes table"""
+ for key in ['urgency', 'maintainer', 'fingerprint', 'changed-by' ]:
+ if (not self.changes.has_key(key)) or (not self.changes[key]):
+ self.changes[key]='missing'
+
+ def add_known_changes(self, dirpath, session=None):
+ """add "missing" in fields which we will require for the known_changes table"""
+ cnf = Config()
+ privatetrans = False
if session is None:
session = DBConn().session()
privatetrans = True
- dirpath = cnf["Dir::Queue::%s" % (queue) ]
changesfile = os.path.join(dirpath, self.changes_file)
filetime = datetime.datetime.fromtimestamp(os.path.getctime(changesfile))
+ self.mark_missing_fields()
+
session.execute(
"""INSERT INTO known_changes
(changesname, seen, source, binaries, architecture, version,
distribution, urgency, maintainer, fingerprint, changedby, date)
VALUES (:changesfile,:filetime,:source,:binary, :architecture,
:version,:distribution,:urgency,:maintainer,:fingerprint,:changedby,:date)""",
- { 'changesfile':changesfile,
+ { 'changesfile':self.changes_file,
'filetime':filetime,
'source':self.changes["source"],
'binary':self.changes["binary"],
sa_major_version = sqlalchemy.__version__[0:3]
if sa_major_version == "0.5":
- from sqlalchemy.databases import postgres
- postgres.ischema_names['debversion'] = DebVersion
+ from sqlalchemy.databases import postgres
+ postgres.ischema_names['debversion'] = DebVersion
else:
- raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
+ raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py")
################################################################################
__all__.append('get_suites_binary_in')
@session_wrapper
-def get_binary_from_id(id, session=None):
+def get_binary_from_id(binary_id, session=None):
"""
Returns DBBinary object for given C{id}
- @type id: int
- @param id: Id of the required binary
+ @type binary_id: int
+ @param binary_id: Id of the required binary
@type session: Session
@param session: Optional SQLA session object (a temporary one will be
@return: DBBinary object for the given binary (None if not present)
"""
- q = session.query(DBBinary).filter_by(binary_id=id)
+ q = session.query(DBBinary).filter_by(binary_id=binary_id)
try:
return q.one()
ret = (False, None)
else:
obj = q.one()
- if obj.md5sum != md5sum or obj.filesize != filesize:
+ if obj.md5sum != md5sum or obj.filesize != int(filesize):
ret = (False, obj)
if ret is None:
def __repr__(self):
return '<Keyring %s>' % self.keyring_name
- def de_escape_gpg_str(self, str):
- esclist = re.split(r'(\\x..)', str)
+ def de_escape_gpg_str(self, txt):
+ esclist = re.split(r'(\\x..)', txt)
for x in range(1,len(esclist),2):
esclist[x] = "%c" % (int(esclist[x][2:],16))
return "".join(esclist)
# Insert paths
pathcache = {}
for fullpath in fullpaths:
- (path, file) = os.path.split(fullpath)
+ (path, filename) = os.path.split(fullpath)
if path.startswith( "./" ):
path = path[2:]
filepath_id = get_or_set_contents_path_id(path, session)
- filename_id = get_or_set_contents_file_id(file, session)
+ filename_id = get_or_set_contents_file_id(filename, session)
pathcache[fullpath] = (filepath_id, filename_id)
primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id))))
mapper(Suite, self.tbl_suite,
- properties = dict(suite_id = self.tbl_suite.c.id))
+ properties = dict(suite_id = self.tbl_suite.c.id,
+ policy_queue = relation(Queue)))
mapper(SuiteArchitecture, self.tbl_suite_architectures,
properties = dict(suite_id = self.tbl_suite_architectures.c.suite,
+++ /dev/null
-#!/usr/bin/env python
-
-"""
-Utility functions for extensions
-
-@contact: Debian FTP Master <ftpmaster@debian.org>
-@copyright: 2008 Anthony Towns <ajt@dbeian.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-dak_functions_to_replace = {}
-dak_replaced_functions = {}
-
-def replace_dak_function(module, name):
- """
- Decorator to make a function replace a standard dak function
- in a given module.
-
- @type module: string
- @param module: name of module where replaced function is in
-
- @type name: string
- @param name: name of the function to replace
- """
-
- def x(f):
- def myfunc(*a,**kw):
- global replaced_funcs
- f(dak_replaced_functions[name], *a, **kw)
- myfunc.__name__ = f.__name__
- myfunc.__doc__ = f.__doc__
- myfunc.__dict__.update(f.__dict__)
-
- fnname = "%s:%s" % (module, name)
- if fnname in dak_functions_to_replace:
- raise Exception, \
- "%s in %s already marked to be replaced" % (name, module)
- dak_functions_to_replace["%s:%s" % (module,name)] = myfunc
- return f
- return x
-
-################################################################################
-
-def init(name, module, userext):
- global dak_replaced_functions
-
- # This bit should be done automatically too
- dak_replaced_functions = {}
- for f,newfunc in dak_functions_to_replace.iteritems():
- m,f = f.split(":",1)
- if len(f) > 0 and m == name:
- dak_replaced_functions[f] = module.__dict__[f]
- module.__dict__[f] = newfunc
import errno
import os
-import pg
import stat
import sys
import time
import commands
import shutil
import textwrap
-import tempfile
from types import *
import yaml
# If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
if not self.pkg.changes.has_key("architecture") or not \
- isinstance(self.pkg.changes["architecture"], DictType):
+ isinstance(self.pkg.changes["architecture"], dict):
self.pkg.changes["architecture"] = { "Unknown" : "" }
# and maintainer2047 may not exist.
fix_maintainer (self.pkg.changes["maintainer"])
except ParseMaintError, msg:
self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
- % (filename, changes["maintainer"], msg))
+ % (filename, self.pkg.changes["maintainer"], msg))
# ...likewise for the Changed-By: field if it exists.
try:
# Validate the component
if not get_component(entry["component"], session):
- self.rejects.append("file '%s' has unknown component '%s'." % (f, component))
+ self.rejects.append("file '%s' has unknown component '%s'." % (f, entry["component"]))
return
# See if the package is NEW
location = cnf["Dir::Pool"]
l = get_location(location, entry["component"], archive, session)
if l is None:
- self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
+ self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (entry["component"], archive))
entry["location id"] = -1
else:
entry["location id"] = l.location_id
rej = False
for f in self.pkg.files.keys():
if self.pkg.files[f].has_key("byhand"):
- self.rejects.append("%s may not upload BYHAND file %s" % (uid, f))
+ self.rejects.append("%s may not upload BYHAND file %s" % (fpr.uid.uid, f))
rej = True
if self.pkg.files[f].has_key("new"):
- self.rejects.append("%s may not upload NEW file %s" % (uid, f))
+ self.rejects.append("%s may not upload NEW file %s" % (fpr.uid.uid, f))
rej = True
if rej:
###########################################################################
- def remove(self, dir=None):
+ def remove(self, from_dir=None):
"""
Used (for instance) in p-u to remove the package from unchecked
"""
- if dir is None:
+ if from_dir is None:
os.chdir(self.pkg.directory)
else:
- os.chdir(dir)
+ os.chdir(from_dir)
for f in self.pkg.files.keys():
os.unlink(f)
return 0
################################################################################
- def in_override_p(self, package, component, suite, binary_type, file, session):
+ def in_override_p(self, package, component, suite, binary_type, filename, session):
"""
Check if a package already has override entries in the DB
@type binary_type: string
@param binary_type: type of the package
- @type file: string
- @param file: filename we check
+ @type filename: string
+ @param filename: filename we check
@return: the database result. But noone cares anyway.
# Remember the section and priority so we can check them later if appropriate
if len(result) > 0:
result = result[0]
- self.pkg.files[file]["override section"] = result.section.section
- self.pkg.files[file]["override priority"] = result.priority.priority
+ self.pkg.files[filename]["override section"] = result.section.section
+ self.pkg.files[filename]["override priority"] = result.priority.priority
return result
return None
################################################################################
- def cross_suite_version_check(self, sv_list, file, new_version, sourceful=False):
+ def cross_suite_version_check(self, sv_list, filename, new_version, sourceful=False):
"""
@type sv_list: list
@param sv_list: list of (suite, version) tuples to check
- @type file: string
- @param file: XXX
+ @type filename: string
+ @param filename: XXX
@type new_version: string
@param new_version: XXX
vercmp = apt_pkg.VersionCompare(new_version, existent_version)
if suite in must_be_newer_than and sourceful and vercmp < 1:
- self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.rejects.append("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
if suite in must_be_older_than and vercmp > -1:
cansave = 0
self.rejects.append("Won't propogate NEW packages.")
elif apt_pkg.VersionCompare(new_version, add_version) < 0:
# propogation would be redundant. no need to reject though.
- self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.warnings.append("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
cansave = 1
elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
apt_pkg.VersionCompare(add_version, target_version) >= 0:
cansave = 1
if not cansave:
- self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
+ self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
################################################################################
- def check_binary_against_db(self, file, session):
+ def check_binary_against_db(self, filename, session):
# Ensure version is sane
q = session.query(BinAssociation)
- q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[file]["package"])
- q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
+ q = q.join(DBBinary).filter(DBBinary.package==self.pkg.files[filename]["package"])
+ q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[filename]["architecture"], 'all']))
self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
- file, self.pkg.files[file]["version"], sourceful=False)
+ filename, self.pkg.files[filename]["version"], sourceful=False)
# Check for any existing copies of the file
- q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
- q = q.filter_by(version=self.pkg.files[file]["version"])
- q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
+ q = session.query(DBBinary).filter_by(package=self.pkg.files[filename]["package"])
+ q = q.filter_by(version=self.pkg.files[filename]["version"])
+ q = q.join(Architecture).filter_by(arch_string=self.pkg.files[filename]["architecture"])
if q.count() > 0:
- self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
+ self.rejects.append("%s: can not overwrite existing copy already in the archive." % filename)
################################################################################
- def check_source_against_db(self, file, session):
+ def check_source_against_db(self, filename, session):
"""
"""
source = self.pkg.dsc.get("source")
q = q.join(DBSource).filter(DBSource.source==source)
self.cross_suite_version_check([ (x.suite.suite_name, x.source.version) for x in q.all() ],
- file, version, sourceful=True)
+ filename, version, sourceful=True)
################################################################################
- def check_dsc_against_db(self, file, session):
+ def check_dsc_against_db(self, filename, session):
"""
@warning: NB: this function can remove entries from the 'files' index [if
orig_files[dsc_name]["path"] = in_otherdir
if not found:
- self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_name))
+ self.rejects.append("%s refers to %s, but I can't find it in the queue or in the pool." % (filename, dsc_name))
continue
else:
- self.rejects.append("%s refers to %s, but I can't find it in the queue." % (file, dsc_name))
+ self.rejects.append("%s refers to %s, but I can't find it in the queue." % (filename, dsc_name))
continue
if actual_md5 != dsc_entry["md5sum"]:
- self.rejects.append("md5sum for %s doesn't match %s." % (found, file))
+ self.rejects.append("md5sum for %s doesn't match %s." % (found, filename))
if actual_size != int(dsc_entry["size"]):
- self.rejects.append("size for %s doesn't match %s." % (found, file))
+ self.rejects.append("size for %s doesn't match %s." % (found, filename))
################################################################################
# This is used by process-new and process-holding to recheck a changes file
import apt_pkg
import time
import re
-import string
import email as modemail
import subprocess
from textutils import fix_maintainer
from regexes import re_html_escaping, html_escaping, re_single_line_field, \
re_multi_line_field, re_srchasver, re_taint_free, \
- re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \
- re_is_orig_source
+ re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource
from formats import parse_format, validate_changes_format
from srcformats import get_format_from_string
known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
-# Monkeypatch commands.getstatusoutput as it returns a "0" exit code in
-# all situations under lenny's Python.
-import commands
+# Monkeypatch commands.getstatusoutput as it may not return the correct exit
+# code in lenny's Python. This also affects commands.getoutput and
+# commands.getstatus.
def dak_getstatusoutput(cmd):
pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
(section, component) = extract_component_from_section(section)
- files[name] = Dict(size=size, section=section,
+ files[name] = dict(size=size, section=section,
priority=priority, component=component)
files[name][hashname] = md5
if len(match) == 0:
del message_raw[field]
else:
- message_raw.replace_header(field, string.join(match, ", "))
+ message_raw.replace_header(field, ', '.join(match))
# Change message fields in order if we don't have a To header
if not message_raw.has_key("To"):
################################################################################
-def TemplateSubst(map, filename):
+def TemplateSubst(subst_map, filename):
""" Perform a substition of template """
templatefile = open_file(filename)
template = templatefile.read()
- for x in map.keys():
- template = template.replace(x, str(map[x]))
+ for k, v in subst_map.iteritems():
+ template = template.replace(k, str(v))
templatefile.close()
return template
################################################################################
-def Dict(**dict): return dict
-
-########################################
-
def gpgv_get_status_output(cmd, status_read, status_write):
"""
Our very own version of commands.getouputstatus(), hacked to support
rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
# Finally ensure there's not something we don't recognise
- known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
+ known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="")
################################################################################
-def get_changes_files(dir):
+def get_changes_files(from_dir):
"""
Takes a directory and lists all .changes files in it (as well as chdir'ing
to the directory; this is due to broken behaviour on the part of p-u/p-a
"""
try:
# Much of the rest of p-u/p-a depends on being in the right place
- os.chdir(dir)
- changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')]
+ os.chdir(from_dir)
+ changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')]
except OSError, e:
- fubar("Failed to read list from directory %s (%s)" % (dir, e))
+ fubar("Failed to read list from directory %s (%s)" % (from_dir, e))
return changes_files
projects / dak.git / commitdiff