From: Joerg Jaspert Date: Fri, 30 Oct 2009 14:14:49 +0000 (+0100) Subject: Merge commit 'stew/knownchanges' into merge X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=a286482465b9952a951ba8d4954a4ccfed4b3fef;hp=1cbe83ac12a34c4969b8725c54423db3d8170577;p=dak.git Merge commit 'stew/knownchanges' into merge * commit 'stew/knownchanges': towpdown import_known_changes Signed-off-by: Joerg Jaspert --- diff --git a/dak/add_user.py b/dak/add_user.py index 77de3e3f..28d31208 100755 --- a/dak/add_user.py +++ b/dak/add_user.py @@ -46,28 +46,28 @@ Adds a new user to the dak databases and keyrings # Stolen from userdir-ldap # Compute a random password using /dev/urandom. def GenPass(): - # Generate a 10 character random string - SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/." - Rand = open("/dev/urandom") - Password = "" - for i in range(0,15): - Password = Password + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)] - return Password + # Generate a 10 character random string + SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/." + Rand = open("/dev/urandom") + Password = "" + for i in range(0,15): + Password = Password + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)] + return Password # Compute the MD5 crypted version of the given password def HashPass(Password): - import crypt - # Hash it telling glibc to use the MD5 algorithm - if you dont have - # glibc then just change Salt = "$1$" to Salt = "" - SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/." - Salt = "$1$" - Rand = open("/dev/urandom") - for x in range(0,10): - Salt = Salt + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)] - Pass = crypt.crypt(Password,Salt) - if len(Pass) < 14: - raise "Password Error", "MD5 password hashing failed, not changing the password!" - return Pass + import crypt + # Hash it telling glibc to use the MD5 algorithm - if you dont have + # glibc then just change Salt = "$1$" to Salt = "" + SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/." + Salt = "$1$" + Rand = open("/dev/urandom") + for x in range(0,10): + Salt = Salt + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)] + Pass = crypt.crypt(Password,Salt) + if len(Pass) < 14: + raise "Password Error", "MD5 password hashing failed, not changing the password!" + return Pass ################################################################################ @@ -112,8 +112,8 @@ def main(): ] for i in [ "help", "create" ]: - if not Cnf.has_key("Add-User::Options::%s" % (i)): - Cnf["Add-User::Options::%s" % (i)] = "" + if not Cnf.has_key("Add-User::Options::%s" % (i)): + Cnf["Add-User::Options::%s" % (i)] = "" apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) @@ -138,10 +138,10 @@ def main(): (result, output) = commands.getstatusoutput(cmd) m = re_gpg_fingerprint.search(output) if not m: - print output + print output utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s" \ - % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, \ - " [GPG output:] "))) + % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, \ + " [GPG output:] "))) primary_key = m.group(1) primary_key = primary_key.replace(" ","") @@ -174,70 +174,69 @@ def main(): yn = utils.our_raw_input(prompt).lower() if yn == "y": -# Create an account for the user? - summary = "" - if Cnf.FindB("Add-User::CreateAccount") or Cnf["Add-User::Options::Create"]: - password = GenPass() - pwcrypt = HashPass(password) - if Cnf.has_key("Add-User::GID"): - cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' -G %s %s" \ - % (pwcrypt, name, Cnf["Add-User::GID"], uid) - else: - cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' %s" \ - % (pwcrypt, name, uid) - (result, output) = commands.getstatusoutput(cmd) - if (result != 0): - utils.fubar("Invocation of '%s' failed:\n%s\n" % (cmd, output), result) - try: - summary+=createMail(uid, password, Cnf["Add-User::Options::Key"], Cnf["Dinstall::GPGKeyring"]) - except: - summary="" - utils.warn("Could not prepare password information for mail, not sending password.") - -# Now add user to the database. - # Note that we provide a session, so we're responsible for committing - uidobj = get_or_set_uid(uid, session=session) - uid_id = uidobj.uid_id - add_database_user(uid) - session.commit() -# The following two are kicked out in rhona, so we don't set them. kelly adds -# them as soon as she installs a package with unknown ones, so no problems to expect here. -# Just leave the comment in, to not think about "Why the hell aren't they added" in -# a year, if we ever touch uma again. -# maint_id = database.get_or_set_maintainer_id(name) -# session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)", -# {'fingerprint': primary_key, 'uid': uid_id}) - -# Lets add user to the email-whitelist file if its configured. - if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "": - file = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a") - for mail in emails: - file.write(mail+'\n') - file.close() - - print "Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, \ - name, primary_key) - -# Should we send mail to the newly added user? - if Cnf.FindB("Add-User::SendEmail"): - mail = name + "<" + emails[0] +">" - Subst = {} - Subst["__NEW_MAINTAINER__"] = mail - Subst["__UID__"] = uid - Subst["__KEYID__"] = Cnf["Add-User::Options::Key"] - Subst["__PRIMARY_KEY__"] = primary_key - Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] - Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"] - Subst["__SUMMARY__"] = summary - new_add_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/add-user.added") - utils.send_mail(new_add_message) + # Create an account for the user? + summary = "" + if Cnf.FindB("Add-User::CreateAccount") or Cnf["Add-User::Options::Create"]: + password = GenPass() + pwcrypt = HashPass(password) + if Cnf.has_key("Add-User::GID"): + cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' -G %s %s" \ + % (pwcrypt, name, Cnf["Add-User::GID"], uid) + else: + cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' %s" \ + % (pwcrypt, name, uid) + (result, output) = commands.getstatusoutput(cmd) + if (result != 0): + utils.fubar("Invocation of '%s' failed:\n%s\n" % (cmd, output), result) + try: + summary+=createMail(uid, password, Cnf["Add-User::Options::Key"], Cnf["Dinstall::GPGKeyring"]) + except: + summary="" + utils.warn("Could not prepare password information for mail, not sending password.") + + # Now add user to the database. + # Note that we provide a session, so we're responsible for committing + uidobj = get_or_set_uid(uid, session=session) + uid_id = uidobj.uid_id + add_database_user(uid) + session.commit() + + # The following two are kicked out in rhona, so we don't set them. kelly adds + # them as soon as she installs a package with unknown ones, so no problems to expect here. + # Just leave the comment in, to not think about "Why the hell aren't they added" in + # a year, if we ever touch uma again. + # maint_id = database.get_or_set_maintainer_id(name) + # session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)", + # {'fingerprint': primary_key, 'uid': uid_id}) + + # Lets add user to the email-whitelist file if its configured. + if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "": + f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a") + for mail in emails: + f.write(mail+'\n') + f.close() + + print "Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, \ + name, primary_key) + + # Should we send mail to the newly added user? + if Cnf.FindB("Add-User::SendEmail"): + mail = name + "<" + emails[0] +">" + Subst = {} + Subst["__NEW_MAINTAINER__"] = mail + Subst["__UID__"] = uid + Subst["__KEYID__"] = Cnf["Add-User::Options::Key"] + Subst["__PRIMARY_KEY__"] = primary_key + Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] + Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"] + Subst["__SUMMARY__"] = summary + new_add_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/add-user.added") + utils.send_mail(new_add_message) else: - uid = None - + uid = None ####################################################################################### if __name__ == '__main__': main() - diff --git a/dak/check_archive.py b/dak/check_archive.py index 2162068e..b9837d30 100755 --- a/dak/check_archive.py +++ b/dak/check_archive.py @@ -185,8 +185,8 @@ def check_override(): print suite_name print "-" * len(suite_name) print - suite = get_suite(suite) - q = s.execute(""" + suite = get_suite(suite_name) + q = session.execute(""" SELECT DISTINCT b.package FROM binaries b, bin_associations ba WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS (SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)""" @@ -195,7 +195,7 @@ SELECT DISTINCT b.package FROM binaries b, bin_associations ba for j in q.fetchall(): print j[0] - q = s.execute(""" + q = session.execute(""" SELECT DISTINCT s.source FROM source s, src_associations sa WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS (SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)""" @@ -427,8 +427,8 @@ def check_indices_files_exist(): """ for suite in [ "stable", "testing", "unstable" ]: for component in Cnf.ValueList("Suite::%s::Components" % (suite)): - architectures = database.get_suite_architectures(suite) - for arch in [ i.lower() for i in architectures ]: + architectures = get_suite_architectures(suite) + for arch in [ i.arch_string.lower() for i in architectures ]: if arch == "source": validate_sources(suite, component) elif arch == "all": @@ -475,6 +475,7 @@ def chk_bd_process_dir (unused, dirname, filenames): def check_build_depends(): """ Validate build-dependencies of .dsc files in the archive """ + cnf = Config() os.path.walk(cnf["Dir::Root"], chk_bd_process_dir, None) ################################################################################ diff --git a/dak/contents.py b/dak/contents.py index 834cbccf..58c3aa6b 100755 --- a/dak/contents.py +++ b/dak/contents.py @@ -39,6 +39,7 @@ import os import logging import gzip import threading +import traceback import Queue import apt_pkg from daklib import utils diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 0a1534c5..4541bf6e 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -92,8 +92,9 @@ def do_anais(architecture, binaries_list, source, session): WHERE ba.suite = :suiteid AND ba.bin = b.id AND b.architecture = a.id AND b.package = :package""", {'suiteid': suite_id, 'package': binary}) + ql = q.fetchall() versions = [] - for i in q.fetchall(): + for i in ql: arch = i[0] version = i[1] if architectures.has_key(arch): diff --git a/dak/dakdb/update20.py b/dak/dakdb/update20.py new file mode 100755 index 00000000..f4e34cb9 --- /dev/null +++ b/dak/dakdb/update20.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add policy queue handling support + +@contact: Debian FTP Master +@copyright: 2009 Mark Hymers +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + + +################################################################################ + +import psycopg2 +import time +import os +import datetime +import traceback + +from daklib.dak_exceptions import DBUpdateError +from daklib.config import Config + +################################################################################ + +def do_update(self): + print "Updating use of queue table" + + try: + c = self.db.cursor() + + cnf = Config() + + print "Adding path to queue table" + c.execute("ALTER TABLE queue ADD COLUMN path TEXT") + c.execute("SELECT * FROM queue") + rows = c.fetchall() + seenqueues = {} + for row in rows: + dir = cnf["Dir::Queue::%s" % row[1]].rstrip('/') + seenqueues[row[1].lower()] = 1 + print "Setting %s queue to use path %s" % (row[1], dir) + c.execute("UPDATE queue SET path = %s WHERE id = %s", (dir, row[0])) + + print "Adding missing queues to the queue table" + for q in cnf.SubTree("Dir::Queue").keys(): + qname = q.lower() + if qname in seenqueues.keys(): + continue + if qname in ["done", "holding", "reject", "newstage", "btsversiontrack"]: + print "Skipping queue %s" % qname + continue + pth = cnf["Dir::Queue::%s" % qname].rstrip('/') + if not os.path.exists(pth): + print "Skipping %s as %s does not exist" % (qname, pth) + continue + + print "Adding %s queue with path %s" % (qname, pth) + c.execute("INSERT INTO queue (queue_name, path) VALUES (%s, %s)", (qname, pth)) + seenqueues[qname] = 1 + + print "Adding queue and approved_for columns to known_changes" + c.execute("ALTER TABLE known_changes ADD COLUMN in_queue INT4 REFERENCES queue(id) DEFAULT NULL") + c.execute("ALTER TABLE known_changes ADD COLUMN approved_for INT4 REFERENCES queue(id) DEFAULT NULL") + + print "Adding policy queue column to suite table" + c.execute("ALTER TABLE suite DROP COLUMN policy_engine") + c.execute("ALTER TABLE suite ADD COLUMN policy_queue_id INT4 REFERENCES queue(id) DEFAULT NULL") + # Handle some of our common cases automatically + if seenqueues.has_key('proposedupdates'): + c.execute("""UPDATE suite SET policy_queue_id = (SELECT id FROM queue WHERE queue_name = 'proposedupdates') + WHERE suite_name = 'proposed-updates'""") + + if seenqueues.has_key('oldproposedupdates'): + c.execute("""UPDATE suite SET policy_queue_id = (SELECT id FROM queue WHERE queue_name = 'oldproposedupdates') + WHERE suite_name = 'oldstable-proposed-updates'""") + + print "Committing" + c.execute("UPDATE config SET value = '20' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.InternalError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply debversion update 20, rollback issued. Error message : %s" % (str(msg)) diff --git a/dak/generate_releases.py b/dak/generate_releases.py index 9de4614d..31cae490 100755 --- a/dak/generate_releases.py +++ b/dak/generate_releases.py @@ -1,9 +1,12 @@ #!/usr/bin/env python -""" Create all the Release files """ - -# Copyright (C) 2001, 2002, 2006 Anthony Towns +""" Create all the Release files +@contact: Debian FTPMaster +@Copyright: 2001, 2002, 2006 Anthony Towns +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -22,8 +25,12 @@ ################################################################################ -import sys, os, stat, time -import gzip, bz2 +import sys +import os +import stat +import time +import gzip +import bz2 import apt_pkg from daklib import utils @@ -372,13 +379,21 @@ def main (): dest = Cnf["Dir::Root"] + tree + "/Release.gpg" if os.path.exists(dest): os.unlink(dest) + inlinedest = Cnf["Dir::Root"] + tree + "/InRelease" + if os.path.exists(inlinedest): + os.unlink(inlinedest) for keyid in signkeyids: - if keyid != "": defkeyid = "--default-key %s" % keyid - else: defkeyid = "" + if keyid != "": + defkeyid = "--default-key %s" % keyid + else: + defkeyid = "" os.system("gpg %s %s %s --detach-sign <%s >>%s" % (keyring, defkeyid, arguments, Cnf["Dir::Root"] + tree + "/Release", dest)) + os.system("gpg %s %s %s --clearsign <%s >>%s" % + (keyring, defkeyid, arguments, + Cnf["Dir::Root"] + tree + "/Release", inlinedest)) ####################################################################################### diff --git a/dak/make_suite_file_list.py b/dak/make_suite_file_list.py index 3c690a51..d3b679ed 100755 --- a/dak/make_suite_file_list.py +++ b/dak/make_suite_file_list.py @@ -309,6 +309,8 @@ def write_filelists(packages, dislocated_files, session): ################################################################################ def do_da_do_da(): + cnf = Config() + # If we're only doing a subset of suites, ensure we do enough to # be able to do arch: all mapping. if Options["Suite"]: diff --git a/dak/show_deferred.py b/dak/show_deferred.py index e8e1621d..2bb643d1 100755 --- a/dak/show_deferred.py +++ b/dak/show_deferred.py @@ -101,7 +101,6 @@ def table_header(): Closes """ - return res def table_footer(): return '

non-NEW uploads are available, see the UploadQueue-README for more information.


\n' diff --git a/dak/update_db.py b/dak/update_db.py index e9dfa9a7..a5273217 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -44,7 +44,7 @@ from daklib.dak_exceptions import DBUpdateError ################################################################################ Cnf = None -required_database_schema = 19 +required_database_schema = 20 ################################################################################ diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 45e000d3..75d271bd 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -2552,7 +2552,8 @@ class DBConn(Singleton): primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id)))) mapper(Suite, self.tbl_suite, - properties = dict(suite_id = self.tbl_suite.c.id)) + properties = dict(suite_id = self.tbl_suite.c.id, + policy_queue = relation(Queue))) mapper(SuiteArchitecture, self.tbl_suite_architectures, properties = dict(suite_id = self.tbl_suite_architectures.c.suite,