]> git.decadent.org.uk Git - dak.git/commitdiff
Start the known_changes table
authorJoerg Jaspert <joerg@debian.org>
Thu, 29 Oct 2009 16:41:31 +0000 (17:41 +0100)
committerJoerg Jaspert <joerg@debian.org>
Thu, 29 Oct 2009 16:41:31 +0000 (17:41 +0100)
Signed-off-by: Joerg Jaspert <joerg@debian.org>
dak/dakdb/update20.py [new file with mode: 0755]
daklib/dbconn.py
daklib/queue.py

diff --git a/dak/dakdb/update20.py b/dak/dakdb/update20.py
new file mode 100755 (executable)
index 0000000..a666326
--- /dev/null
@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Adding table to get rid of queue/done checks
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009  Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+import os
+import datetime
+from daklib.dak_exceptions import DBUpdateError, InvalidDscError, ChangesUnicodeError
+from daklib.config import Config
+from daklib.utils import parse_changes, warn, gpgv_get_status_output, process_gpgv_output
+
+################################################################################
+
+def check_signature (sig_filename, data_filename=""):
+    keyrings = [
+        "/home/joerg/keyring/keyrings/debian-keyring.gpg",
+        "/home/joerg/keyring/keyrings/debian-keyring.pgp",
+        "/home/joerg/keyring/keyrings/debian-maintainers.gpg",
+        "/home/joerg/keyring/keyrings/debian-role-keys.gpg",
+        "/home/joerg/keyring/keyrings/emeritus-keyring.pgp",
+        "/home/joerg/keyring/keyrings/emeritus-keyring.gpg",
+        "/home/joerg/keyring/keyrings/removed-keys.gpg",
+        "/home/joerg/keyring/keyrings/removed-keys.pgp"
+        ]
+
+    keyringargs = " ".join(["--keyring %s" % x for x in keyrings ])
+
+    # Build the command line
+    status_read, status_write = os.pipe()
+    cmd = "gpgv --status-fd %s %s %s" % (status_write, keyringargs, sig_filename)
+
+    # Invoke gpgv on the file
+    (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
+
+    # Process the status-fd output
+    (keywords, internal_error) = process_gpgv_output(status)
+
+    # If we failed to parse the status-fd output, let's just whine and bail now
+    if internal_error:
+        warn("Couldn't parse signature")
+        return (None, rejects)
+
+    # usually one would check for bad things here. We, however, do not care.
+
+    # Next check gpgv exited with a zero return code
+    if exit_status:
+        warn("Couldn't parse signature")
+        return (None, rejects)
+
+    # Sanity check the good stuff we expect
+    if not keywords.has_key("VALIDSIG"):
+        warn("Couldn't parse signature")
+    else:
+        args = keywords["VALIDSIG"]
+        if len(args) < 1:
+            warn("Couldn't parse signature")
+        else:
+            fingerprint = args[0]
+
+    return (fingerprint, [])
+
+################################################################################
+
+def do_update(self):
+    print "Adding known_changes table"
+
+    try:
+        c = self.db.cursor()
+        c.execute("""
+                    CREATE TABLE known_changes (
+                    id SERIAL PRIMARY KEY,
+                    changesname TEXT NOT NULL,
+                    seen TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+                    source TEXT NOT NULL,
+                    binaries TEXT NOT NULL,
+                    architecture TEXT NOT NULL,
+                    version TEXT NOT NULL,
+                    distribution TEXT NOT NULL,
+                    urgency TEXT NOT NULL,
+                    maintainer TEXT NOT NULL,
+                    changedby TEXT NOT NULL,
+                    date TEXT NOT NULL,
+                    UNIQUE (changesname)
+            )
+        """)
+        c.execute("CREATE INDEX changesname_ind ON known_changes(changesname)")
+        c.execute("CREATE INDEX changestimestamp_ind ON known_changes(seen)")
+        c.execute("CREATE INDEX changessource_ind ON known_changes(source)")
+        c.execute("CREATE INDEX changesdistribution_ind ON known_changes(distribution)")
+        c.execute("CREATE INDEX changesurgency_ind ON known_changes(urgency)")
+
+        print "Done. Now looking for old changes files"
+        count = 0
+        failure = 0
+        cnf = Config()
+        for directory in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
+            checkdir = cnf["Dir::Queue::%s" % (directory) ]
+            if os.path.exists(checkdir):
+                print "Looking into %s" % (checkdir)
+                for dirpath, dirnames, filenames in os.walk(checkdir, topdown=False):
+                    if not filenames:
+                        # Empty directory (or only subdirectories), next
+                        continue
+                    for changesfile in filenames:
+                        if not changesfile.endswith(".changes"):
+                            # Only interested in changes files.
+                            continue
+                        try:
+                            count += 1
+                            print "Directory %s, file %7d, failures %3d. (%s)" % (dirpath[-10:], count, failure, changesfile)
+                            changes = parse_changes(os.path.join(dirpath, changesfile), signing_rules=-1)
+                            (changes["fingerprint"], _) = check_signature(os.path.join(dirpath, changesfile))
+                        except InvalidDscError, line:
+                            warn("syntax error in .dsc file '%s', line %s." % (f, line))
+                            failure += 1
+                        except ChangesUnicodeError:
+                            warn("found invalid changes file, not properly utf-8 encoded")
+                            failure += 1
+
+                        filetime = datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, changesfile)))
+                        c.execute("INSERT INTO known_changes(changesname, seen, source, binaries, architecture, version, distribution, urgency, maintainer, changedby, date) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" , [changesfile, filetime, changes["source"], changes["binary"], changes["architecture"], changes["version"], changes["distribution"], changes["urgency"], changes["maintainer"], changes["changed-by"], changes["date"]])
+
+        c.execute("UPDATE config SET value = '20' WHERE name = 'db_revision'")
+        self.db.commit()
+
+    except psycopg2.ProgrammingError, msg:
+        self.db.rollback()
+        raise DBUpdateError, "Unable to apply source format update 15, rollback issued. Error message : %s" % (str(msg))
index 6d5497fc2d5f4b096d972631637e284eb2ce00aa..2b0a849254d928bc0d134d294d77a13f98ebf71c 100755 (executable)
@@ -901,6 +901,41 @@ __all__.append('get_or_set_keyring')
 
 ################################################################################
 
+class KnownChange(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __repr__(self):
+        return '<KnownChange %s>' % self.changesname
+
+__all__.append('KnownChange')
+
+@session_wrapper
+def get_knownchange(filename, session=None):
+    """
+    returns knownchange object for given C{filename}.
+
+    @type archive: string
+    @param archive: the name of the arhive
+
+    @type session: Session
+    @param session: Optional SQLA session object (a temporary one will be
+    generated if not supplied)
+
+    @rtype: Archive
+    @return: Archive object for the given name (None if not present)
+
+    """
+    q = session.query(KnownChange).filter_by(changesname=filename)
+
+    try:
+        return q.one()
+    except NoResultFound:
+        return None
+
+__all__.append('get_knownchange')
+
+################################################################################
 class Location(object):
     def __init__(self, *args, **kwargs):
         pass
@@ -2116,6 +2151,7 @@ class DBConn(Singleton):
         self.tbl_files = Table('files', self.db_meta, autoload=True)
         self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True)
         self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True)
+        self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True)
         self.tbl_location = Table('location', self.db_meta, autoload=True)
         self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True)
         self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True)
@@ -2218,6 +2254,9 @@ class DBConn(Singleton):
                properties = dict(keyring_name = self.tbl_keyrings.c.name,
                                  keyring_id = self.tbl_keyrings.c.id))
 
+        mapper(KnownChange, self.tbl_known_changes,
+               properties = dict(known_change_id = self.tbl_known_changes.c.id))
+
         mapper(Location, self.tbl_location,
                properties = dict(location_id = self.tbl_location.c.id,
                                  component_id = self.tbl_location.c.component,
index d73b4797732bc278a64996661d9c1934b6783b92..7e7f468fea248d5b402a8c492c7d1c8c4017bf1a 100755 (executable)
@@ -452,9 +452,8 @@ class Upload(object):
         # Check there isn't already a changes file of the same name in one
         # of the queue directories.
         base_filename = os.path.basename(filename)
-        for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]:
-            if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)):
-                self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d))
+        if get_knownchange(base_filename):
+            self.rejects.append("%s: a file with this name already exists." % (base_filename))
 
         # Check the .changes is non-empty
         if not self.pkg.files: