]> git.decadent.org.uk Git - dak.git/blobdiff - dak/process_new.py
Port bts_categorize to python-debianbts
[dak.git] / dak / process_new.py
index f89e40c8ad667ef7fa00a23292c35234340eb823..fc952d2a346d86ba60dc963e35aacb7c0f4c15d0 100755 (executable)
@@ -60,10 +60,11 @@ from daklib.dbconn import *
 from daklib.queue import *
 from daklib import daklog
 from daklib import utils
-from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
+from daklib.regexes import re_no_epoch, re_default_answer, re_isanum, re_package
 from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
 from daklib.summarystats import SummaryStats
 from daklib.config import Config
+from daklib.changesutils import *
 
 # Globals
 Options = None
@@ -77,7 +78,7 @@ Sections = None
 ################################################################################
 
 def recheck(upload, session):
-    upload.recheck()
+# STU: I'm not sure, but I don't thin kthis is necessary any longer:    upload.recheck(session)
     if len(upload.rejects) > 0:
         answer = "XXX"
         if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
@@ -95,7 +96,8 @@ def recheck(upload, session):
 
         if answer == 'R':
             upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
-            os.unlink(upload.pkg.changes_file[:-8]+".dak")
+            upload.pkg.remove_known_changes(session=session)
+            session.commit()
             return 0
         elif answer == 'S':
             return 0
@@ -107,103 +109,6 @@ def recheck(upload, session):
 
 ################################################################################
 
-def indiv_sg_compare (a, b):
-    """Sort by source name, source, version, 'have source', and
-       finally by filename."""
-    # Sort by source version
-    q = apt_pkg.VersionCompare(a["version"], b["version"])
-    if q:
-        return -q
-
-    # Sort by 'have source'
-    a_has_source = a["architecture"].get("source")
-    b_has_source = b["architecture"].get("source")
-    if a_has_source and not b_has_source:
-        return -1
-    elif b_has_source and not a_has_source:
-        return 1
-
-    return cmp(a["filename"], b["filename"])
-
-############################################################
-
-def sg_compare (a, b):
-    a = a[1]
-    b = b[1]
-    """Sort by have note, source already in database and time of oldest upload."""
-    # Sort by have note
-    a_note_state = a["note_state"]
-    b_note_state = b["note_state"]
-    if a_note_state < b_note_state:
-        return -1
-    elif a_note_state > b_note_state:
-        return 1
-    # Sort by source already in database (descending)
-    source_in_database = cmp(a["source_in_database"], b["source_in_database"])
-    if source_in_database:
-        return -source_in_database
-
-    # Sort by time of oldest upload
-    return cmp(a["oldest"], b["oldest"])
-
-def sort_changes(changes_files, session):
-    """Sort into source groups, then sort each source group by version,
-    have source, filename.  Finally, sort the source groups by have
-    note, time of oldest upload of each source upload."""
-    if len(changes_files) == 1:
-        return changes_files
-
-    sorted_list = []
-    cache = {}
-    # Read in all the .changes files
-    for filename in changes_files:
-        u = Upload()
-        try:
-            u.pkg.load_dot_dak(filename)
-            u.update_subst()
-            cache[filename] = copy.copy(u.pkg.changes)
-            cache[filename]["filename"] = filename
-        except:
-            sorted_list.append(filename)
-            break
-    # Divide the .changes into per-source groups
-    per_source = {}
-    for filename in cache.keys():
-        source = cache[filename]["source"]
-        if not per_source.has_key(source):
-            per_source[source] = {}
-            per_source[source]["list"] = []
-        per_source[source]["list"].append(cache[filename])
-    # Determine oldest time and have note status for each source group
-    for source in per_source.keys():
-        q = session.query(DBSource).filter_by(source = source).all()
-        per_source[source]["source_in_database"] = len(q)>0
-        source_list = per_source[source]["list"]
-        first = source_list[0]
-        oldest = os.stat(first["filename"])[stat.ST_MTIME]
-        have_note = 0
-        for d in per_source[source]["list"]:
-            mtime = os.stat(d["filename"])[stat.ST_MTIME]
-            if mtime < oldest:
-                oldest = mtime
-            have_note += has_new_comment(d["source"], d["version"], session)
-        per_source[source]["oldest"] = oldest
-        if not have_note:
-            per_source[source]["note_state"] = 0; # none
-        elif have_note < len(source_list):
-            per_source[source]["note_state"] = 1; # some
-        else:
-            per_source[source]["note_state"] = 2; # all
-        per_source[source]["list"].sort(indiv_sg_compare)
-    per_source_items = per_source.items()
-    per_source_items.sort(sg_compare)
-    for i in per_source_items:
-        for j in i[1]["list"]:
-            sorted_list.append(j["filename"])
-    return sorted_list
-
-################################################################################
-
 class Section_Completer:
     def __init__ (self, session):
         self.sections = []
@@ -475,17 +380,17 @@ def check_pkg (upload):
         try:
             sys.stdout = less_fd
             changes = utils.parse_changes (upload.pkg.changes_file)
-            examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+            print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
             files = upload.pkg.files
             for f in files.keys():
                 if files[f].has_key("new"):
                     ftype = files[f]["type"]
                     if ftype == "deb":
-                        examine_package.check_deb(changes['distribution'], f)
+                        print examine_package.check_deb(changes['distribution'], f)
                     elif ftype == "dsc":
-                        examine_package.check_dsc(changes['distribution'], f)
+                        print examine_package.check_dsc(changes['distribution'], f)
         finally:
-            examine_package.output_package_relations()
+            print examine_package.output_package_relations()
             sys.stdout = stdout_fd
     except IOError, e:
         if e.errno == errno.EPIPE:
@@ -543,14 +448,13 @@ def add_overrides (new, upload, session):
 
 ################################################################################
 
-def prod_maintainer (note, upload):
+def prod_maintainer (notes, upload):
     cnf = Config()
     # Here we prepare an editor and get them ready to prod...
     (fd, temp_filename) = utils.temp_filename()
     temp_file = os.fdopen(fd, 'w')
-    if len(note) > 0:
-        for line in note:
-            temp_file.write(line)
+    for note in notes:
+        temp_file.write(note.comment)
     temp_file.close()
     editor = os.environ.get("EDITOR","vi")
     answer = 'E'
@@ -588,9 +492,8 @@ def prod_maintainer (note, upload):
     prod_mail_message = utils.TemplateSubst(
         Subst,cnf["Dir::Templates"]+"/process-new.prod")
 
-    # Send the prod mail if appropriate
-    if not cnf["Dinstall::Options::No-Mail"]:
-        utils.send_mail(prod_mail_message)
+    # Send the prod mail
+    utils.send_mail(prod_mail_message)
 
     print "Sent proding message"
 
@@ -599,31 +502,16 @@ def prod_maintainer (note, upload):
 def do_new(upload, session):
     print "NEW\n"
     files = upload.pkg.files
+    upload.check_files(not Options["No-Action"])
     changes = upload.pkg.changes
     cnf = Config()
 
+    # Check for a valid distribution
+    upload.check_distributions()
+
     # Make a copy of distribution we can happily trample on
     changes["suite"] = copy.copy(changes["distribution"])
 
-    # Fix up the list of target suites
-    for suite in changes["suite"].keys():
-        override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
-        if override:
-            (olderr, newerr) = (get_suite(suite, session) == None,
-                                get_suite(override, session) == None)
-            if olderr or newerr:
-                (oinv, newinv) = ("", "")
-                if olderr: oinv = "invalid "
-                if newerr: ninv = "invalid "
-                print "warning: overriding %ssuite %s to %ssuite %s" % (
-                        oinv, suite, ninv, override)
-            del changes["suite"][suite]
-            changes["suite"][override] = 1
-    # Validate suites
-    for suite in changes["suite"].keys():
-        if get_suite(suite, session) is None:
-            utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
-
     # The main NEW processing loop
     done = 0
     while not done:
@@ -665,6 +553,7 @@ def do_new(upload, session):
             try:
                 check_daily_lock()
                 done = add_overrides (new, upload, session)
+                new_accept(upload, Options["No-Action"], session)
                 Logger.log(["NEW ACCEPT: %s" % (upload.pkg.changes_file)])
             except CantGetLockError:
                 print "Hello? Operator! Give me the number for 911!"
@@ -674,13 +563,13 @@ def do_new(upload, session):
         elif answer == 'E' and not Options["Trainee"]:
             new = edit_overrides (new, upload, session)
         elif answer == 'M' and not Options["Trainee"]:
-            upload.pkg.remove_known_changes()
             aborted = upload.do_reject(manual=1,
                                        reject_message=Options["Manual-Reject"],
-                                       note=get_new_comments(changes.get("source", ""), session=session))
+                                       notes=get_new_comments(changes.get("source", ""), session=session))
             if not aborted:
+                upload.pkg.remove_known_changes(session=session)
+                session.commit()
                 Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
-                os.unlink(upload.pkg.changes_file[:-8]+".dak")
                 done = 1
         elif answer == 'N':
             edit_note(get_new_comments(changes.get("source", ""), session=session),
@@ -769,7 +658,8 @@ def do_byhand(upload, session):
         elif answer == 'M':
             Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
             upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
-            os.unlink(upload.pkg.changes_file[:-8]+".dak")
+            upload.pkg.remove_known_changes(session=session)
+            session.commit()
             done = 1
         elif answer == 'S':
             done = 1
@@ -817,33 +707,31 @@ def lock_package(package):
     finally:
         os.unlink(path)
 
-def _accept(upload):
-    if Options["No-Action"]:
-        return
-    (summary, short_summary) = upload.build_summaries()
-    upload.accept(summary, short_summary, targetqueue)
-    os.unlink(upload.pkg.changes_file[:-8]+".dak")
+class clean_holding(object):
+    def __init__(self,pkg):
+        self.pkg = pkg
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, type, value, traceback):
+        h = Holding()
+
+        for f in self.pkg.files.keys():
+            if os.path.exists(os.path.join(h.holding_dir, f)):
+                os.unlink(os.path.join(h.holding_dir, f))
 
-def do_accept(upload):
-    print "ACCEPT"
-    cnf = Config()
-    if not Options["No-Action"]:
-        (summary, short_summary) = upload.build_summaries()
-
-        if cnf.FindB("Dinstall::SecurityQueueHandling"):
-            upload.dump_vars(cnf["Dir::Queue::Embargoed"])
-            upload.move_to_queue(get_queue('embargoed'))
-            upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
-            # Check for override disparities
-            upload.Subst["__SUMMARY__"] = summary
-        else:
-            # Just a normal upload, accept it...
-            _accept(upload)
 
 def do_pkg(changes_file, session):
+    new_queue = get_policy_queue('new', session );
     u = Upload()
-    u.pkg.load_dot_dak(changes_file)
+    u.pkg.changes_file = changes_file
+    (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
+    u.load_changes(changes_file)
+    u.pkg.directory = new_queue.path
     u.update_subst()
+    u.logger = Logger
+    origchanges = os.path.abspath(u.pkg.changes_file)
 
     cnf = Config()
     bcc = "X-DAK: dak process-new"
@@ -853,27 +741,34 @@ def do_pkg(changes_file, session):
         u.Subst["__BCC__"] = bcc
 
     files = u.pkg.files
+    for deb_filename, f in files.items():
+        if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
+            u.binary_file_checks(deb_filename, session)
+            u.check_binary_against_db(deb_filename, session)
+        else:
+            u.source_file_checks(deb_filename, session)
+            u.check_source_against_db(deb_filename, session)
+
+        u.pkg.changes["suite"] = copy.copy(u.pkg.changes["distribution"])
 
     try:
         with lock_package(u.pkg.changes["source"]):
-            if not recheck(u, session):
-                return
+            with clean_holding(u.pkg):
+                if not recheck(u, session):
+                    return
 
-            (new, byhand) = check_status(files)
-            if new or byhand:
+                # FIXME: This does need byhand checks added!
+                new = determine_new(u.pkg.changes, files)
                 if new:
                     do_new(u, session)
-                if byhand:
-                    do_byhand(u, session)
-                (new, byhand) = check_status(files)
-
-            if not new and not byhand:
-                try:
-                    check_daily_lock()
-                    do_accept(u)
-                except CantGetLockError:
-                    print "Hello? Operator! Give me the number for 911!"
-                    print "Dinstall in the locked area, cant process packages, come back later"
+                else:
+                    try:
+                        check_daily_lock()
+                        new_accept(u, Options["No-Action"], session)
+                    except CantGetLockError:
+                        print "Hello? Operator! Give me the number for 911!"
+                        print "Dinstall in the locked area, cant process packages, come back later"
+
     except AlreadyLockedError, e:
         print "Seems to be locked by %s already, skipping..." % (e)
 
@@ -913,7 +808,8 @@ def main():
 
     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
     if len(changes_files) == 0:
-        changes_files = utils.get_changes_files(cnf["Dir::Queue::New"])
+        new_queue = get_policy_queue('new', session );
+        changes_files = utils.get_changes_files(new_queue.path)
 
     Options = cnf.SubTree("Process-New::Options")
 
@@ -934,9 +830,6 @@ def main():
         sys.stderr.write("Sorting changes...\n")
     changes_files = sort_changes(changes_files, session)
 
-    # Kill me now? **FIXME**
-    cnf["Dinstall::Options::No-Mail"] = ""
-
     for changes_file in changes_files:
         changes_file = utils.validate_changes_file_arg(changes_file, 0)
         if not changes_file: