Merge remote-tracking branch 'drkranz/fixes' into merge

[dak.git] / dak / process_new.py

diff --git a/dak/process_new.py b/dak/process_new.py
index a934c91a766fee380d777a01e18f4af5e9c84c2d..93ebb83d5bf2ea5b810c45ab206d4df71b0c1579 100755 (executable)
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -64,6 +64,7 @@ from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLock
 from daklib.summarystats import SummaryStats
 from daklib.config import Config
 from daklib.policy import UploadCopy, PolicyQueueUploadHandler
 from daklib.summarystats import SummaryStats
 from daklib.config import Config
 from daklib.policy import UploadCopy, PolicyQueueUploadHandler

+from sqlalchemy.sql import not_

 
 # Globals
 Options = None
 
 # Globals
 Options = None


@@ -118,6 +119,21 @@ class Priority_Completer:
 
 ################################################################################
 
 
 ################################################################################
 

+def claimed_overrides(upload, missing, session):
+    source = [upload.source.source]
+    binaries = set([x.package for x in upload.binaries])
+    suites = ('unstable','experimental')
+    for m in missing:
+        if m['type'] != 'dsc':
+            binaries.remove(m['package'])
+    return session.query(DBBinary).filter(DBBinary.package.in_(binaries)). \
+                         join(DBBinary.source). \
+                         filter(not_(DBSource.source.in_(source))). \
+                         join(DBBinary.suites). \
+                         filter(Suite.suite_name.in_(suites))
+
+################################################################################
+

 def print_new (upload, missing, indexed, session, file=sys.stdout):
     check_valid(missing, session)
     index = 0
 def print_new (upload, missing, indexed, session, file=sys.stdout):
     check_valid(missing, session)
     index = 0


@@ -137,7 +153,12 @@ def print_new (upload, missing, indexed, session, file=sys.stdout):
         if not m['valid']:
             line = line + ' [!]'
         print >>file, line
         if not m['valid']:
             line = line + ' [!]'
         print >>file, line

-    notes = get_new_comments(upload.changes.source)
+    claimed = claimed_overrides(upload, missing, session)
+    if claimed.count():
+        print '\nCLAIMED OVERRIDES'
+        for c in claimed:
+            print '%s:\t%s' % (c.source.source, c.package)
+    notes = get_new_comments(upload.policy_queue, upload.changes.source)

     for note in notes:
         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
               % (note.author, note.version, note.notedate, note.comment)
     for note in notes:
         print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
               % (note.author, note.version, note.notedate, note.comment)


@@ -309,10 +330,13 @@ def edit_overrides (new, upload, session):
 
 ################################################################################
 
 
 ################################################################################
 

-def check_pkg (upload, upload_copy):
+def check_pkg (upload, upload_copy, session):
+    missing = []

     save_stdout = sys.stdout
     changes = os.path.join(upload_copy.directory, upload.changes.changesname)
     suite_name = upload.target_suite.suite_name
     save_stdout = sys.stdout
     changes = os.path.join(upload_copy.directory, upload.changes.changesname)
     suite_name = upload.target_suite.suite_name

+    handler = PolicyQueueUploadHandler(upload, session)
+    missing = [(m['type'], m["package"]) for m in handler.missing_overrides(hints=missing)]

     try:
         sys.stdout = os.popen("less -R -", 'w', 0)
         print examine_package.display_changes(suite_name, changes)
     try:
         sys.stdout = os.popen("less -R -", 'w', 0)
         print examine_package.display_changes(suite_name, changes)


@@ -324,7 +348,11 @@ def check_pkg (upload, upload_copy):
 
         for binary in upload.binaries:
             binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
 
         for binary in upload.binaries:
             binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))

-            print examine_package.check_deb(suite_name, binary_file)
+            examined = examine_package.check_deb(suite_name, binary_file)
+            # We always need to call check_deb to display package relations for every binary,
+            # but we print its output only if new overrides are being added.
+            if ("deb", binary.package) in missing:
+                print examined

 
         print examine_package.output_package_relations()
     except IOError as e:
 
         print examine_package.output_package_relations()
     except IOError as e:


@@ -450,7 +478,6 @@ def get_reject_reason(reason=''):
 ################################################################################
 
 def do_new(upload, upload_copy, handler, session):
 ################################################################################
 
 def do_new(upload, upload_copy, handler, session):

-    print "NEW\n"

     cnf = Config()
 
     run_user_inspect_command(upload, upload_copy)
     cnf = Config()
 
     run_user_inspect_command(upload, upload_copy)


@@ -465,9 +492,22 @@ def do_new(upload, upload_copy, handler, session):
         missing = handler.missing_overrides(hints=missing)
         broken = not check_valid(missing, session)
 
         missing = handler.missing_overrides(hints=missing)
         broken = not check_valid(missing, session)
 

+        changesname = os.path.basename(upload.changes.changesname)
+
+        print
+        print changesname
+        print "-" * len(changesname)
+        print
+        print "   Target:     {0}".format(upload.target_suite.suite_name)
+        print "   Changed-By: {0}".format(upload.changes.changedby)
+        print
+

         #if len(byhand) == 0 and len(missing) == 0:
         #    break
 
         #if len(byhand) == 0 and len(missing) == 0:
         #    break
 

+        if missing:
+            print "NEW\n"
+

         answer = "XXX"
         if Options["No-Action"] or Options["Automatic"]:
             answer = 'S'
         answer = "XXX"
         if Options["No-Action"] or Options["Automatic"]:
             answer = 'S'


@@ -508,36 +548,31 @@ def do_new(upload, upload_copy, handler, session):
             continue
 
         if answer == 'A' and not Options["Trainee"]:
             continue
 
         if answer == 'A' and not Options["Trainee"]:

-            try:
-                check_daily_lock()
-                add_overrides(missing, upload.target_suite, session)
-                if Config().find_b("Dinstall::BXANotify"):
-                    do_bxa_notification(missing, upload, session)
-                handler.accept()
-                done = True
-                Logger.log(["NEW ACCEPT", upload.changes.changesname])
-            except CantGetLockError:
-                print "Hello? Operator! Give me the number for 911!"
-                print "Dinstall in the locked area, cant process packages, come back later"
+            add_overrides(missing, upload.target_suite, session)
+            if Config().find_b("Dinstall::BXANotify"):
+                do_bxa_notification(missing, upload, session)
+            handler.accept()
+            done = True
+            Logger.log(["NEW ACCEPT", upload.changes.changesname])

         elif answer == 'C':
         elif answer == 'C':

-            check_pkg(upload, upload_copy)
+            check_pkg(upload, upload_copy, session)

         elif answer == 'E' and not Options["Trainee"]:
             missing = edit_overrides (missing, upload, session)
         elif answer == 'M' and not Options["Trainee"]:
             reason = Options.get('Manual-Reject', '') + "\n"
         elif answer == 'E' and not Options["Trainee"]:
             missing = edit_overrides (missing, upload, session)
         elif answer == 'M' and not Options["Trainee"]:
             reason = Options.get('Manual-Reject', '') + "\n"

-            reason = reason + "\n".join([n.comment for n in get_new_comments(upload.changes.source, session=session)])
+            reason = reason + "\n\n=====\n\n".join([n.comment for n in get_new_comments(upload.policy_queue, upload.changes.source, session=session)])

             reason = get_reject_reason(reason)
             if reason is not None:
                 Logger.log(["NEW REJECT", upload.changes.changesname])
                 handler.reject(reason)
                 done = True
         elif answer == 'N':
             reason = get_reject_reason(reason)
             if reason is not None:
                 Logger.log(["NEW REJECT", upload.changes.changesname])
                 handler.reject(reason)
                 done = True
         elif answer == 'N':

-            if edit_note(get_new_comments(upload.changes.source, session=session),
+            if edit_note(get_new_comments(upload.policy_queue, upload.changes.source, session=session),

                          upload, session, bool(Options["Trainee"])) == 0:
                 end()
                 sys.exit(0)
         elif answer == 'P' and not Options["Trainee"]:
                          upload, session, bool(Options["Trainee"])) == 0:
                 end()
                 sys.exit(0)
         elif answer == 'P' and not Options["Trainee"]:

-            if prod_maintainer(get_new_comments(upload.changes.source, session=session),
+            if prod_maintainer(get_new_comments(upload.policy_queue, upload.changes.source, session=session),

                                upload) == 0:
                 end()
                 sys.exit(0)
                                upload) == 0:
                 end()
                 sys.exit(0)


@@ -545,13 +580,13 @@ def do_new(upload, upload_copy, handler, session):
         elif answer == 'R' and not Options["Trainee"]:
             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
             if confirm == "y":
         elif answer == 'R' and not Options["Trainee"]:
             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
             if confirm == "y":

-                for c in get_new_comments(upload.changes.source, upload.changes.version, session=session):
+                for c in get_new_comments(upload.policy_queue, upload.changes.source, upload.changes.version, session=session):

                     session.delete(c)
                 session.commit()
         elif answer == 'O' and not Options["Trainee"]:
             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
             if confirm == "y":
                     session.delete(c)
                 session.commit()
         elif answer == 'O' and not Options["Trainee"]:
             confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower()
             if confirm == "y":

-                for c in get_new_comments(upload.changes.source, session=session):
+                for c in get_new_comments(upload.policy_queue, upload.changes.source, session=session):

                     session.delete(c)
                 session.commit()
 
                     session.delete(c)
                 session.commit()
 


@@ -561,6 +596,9 @@ def do_new(upload, upload_copy, handler, session):
             end()
             sys.exit(0)
 
             end()
             sys.exit(0)
 

+        if handler.get_action():
+            print "PENDING %s\n" % handler.get_action()
+

 ################################################################################
 ################################################################################
 ################################################################################
 ################################################################################
 ################################################################################
 ################################################################################


@@ -573,6 +611,7 @@ def usage (exit_code=0):
   -h, --help                show this help and exit.
   -m, --manual-reject=MSG   manual reject with `msg'
   -n, --no-action           don't do anything
   -h, --help                show this help and exit.
   -m, --manual-reject=MSG   manual reject with `msg'
   -n, --no-action           don't do anything

+  -q, --queue=QUEUE         operate on a different queue

   -t, --trainee             FTP Trainee mode
   -V, --version             display the version number and exit
 
   -t, --trainee             FTP Trainee mode
   -V, --version             display the version number and exit
 


@@ -602,24 +641,6 @@ ENVIRONMENT VARIABLES
 
 ################################################################################
 
 
 ################################################################################
 

-def check_daily_lock():
-    """
-    Raises CantGetLockError if the dinstall daily.lock exists.
-    """
-
-    cnf = Config()
-    try:
-        lockfile = cnf.get("Process-New::DinstallLockFile",
-                           os.path.join(cnf['Dir::Lock'], 'processnew.lock'))
-
-        os.open(lockfile,
-                os.O_RDONLY | os.O_CREAT | os.O_EXCL)
-    except OSError as e:
-        if e.errno == errno.EEXIST or e.errno == errno.EACCES:
-            raise CantGetLockError
-
-    os.unlink(lockfile)
-

 @contextlib.contextmanager
 def lock_package(package):
     """
 @contextlib.contextmanager
 def lock_package(package):
     """


@@ -650,6 +671,8 @@ def do_pkg(upload, session):
     dsc = upload.source
 
     cnf = Config()
     dsc = upload.source
 
     cnf = Config()

+    group = cnf.get('Dinstall::UnprivGroup') or None
+

     #bcc = "X-DAK: dak process-new"
     #if cnf.has_key("Dinstall::Bcc"):
     #    u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
     #bcc = "X-DAK: dak process-new"
     #if cnf.has_key("Dinstall::Bcc"):
     #    u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])


@@ -658,9 +681,10 @@ def do_pkg(upload, session):
 
     try:
       with lock_package(upload.changes.source):
 
     try:
       with lock_package(upload.changes.source):

-       with UploadCopy(upload) as upload_copy:
+       with UploadCopy(upload, group=group) as upload_copy:

         handler = PolicyQueueUploadHandler(upload, session)
         if handler.get_action() is not None:
         handler = PolicyQueueUploadHandler(upload, session)
         if handler.get_action() is not None:

+            print "PENDING %s\n" % handler.get_action()

             return
 
         do_new(upload, upload_copy, handler, session)
             return
 
         do_new(upload, upload_copy, handler, session)


@@ -686,6 +710,43 @@ def show_new_comments(uploads, session):
 
 ################################################################################
 
 
 ################################################################################
 

+def sort_uploads(new_queue, uploads, session, nobinaries=False):
+    sources = {}
+    sorteduploads = []
+    suitesrc = [s.source for s in session.query(DBSource.source). \
+      filter(DBSource.suites.any(Suite.suite_name.in_(['unstable', 'experimental'])))]
+    comments = [p.package for p in session.query(NewComment.package). \
+      filter_by(trainee=False, policy_queue=new_queue).distinct()]
+    for upload in uploads:
+        source = upload.changes.source
+        if not source in sources:
+            sources[source] = []
+        sources[source].append({'upload': upload,
+                                'date': upload.changes.created,
+                                'stack': 1,
+                                'binary': True if source in suitesrc else False,
+                                'comments': True if source in comments else False})
+    for src in sources:
+        if len(sources[src]) > 1:
+            changes = sources[src]
+            firstseen = sorted(changes, key=lambda k: (k['date']))[0]['date']
+            changes.sort(key=lambda item:item['date'])
+            for i in range (0, len(changes)):
+                changes[i]['date'] = firstseen
+                changes[i]['stack'] = i + 1
+        sorteduploads += sources[src]
+    if nobinaries:
+        sorteduploads = [u["upload"] for u in sorted(sorteduploads,
+                         key=lambda k: (k["comments"], k["binary"],
+                         k["date"], -k["stack"]))]
+    else:
+        sorteduploads = [u["upload"] for u in sorted(sorteduploads,
+                         key=lambda k: (k["comments"], -k["binary"],
+                         k["date"], -k["stack"]))]
+    return sorteduploads
+
+################################################################################
+

 def end():
     accept_count = SummaryStats().accept_count
     accept_bytes = SummaryStats().accept_bytes
 def end():
     accept_count = SummaryStats().accept_count
     accept_bytes = SummaryStats().accept_bytes


@@ -748,14 +809,12 @@ def main():
 
     if len(uploads) > 1:
         sys.stderr.write("Sorting changes...\n")
 
     if len(uploads) > 1:
         sys.stderr.write("Sorting changes...\n")

-        uploads.sort()
+        uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"])

 
     if Options["Comments"]:
         show_new_comments(uploads, session)
     else:
         for upload in uploads:
 
     if Options["Comments"]:
         show_new_comments(uploads, session)
     else:
         for upload in uploads:

-            print "\n" + os.path.basename(upload.changes.changesname)
-

             do_pkg (upload, session)
 
     end()
             do_pkg (upload, session)
 
     end()