And also catch double-sign-edit-failure-sign-whatever files

[dak.git] / dak / process_new.py

diff --git a/dak/process_new.py b/dak/process_new.py
index 74bae8fe4cef50f98e52c5bf39837c5664a4d0bb..1c2231a305709baf6ff7379fd55fbbe9868ce1d0 100755 (executable)
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -42,8 +42,6 @@
 
 ################################################################################
 
 
 ################################################################################
 

-from __future__ import with_statement
-

 import copy
 import errno
 import os
 import copy
 import errno
 import os


@@ -333,74 +331,33 @@ def edit_overrides (new, upload, session):
 
     return new
 
 
     return new
 

-################################################################################
-
-def edit_note(note, upload, session):
-    # Write the current data to a temporary file
-    (fd, temp_filename) = utils.temp_filename()
-    editor = os.environ.get("EDITOR","vi")
-    answer = 'E'
-    while answer == 'E':
-        os.system("%s %s" % (editor, temp_filename))
-        temp_file = utils.open_file(temp_filename)
-        newnote = temp_file.read().rstrip()
-        temp_file.close()
-        print "New Note:"
-        print utils.prefix_multi_line_string(newnote,"  ")
-        prompt = "[D]one, Edit, Abandon, Quit ?"
-        answer = "XXX"
-        while prompt.find(answer) == -1:
-            answer = utils.our_raw_input(prompt)
-            m = re_default_answer.search(prompt)
-            if answer == "":
-                answer = m.group(1)
-            answer = answer[:1].upper()
-    os.unlink(temp_filename)
-    if answer == 'A':
-        return
-    elif answer == 'Q':
-        end()
-        sys.exit(0)
-
-    comment = NewComment()
-    comment.package = upload.pkg.changes["source"]
-    comment.version = upload.pkg.changes["version"]
-    comment.comment = newnote
-    comment.author  = utils.whoami()
-    comment.trainee = bool(Options["Trainee"])
-    session.add(comment)
-    session.commit()

 
 ################################################################################
 
 def check_pkg (upload):
 
 ################################################################################
 
 def check_pkg (upload):

+    save_stdout = sys.stdout

     try:
     try:

-        less_fd = os.popen("less -R -", 'w', 0)
-        stdout_fd = sys.stdout
-        try:
-            sys.stdout = less_fd
-            changes = utils.parse_changes (upload.pkg.changes_file)
-            examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
-            files = upload.pkg.files
-            for f in files.keys():
-                if files[f].has_key("new"):
-                    ftype = files[f]["type"]
-                    if ftype == "deb":
-                        examine_package.check_deb(changes['distribution'], f)
-                    elif ftype == "dsc":
-                        examine_package.check_dsc(changes['distribution'], f)
-        finally:
-            examine_package.output_package_relations()
-            sys.stdout = stdout_fd
-    except IOError, e:
+        sys.stdout = os.popen("less -R -", 'w', 0)
+        changes = utils.parse_changes (upload.pkg.changes_file)
+        print examine_package.display_changes(changes['distribution'], upload.pkg.changes_file)
+        files = upload.pkg.files
+        for f in files.keys():
+            if files[f].has_key("new"):
+                ftype = files[f]["type"]
+                if ftype == "deb":
+                    print examine_package.check_deb(changes['distribution'], f)
+                elif ftype == "dsc":
+                    print examine_package.check_dsc(changes['distribution'], f)
+        print examine_package.output_package_relations()
+    except IOError as e:

         if e.errno == errno.EPIPE:
             utils.warn("[examine_package] Caught EPIPE; skipping.")
         if e.errno == errno.EPIPE:
             utils.warn("[examine_package] Caught EPIPE; skipping.")

-            pass

         else:
         else:

+            sys.stdout = save_stdout

             raise
     except KeyboardInterrupt:
         utils.warn("[examine_package] Caught C-c; skipping.")
             raise
     except KeyboardInterrupt:
         utils.warn("[examine_package] Caught C-c; skipping.")

-        pass
+    sys.stdout = save_stdout

 
 ################################################################################
 
 
 ################################################################################
 


@@ -433,7 +390,7 @@ def add_overrides (new, upload, session):
             type_id = get_override_type(new[pkg]["type"]).overridetype_id
             priority_id = new[pkg]["priority id"]
             section_id = new[pkg]["section id"]
             type_id = get_override_type(new[pkg]["type"]).overridetype_id
             priority_id = new[pkg]["priority id"]
             section_id = new[pkg]["section id"]

-            Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])
+            Logger.log(["%s (%s) overrides" % (pkg, srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]])

             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
             for f in new[pkg]["files"]:
             session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')",
                             { 'sid': suite_id, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id})
             for f in new[pkg]["files"]:


@@ -448,57 +405,6 @@ def add_overrides (new, upload, session):
 
 ################################################################################
 
 
 ################################################################################
 

-def prod_maintainer (notes, upload):
-    cnf = Config()
-    # Here we prepare an editor and get them ready to prod...
-    (fd, temp_filename) = utils.temp_filename()
-    temp_file = os.fdopen(fd, 'w')
-    for note in notes:
-        temp_file.write(note.comment)
-    temp_file.close()
-    editor = os.environ.get("EDITOR","vi")
-    answer = 'E'
-    while answer == 'E':
-        os.system("%s %s" % (editor, temp_filename))
-        temp_fh = utils.open_file(temp_filename)
-        prod_message = "".join(temp_fh.readlines())
-        temp_fh.close()
-        print "Prod message:"
-        print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
-        prompt = "[P]rod, Edit, Abandon, Quit ?"
-        answer = "XXX"
-        while prompt.find(answer) == -1:
-            answer = utils.our_raw_input(prompt)
-            m = re_default_answer.search(prompt)
-            if answer == "":
-                answer = m.group(1)
-            answer = answer[:1].upper()
-    os.unlink(temp_filename)
-    if answer == 'A':
-        return
-    elif answer == 'Q':
-        end()
-        sys.exit(0)
-    # Otherwise, do the proding...
-    user_email_address = utils.whoami() + " <%s>" % (
-        cnf["Dinstall::MyAdminAddress"])
-
-    Subst = upload.Subst
-
-    Subst["__FROM_ADDRESS__"] = user_email_address
-    Subst["__PROD_MESSAGE__"] = prod_message
-    Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
-
-    prod_mail_message = utils.TemplateSubst(
-        Subst,cnf["Dir::Templates"]+"/process-new.prod")
-
-    # Send the prod mail
-    utils.send_mail(prod_mail_message)
-
-    print "Sent proding message"
-
-################################################################################
-

 def do_new(upload, session):
     print "NEW\n"
     files = upload.pkg.files
 def do_new(upload, session):
     print "NEW\n"
     files = upload.pkg.files


@@ -512,11 +418,18 @@ def do_new(upload, session):
     # Make a copy of distribution we can happily trample on
     changes["suite"] = copy.copy(changes["distribution"])
 
     # Make a copy of distribution we can happily trample on
     changes["suite"] = copy.copy(changes["distribution"])
 

+    # Try to get an included dsc
+    dsc = None
+    (status, _) = upload.load_dsc()
+    if status:
+        dsc = upload.pkg.dsc
+

     # The main NEW processing loop
     done = 0
     # The main NEW processing loop
     done = 0

+    new = {}

     while not done:
         # Find out what's new
     while not done:
         # Find out what's new

-        new = determine_new(changes, files)
+        new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session, new=new)

 
         if not new:
             break
 
         if not new:
             break


@@ -573,7 +486,7 @@ def do_new(upload, session):
                 done = 1
         elif answer == 'N':
             edit_note(get_new_comments(changes.get("source", ""), session=session),
                 done = 1
         elif answer == 'N':
             edit_note(get_new_comments(changes.get("source", ""), session=session),

-                      upload, session)
+                      upload, session, bool(Options["Trainee"]))

         elif answer == 'P' and not Options["Trainee"]:
             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
                             upload)
         elif answer == 'P' and not Options["Trainee"]:
             prod_maintainer(get_new_comments(changes.get("source", ""), session=session),
                             upload)


@@ -604,6 +517,8 @@ def do_new(upload, session):
 def usage (exit_code=0):
     print """Usage: dak process-new [OPTION]... [CHANGES]...
   -a, --automatic           automatic run
 def usage (exit_code=0):
     print """Usage: dak process-new [OPTION]... [CHANGES]...
   -a, --automatic           automatic run

+  -b, --no-binaries         do not sort binary-NEW packages first
+  -c, --comments            show NEW comments

   -h, --help                show this help and exit.
   -m, --manual-reject=MSG   manual reject with `msg'
   -n, --no-action           don't do anything
   -h, --help                show this help and exit.
   -m, --manual-reject=MSG   manual reject with `msg'
   -n, --no-action           don't do anything


@@ -617,14 +532,14 @@ def do_byhand(upload, session):
     done = 0
     while not done:
         files = upload.pkg.files
     done = 0
     while not done:
         files = upload.pkg.files

-        will_install = 1
+        will_install = True

         byhand = []
 
         for f in files.keys():
         byhand = []
 
         for f in files.keys():

-            if files[f]["type"] == "byhand":
+            if files[f]["section"] == "byhand":

                 if os.path.exists(f):
                     print "W: %s still present; please process byhand components and try again." % (f)
                 if os.path.exists(f):
                     print "W: %s still present; please process byhand components and try again." % (f)

-                    will_install = 0
+                    will_install = False

                 else:
                     byhand.append(f)
 
                 else:
                     byhand.append(f)
 


@@ -646,21 +561,39 @@ def do_byhand(upload, session):
             answer = answer[:1].upper()
 
         if answer == 'A':
             answer = answer[:1].upper()
 
         if answer == 'A':

-            try:
-                check_daily_lock()
-                done = 1
-                for f in byhand:
-                    del files[f]
-                Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
-            except CantGetLockError:
-                print "Hello? Operator! Give me the number for 911!"
-                print "Dinstall in the locked area, cant process packages, come back later"
+            dbchg = get_dbchange(upload.pkg.changes_file, session)
+            if dbchg is None:
+                print "Warning: cannot find changes file in database; can't process BYHAND"
+            else:
+                try:
+                    check_daily_lock()
+                    done = 1
+                    for b in byhand:
+                        # Find the file entry in the database
+                        found = False
+                        for f in dbchg.files:
+                            if f.filename == b:
+                                found = True
+                                f.processed = True
+                                break
+
+                        if not found:
+                            print "Warning: Couldn't find BYHAND item %s in the database to mark it processed" % b
+
+                    session.commit()
+                    Logger.log(["BYHAND ACCEPT: %s" % (upload.pkg.changes_file)])
+                except CantGetLockError:
+                    print "Hello? Operator! Give me the number for 911!"
+                    print "Dinstall in the locked area, cant process packages, come back later"

         elif answer == 'M':
         elif answer == 'M':

-            Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
-            upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
-            upload.pkg.remove_known_changes(session=session)
-            session.commit()
-            done = 1
+            aborted = upload.do_reject(manual=1,
+                                       reject_message=Options["Manual-Reject"],
+                                       notes=get_new_comments(changes.get("source", ""), session=session))
+            if not aborted:
+                upload.pkg.remove_known_changes(session=session)
+                session.commit()
+                Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
+                done = 1

         elif answer == 'S':
             done = 1
         elif answer == 'Q':
         elif answer == 'S':
             done = 1
         elif answer == 'Q':


@@ -676,13 +609,16 @@ def check_daily_lock():
 
     cnf = Config()
     try:
 
     cnf = Config()
     try:

-        os.open(cnf["Process-New::DinstallLockFile"],
+        lockfile = cnf.get("Process-New::DinstallLockFile",
+                           os.path.join(cnf['Dir::Lock'], 'processnew.lock'))
+
+        os.open(lockfile,

                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)
                 os.O_RDONLY | os.O_CREAT | os.O_EXCL)

-    except OSError, e:
+    except OSError as e:

         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
             raise CantGetLockError
 
         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
             raise CantGetLockError
 

-    os.unlink(cnf["Process-New::DinstallLockFile"])
+    os.unlink(lockfile)

 
 
 @contextlib.contextmanager
 
 
 @contextlib.contextmanager


@@ -694,13 +630,16 @@ def lock_package(package):
     @param package: source package name to lock
     """
 
     @param package: source package name to lock
     """
 

-    path = os.path.join(Config()["Process-New::LockDir"], package)
+    cnf = Config()
+
+    path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package)
+

     try:
         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
     try:
         fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)

-    except OSError, e:
+    except OSError as e:

         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
         if e.errno == errno.EEXIST or e.errno == errno.EACCES:
             user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')

-            raise AlreadyLockedError, user
+            raise AlreadyLockedError(user)

 
     try:
         yield fd
 
     try:
         yield fd


@@ -722,17 +661,25 @@ class clean_holding(object):
                 os.unlink(os.path.join(h.holding_dir, f))
 
 
                 os.unlink(os.path.join(h.holding_dir, f))
 
 

-def do_pkg(changes_file, session):
-    new_queue = get_policy_queue('new', session );
+def do_pkg(changes_full_path, session):
+    changes_dir = os.path.dirname(changes_full_path)
+    changes_file = os.path.basename(changes_full_path)
+

     u = Upload()
     u.pkg.changes_file = changes_file
     (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
     u.load_changes(changes_file)
     u = Upload()
     u.pkg.changes_file = changes_file
     (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changes_file)
     u.load_changes(changes_file)

-    u.pkg.directory = new_queue.path
+    u.pkg.directory = changes_dir

     u.update_subst()
     u.logger = Logger
     origchanges = os.path.abspath(u.pkg.changes_file)
 
     u.update_subst()
     u.logger = Logger
     origchanges = os.path.abspath(u.pkg.changes_file)
 

+    # Try to get an included dsc
+    dsc = None
+    (status, _) = u.load_dsc()
+    if status:
+        dsc = u.pkg.dsc
+

     cnf = Config()
     bcc = "X-DAK: dak process-new"
     if cnf.has_key("Dinstall::Bcc"):
     cnf = Config()
     bcc = "X-DAK: dak process-new"
     if cnf.has_key("Dinstall::Bcc"):


@@ -741,6 +688,7 @@ def do_pkg(changes_file, session):
         u.Subst["__BCC__"] = bcc
 
     files = u.pkg.files
         u.Subst["__BCC__"] = bcc
 
     files = u.pkg.files

+    u.check_distributions()

     for deb_filename, f in files.items():
         if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
             u.binary_file_checks(deb_filename, session)
     for deb_filename, f in files.items():
         if deb_filename.endswith(".udeb") or deb_filename.endswith(".deb"):
             u.binary_file_checks(deb_filename, session)


@@ -757,9 +705,10 @@ def do_pkg(changes_file, session):
                 if not recheck(u, session):
                     return
 
                 if not recheck(u, session):
                     return
 

-                # FIXME: This does need byhand checks added!
-                new = determine_new(u.pkg.changes, files)
-                if new:
+                new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session)
+                if byhand:
+                    do_byhand(u, session)
+                elif new:

                     do_new(u, session)
                 else:
                     try:
                     do_new(u, session)
                 else:
                     try:


@@ -769,9 +718,26 @@ def do_pkg(changes_file, session):
                         print "Hello? Operator! Give me the number for 911!"
                         print "Dinstall in the locked area, cant process packages, come back later"
 
                         print "Hello? Operator! Give me the number for 911!"
                         print "Dinstall in the locked area, cant process packages, come back later"
 

-    except AlreadyLockedError, e:
+    except AlreadyLockedError as e:

         print "Seems to be locked by %s already, skipping..." % (e)
 
         print "Seems to be locked by %s already, skipping..." % (e)
 

+def show_new_comments(changes_files, session):
+    sources = set()
+    query = """SELECT package, version, comment, author
+               FROM new_comments
+               WHERE package IN ('"""
+
+    for changes in changes_files:
+        sources.add(os.path.basename(changes).split("_")[0])
+
+    query += "%s') ORDER BY package, version" % "', '".join(sources)
+    r = session.execute(query)
+
+    for i in r:
+        print "%s_%s\n%s\n(%s)\n\n\n" % (i[0], i[1], i[2], i[3])
+
+    session.commit()
+

 ################################################################################
 
 def end():
 ################################################################################
 
 def end():


@@ -797,19 +763,23 @@ def main():
     session = DBConn().session()
 
     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
     session = DBConn().session()
 
     Arguments = [('a',"automatic","Process-New::Options::Automatic"),

+                 ('b',"no-binaries","Process-New::Options::No-Binaries"),
+                 ('c',"comments","Process-New::Options::Comments"),

                  ('h',"help","Process-New::Options::Help"),
                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
                  ('t',"trainee","Process-New::Options::Trainee"),
                  ('n',"no-action","Process-New::Options::No-Action")]
 
                  ('h',"help","Process-New::Options::Help"),
                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
                  ('t',"trainee","Process-New::Options::Trainee"),
                  ('n',"no-action","Process-New::Options::No-Action")]
 

-    for i in ["automatic", "help", "manual-reject", "no-action", "version", "trainee"]:
+    for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]:

         if not cnf.has_key("Process-New::Options::%s" % (i)):
             cnf["Process-New::Options::%s" % (i)] = ""
 
     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
     if len(changes_files) == 0:
         new_queue = get_policy_queue('new', session );
         if not cnf.has_key("Process-New::Options::%s" % (i)):
             cnf["Process-New::Options::%s" % (i)] = ""
 
     changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv)
     if len(changes_files) == 0:
         new_queue = get_policy_queue('new', session );

-        changes_files = utils.get_changes_files(new_queue.path)
+        changes_paths = [ os.path.join(new_queue.path, j) for j in utils.get_changes_files(new_queue.path) ]
+    else:
+        changes_paths = [ os.path.abspath(j) for j in changes_files ]

 
     Options = cnf.SubTree("Process-New::Options")
 
 
     Options = cnf.SubTree("Process-New::Options")
 


@@ -818,25 +788,28 @@ def main():
 
     if not Options["No-Action"]:
         try:
 
     if not Options["No-Action"]:
         try:

-            Logger = daklog.Logger(cnf, "process-new")
-        except CantOpenError, e:
+            Logger = daklog.Logger("process-new")
+        except CantOpenError as e:

             Options["Trainee"] = "True"
 
     Sections = Section_Completer(session)
     Priorities = Priority_Completer(session)
     readline.parse_and_bind("tab: complete")
 
             Options["Trainee"] = "True"
 
     Sections = Section_Completer(session)
     Priorities = Priority_Completer(session)
     readline.parse_and_bind("tab: complete")
 

-    if len(changes_files) > 1:
+    if len(changes_paths) > 1:

         sys.stderr.write("Sorting changes...\n")
         sys.stderr.write("Sorting changes...\n")

-    changes_files = sort_changes(changes_files, session)
+    changes_files = sort_changes(changes_paths, session, Options["No-Binaries"])

 
 

-    for changes_file in changes_files:
-        changes_file = utils.validate_changes_file_arg(changes_file, 0)
-        if not changes_file:
-            continue
-        print "\n" + changes_file
+    if Options["Comments"]:
+        show_new_comments(changes_files, session)
+    else:
+        for changes_file in changes_files:
+            changes_file = utils.validate_changes_file_arg(changes_file, 0)
+            if not changes_file:
+                continue
+            print "\n" + os.path.basename(changes_file)

 
 

-        do_pkg (changes_file, session)
+            do_pkg (changes_file, session)

 
     end()
 
 
     end()