Merge remote branch 'tolimar/rm-close-bugs' into merge

* tolimar/rm-close-bugs:
  Make the wnpp parse more robust for missing files
  Only log, if there is something to log
  Make it the qa wnpp list download more fault tolerant
  Fix comment of cron.daily (copy and paste from cron.monthly)
  Use different mail templates to close removal bugs, if related bugs got closed
  Refresh the wnpp bugs list on a daily basis
  Also close wnpp bugs on package removal

Signed-off-by: Joerg Jaspert <joerg@debian.org>

diff --git a/config/debian/cron.daily b/config/debian/cron.daily
new file mode 100755 (executable)
index 0000000..1815260
--- /dev/null
+++ b/config/debian/cron.daily
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Run daily via cron, out of dak's crontab.
+
+set -e
+set -u
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
+
+################################################################################
+
+
+# get the latest list of wnpp bugs and their source packages
+if $( wget -q -O/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm.tmp http://qa.debian.org/data/bts/wnpp_rm ) ; then
+    mv /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm.tmp /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
+else
+    if [ -f /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm.tmp ] ; then
+        rm /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm.tmp
+    fi
+fi
+
+################################################################################

diff --git a/config/debian/crontab b/config/debian/crontab
index 4c0b96932650cef52928825b1872233e2199fab3..f1bc068513951cb23d9c8ade53fb1657e5a88613 100644 (file)
--- a/config/debian/crontab
+++ b/config/debian/crontab
@@ -6,6 +6,7 @@ PATH=/usr/local/bin:/usr/bin:/bin
 52         1,7,13,19  *   *   *   /srv/ftp-master.debian.org/dak/config/debian/cron.dinstall
 #### And an alternate line, for the times the release team wants it half an hour late.
 ####22         2,8,14,20  *   *   *   /srv/ftp-master.debian.org/dak/config/debian/cron.dinstall
+3          9          *   *   *   /srv/ftp-master.debian.org/dak/config/debian/cron.daily
 0          12         *   *   0   /srv/ftp-master.debian.org/dak/config/debian/cron.weekly
 6          0          1   *   *   /srv/ftp-master.debian.org/dak/config/debian/cron.monthly
 @reboot                           /srv/ftp-master.debian.org/dak/config/debian/cron.reboot

diff --git a/dak/rm.py b/dak/rm.py
index db43de190dc82f02e59e7c185d1cfdf85fc3df5c..7377f23b5b7236382e67d002005e71ac0932d800 100755 (executable)
--- a/dak/rm.py
+++ b/dak/rm.py
@@ -619,15 +619,17 @@ def main ():
         Subst_close_rm["__PRIMARY_MIRROR__"] = Archive["PrimaryMirror"]
         for bug in utils.split_args(Options["Done"]):
             Subst_close_rm["__BUG_NUMBER__"] = bug
-            mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
+            if Options["Do-Close"]:
+                mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related")
+            else:
+                mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close")
             utils.send_mail(mail_message)
 
     # close associated bug reports
-    # FIXME:  We should also close possible WNPP bugs for that package, but
-    # currently there's no sane way to determine them
     if Options["Do-Close"]:
         Subst_close_other = Subst_common
         bcc = []
+        wnpp = utils.parse_wnpp_bug_file()
         if len(versions) == 1:
             Subst_close_other["__VERSION__"] = versions[0]
         else:
@@ -639,21 +641,39 @@ def main ():
         # at this point, I just assume, that the first closed bug gives
         # some useful information on why the package got removed
         Subst_close_other["__BUG_NUMBER__"] = utils.split_args(Options["Done"])[0]
-        if len(sources) > 1:
+        if len(sources) == 1:
+            source_pkg = source.split("_", 1)[0]
+        else:
             utils.fubar("Closing bugs for multiple source pakcages is not supported.  Do it yourself.")
         Subst_close_other["__BUG_NUMBER_ALSO__"] = ""
-        Subst_close_other["__SOURCE__"] = source.split("_", 1)[0]
-        logfile.write("Also closing bugs: ")
-        logfile822.write("Also-Bugs: ")
-        for bug in bts.get_bugs('src', source.split("_", 1)[0], 'status', 'open'):
-            Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
-            logfile.write(" " + str(bug))
-            logfile822.write(" " + str(bug))
+        Subst_close_other["__SOURCE__"] = source_pkg
+        other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open')
+        if other_bugs:
+            logfile.write("Also closing bug(s):")
+            logfile822.write("Also-Bugs:")
+            for bug in other_bugs:
+                Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
+                logfile.write(" " + str(bug))
+                logfile822.write(" " + str(bug))
+            logfile.write("\n")
+            logfile822.write("\n")
+        if source_pkg in wnpp.keys():
+            logfile.write("Also closing WNPP bug(s):")
+            logfile822.write("Also-WNPP:")
+            for bug in wnpp[source_pkg]:
+                # the wnpp-rm file we parse also contains our removal
+                # bugs, filtering that out
+                if bug != Subst_close_other["__BUG_NUMBER__"]:
+                    Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + ","
+                    logfile.write(" " + str(bug))
+                    logfile822.write(" " + str(bug))
+            logfile.write("\n")
+            logfile822.write("\n")
+
         mail_message = utils.TemplateSubst(Subst_close_other,cnf["Dir::Templates"]+"/rm.bug-close-related")
         if Subst_close_other["__BUG_NUMBER_ALSO__"]:
             utils.send_mail(mail_message)
-        logfile.write("\n")
-        logfile822.write("\n")
+
 
     logfile.write("=========================================================================\n")
     logfile.close()

diff --git a/daklib/utils.py b/daklib/utils.py
index c9b54d078db3bc6b85e05e214823eb8b54f2aa74..7ed4089a4dd2610a09a7c5f7a3852eda987b5456 100755 (executable)
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -1536,3 +1536,38 @@ if not os.getenv("DAK_TEST"):
 
 if which_conf_file() != default_config:
     apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
+
+################################################################################
+
+def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"):
+    """
+    Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm
+    Well, actually it parsed a local copy, but let's document the source
+    somewhere ;)
+
+    returns a dict associating source package name with a list of open wnpp
+    bugs (Yes, there might be more than one)
+    """
+
+    line = []
+    try:
+        f = open(file)
+        lines = f.readlines()
+    except IOerror, e:
+        print "Warning:  Couldn't open %s; don't know about WNPP bugs, so won't close any." % file
+       lines = []
+    wnpp = {}
+
+    for line in lines:
+        splited_line = line.split(": ", 1)
+        if len(splited_line) > 1:
+            wnpp[splited_line[0]] = splited_line[1].split("|")
+
+    for source in wnpp.keys():
+        bugs = []
+        for wnpp_bug in wnpp[source]:
+            bug_no = re.search("(\d)+", wnpp_bug).group()
+            if bug_no:
+                bugs.append(bug_no)
+        wnpp[source] = bugs
+    return wnpp

diff --git a/templates/rm.bug-close-with-related b/templates/rm.bug-close-with-related
new file mode 100644 (file)
index 0000000..0dcc1b4
--- /dev/null
+++ b/templates/rm.bug-close-with-related
@@ -0,0 +1,45 @@
+From: __RM_ADDRESS__
+To: __BUG_NUMBER__-close@__BUG_SERVER__
+__CC__
+__BCC__
+X-Debian: DAK
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Bug#__BUG_NUMBER__: __SUBJECT__
+
+We believe that the bug you reported is now fixed; the following
+package(s) have been removed from __SUITE_LIST__:
+
+__SUMMARY__
+Note that the package(s) have simply been removed from the tag
+database and may (or may not) still be in the pool; this is not a bug.
+The package(s) will be physically removed automatically when no suite
+references them (and in the case of source, when no binary references
+it).  Please also remember that the changes have been done on the
+master archive (__MASTER_ARCHIVE__) and will not propagate to any
+mirrors (__PRIMARY_MIRROR__ included) until the next cron.daily run at the
+earliest.
+
+Packages are usually not removed from testing by hand. Testing tracks
+unstable and will automatically remove packages which were removed
+from unstable when removing them from testing causes no dependency
+problems. The release team can force a removal from testing if it is
+really needed, please contact them if this should be the case.
+
+We try to close Bugs which have been reported against this package
+automatically.  But please check all old bugs, if they where closed
+correctly or should have been re-assign to another package.
+
+Thank you for reporting the bug, which will now be closed.  If you
+have further comments please address them to __BUG_NUMBER__@__BUG_SERVER__.
+
+The full log for this bug can be viewed at http://__BUG_SERVER__/__BUG_NUMBER__
+
+This message was generated automatically; if you believe that there is
+a problem with it please contact the archive administrators by mailing
+__ADMIN_ADDRESS__.
+
+__DISTRO__ distribution maintenance software
+pp.
+__WHOAMI__ (the ftpmaster behind the curtain)