Merge commit 'mhy/master' into merge

* commit 'mhy/master':
  docstring
  move logic into a function
  more tidy ups for paths/variables
  use variables for paths
  allow p-a and p-u to take a directory to parse themselves
  dinstall
  p-a
  dinstall
  Ignore errors to updates
  2 more fixes
  dinstall
  Add a space
  utf-8

Signed-off-by: Joerg Jaspert <joerg@debian.org>

diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked
index 41b664016126c642d12d7bb159d038e9f99f5516..5e7a035dcfeeafb0592bab7bbddd6e6b9e1dc645 100755 (executable)
--- a/config/debian/cron.unchecked
+++ b/config/debian/cron.unchecked
@@ -31,7 +31,7 @@ if lockfile -r3 $LOCKFILE; then
 
     if [ ! -z "$changes" ]; then
        echo "$timestamp": "$changes"  >> $report
-       dak process-unchecked -a $changes >> $report
+       dak process-unchecked -a -d "$unchecked" >> $report
        echo "--" >> $report
 
        # sync with debbugs

diff --git a/config/debian/dinstall b/config/debian/dinstall
index 0f6a7c8fef7f07c8f9324bfd575c8d9c33d1f9bd..b6ed073a3c2c82f90046febc38d15b6cc921fcbc 100755 (executable)
--- a/config/debian/dinstall
+++ b/config/debian/dinstall
@@ -63,7 +63,8 @@ function ts() {
 
 # Cleanup actions
 function cleanup() {
-    savelog -c ${LOGROTATE} -j "$LOGFILE"
+       rm -f ${LOCK_DAILY}
+       rm -f ${LOCK_ACCEPTED}
 }
 
 # Setup the notice file to tell bad mirrors they used the wrong time
@@ -166,10 +167,9 @@ function i18n1() {
 # Process the accepted queue
 function accepted() {
     log "Processing queue/accepted"
-    cd "$accepted"
-    rm -f REPORT
-    dak process-accepted -pa *.changes | tee REPORT | \
-        mail -s "Install for $(date +"%D - %R")" ftpmaster@ftp-master.debian.org
+    rm -f "$accepted/REPORT"
+    dak process-accepted -pa -d "$accepted" > "$accepted/REPORT"
+    cat REPORT | mail -s "Install for $(date +"%D - %R")" ftpmaster@ftp-master.debian.org
     chgrp debadmin REPORT
     chmod 664 REPORT
 }
@@ -223,7 +223,7 @@ function release() {
     dak generate-releases
 }
 
-function cleanup() {
+function dakcleanup() {
     log "Cleanup old packages/files"
     dak clean-suites
     dak clean-queues
@@ -384,7 +384,7 @@ function stage() {
     # it has to cd first!
     cd ${configdir}
 
-    if [ "${ERR}" = "false"]; then
+    if [ "${ERR}" = "false" ]; then
         set +e
     fi
     ${FUNC} ${ARGS}
@@ -407,7 +407,7 @@ function stage() {
 # We need logs.
 LOGFILE="$logdir/dinstall.log"
 
-exec > "$LOGFILE" 2>&1
+exec >> "$LOGFILE" 2>&1
 
 # usually we are not using debug logs. Set to 1 if you want them.
 DEBUG=0
@@ -478,7 +478,7 @@ GO=(
     FUNC="updates"
     TIME="External Updates"
     ARGS=""
-    ERR=""
+    ERR="false"
 )
 stage $GO
 
@@ -524,7 +524,7 @@ GO=(
 )
 stage $GO
 
-rm -f $LOCKAC
+rm -f "$LOCK_ACCEPTED"
 
 GO=(
     FUNC="msfl"
@@ -583,7 +583,7 @@ GO=(
 stage $GO
 
 GO=(
-    FUNC="cleanup"
+    FUNC="dakcleanup"
     TIME="cleanup"
     ARGS=""
     ERR=""
@@ -733,6 +733,8 @@ stage $GO
 
 cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org
 
+savelog -c ${LOGROTATE} -j "$LOGFILE"
+
 # Now, at the very (successful) end of dinstall, make sure we remove
 # our stage files, so the next dinstall run will do it all again.
 rm -f "${stagedir}/*"

diff --git a/dak/check_archive.py b/dak/check_archive.py
index 80782908864f5a72ed52ee5fbd55b76976fc3d1a..bed974de47293620d98d7964caed4757589dacbc 100755 (executable)
--- a/dak/check_archive.py
+++ b/dak/check_archive.py
@@ -161,6 +161,9 @@ def check_dscs():
             except InvalidDscError, line:
                 utils.warn("syntax error in .dsc file '%s', line %s." % (f, line))
                 count += 1
+            except ChangesUnicodeError:
+                utils.warn("found invalid changes file, not properly utf-8 encoded")
+                count += 1
 
     if count:
         utils.warn("Found %s invalid .dsc files." % (count))

diff --git a/dak/check_proposed_updates.py b/dak/check_proposed_updates.py
index f12df878948113576715d94d768f7d5839c3a2a6..16a9876348d24f78b80179950e338915b58057c5 100755 (executable)
--- a/dak/check_proposed_updates.py
+++ b/dak/check_proposed_updates.py
@@ -176,6 +176,9 @@ def check_changes (filename):
     try:
         changes = utils.parse_changes(filename)
         files = utils.build_file_list(changes)
+    except ChangesUnicodeError:
+        utils.warn("Improperly encoded changes file, not utf-8")
+        return
     except:
         utils.warn("Error parsing changes file '%s'" % (filename))
         return

diff --git a/dak/examine_package.py b/dak/examine_package.py
index d0e1e53aed7a0d357a67838fda6a84fe7231a977..eb602794304cf5e562f3d168386d4373e1d5a18d 100755 (executable)
--- a/dak/examine_package.py
+++ b/dak/examine_package.py
@@ -470,7 +470,10 @@ def display_changes(suite, changes_filename):
     foldable_output(changes_filename, "changes", changes, norow=True)
 
 def check_changes (changes_filename):
-    changes = utils.parse_changes (changes_filename)
+    try:
+        changes = utils.parse_changes (changes_filename)
+    except ChangesUnicodeError:
+        utils.warn("Encoding problem with changes file %s" % (changes_filename))
     display_changes(changes['distribution'], changes_filename)
 
     files = utils.build_file_list(changes)

diff --git a/dak/process_accepted.py b/dak/process_accepted.py
index dce0cdbe8f0c40d502cc5fe03e37bde5a7d4422b..e53a969f48c1f35a88d292c3cb94d3ce86b3d855 100755 (executable)
--- a/dak/process_accepted.py
+++ b/dak/process_accepted.py
@@ -162,9 +162,11 @@ def init():
                  ('h',"help","Dinstall::Options::Help"),
                  ('n',"no-action","Dinstall::Options::No-Action"),
                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
-                 ('s',"no-mail", "Dinstall::Options::No-Mail")]
+                 ('s',"no-mail", "Dinstall::Options::No-Mail"),
+                 ('d',"directory", "Dinstall::Options::Directory")]
 
-    for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version"]:
+    for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
+              "version", "directory"]:
         if not Cnf.has_key("Dinstall::Options::%s" % (i)):
             Cnf["Dinstall::Options::%s" % (i)] = ""
 
@@ -174,6 +176,15 @@ def init():
     if Options["Help"]:
         usage()
 
+    # If we have a directory flag, use it to find our files
+    if Cnf["Dinstall::Options::Directory"] != "":
+        # Note that we clobber the list of files we were given in this case
+        # so warn if the user has done both
+        if len(changes_files) > 0:
+            utils.warn("Directory provided so ignoring files given on command line")
+
+        changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
+
     Upload = queue.Upload(Cnf)
     projectB = Upload.projectB
 
@@ -372,6 +383,9 @@ def install ():
                 suite_id = database.get_suite_id(suite)
                 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id))
 
+    orig_tar_id = Upload.pkg.orig_tar_id
+    orig_tar_location = Upload.pkg.orig_tar_location
+
     # If this is a sourceful diff only upload that is moving
     # cross-component we need to copy the .orig.tar.gz into the new
     # component too for the same reasons as above.

diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py
index 2e60435d1de032578c105dd20ee7bca8a1e20675..c2adb6393529d0ab6dc264f84d5b2933b79316d0 100755 (executable)
--- a/dak/process_unchecked.py
+++ b/dak/process_unchecked.py
@@ -78,10 +78,11 @@ def init():
                  ('h',"help","Dinstall::Options::Help"),
                  ('n',"no-action","Dinstall::Options::No-Action"),
                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
-                 ('s',"no-mail", "Dinstall::Options::No-Mail")]
+                 ('s',"no-mail", "Dinstall::Options::No-Mail"),
+                 ('d',"directory", "Dinstall::Options::Directory")]
 
     for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
-              "override-distribution", "version"]:
+              "override-distribution", "version", "directory"]:
         Cnf["Dinstall::Options::%s" % (i)] = ""
 
     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
@@ -90,6 +91,15 @@ def init():
     if Options["Help"]:
         usage()
 
+    # If we have a directory flag, use it to find our files
+    if Cnf["Dinstall::Options::Directory"] != "":
+        # Note that we clobber the list of files we were given in this case
+        # so warn if the user has done both
+        if len(changes_files) > 0:
+            utils.warn("Directory provided so ignoring files given on command line")
+
+        changes_files = utils.get_changes_files(Cnf["Dinstall::Options::Directory"])
+
     Upload = queue.Upload(Cnf)
 
     changes = Upload.pkg.changes
@@ -186,6 +196,9 @@ def check_changes():
     except ParseChangesError, line:
         reject("%s: parse error, can't grok: %s." % (filename, line))
         return 0
+    except ChangesUnicodeError:
+        reject("%s: changes file not proper utf-8" % (filename))
+        return 0
 
     # Parse the Files field from the .changes into another dictionary
     try:
@@ -695,6 +708,9 @@ def check_dsc():
         reject("%s: parse error, can't grok: %s." % (dsc_filename, line))
     except InvalidDscError, line:
         reject("%s: syntax error on line %s." % (dsc_filename, line))
+    except ChangesUnicodeError:
+        reject("%s: dsc file not proper utf-8." % (dsc_filename))
+
     # Build up the file list of files mentioned by the .dsc
     try:
         dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1))

diff --git a/daklib/dak_exceptions.py b/daklib/dak_exceptions.py
index d18bee1e6f1691c6940268175fddcd9fa18b1e17..33fa5ad3cacde8d62b3475914e03f4fa65ae9df4 100755 (executable)
--- a/daklib/dak_exceptions.py
+++ b/daklib/dak_exceptions.py
@@ -58,7 +58,8 @@ dakerrors = {
     "NoFreeFilenameError": """Exception raised when no alternate filename was found.""",
     "TransitionsError":    """Exception raised when transitions file can't be parsed.""",
     "NoSourceFieldError":  """Exception raised - we cant find the source - wtf?""",
-    "DBUpdateError":       """Exception raised - could not update the database"""
+    "DBUpdateError":       """Exception raised - could not update the database""",
+    "ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded"""
 } #: All dak exceptions
 
 def construct_dak_exception(name, description):

diff --git a/daklib/utils.py b/daklib/utils.py
index b20a063a452d4e9d1354f8e4a19c519205855295..fd790b5930df56d0e49804a13178db33f7bca090 100755 (executable)
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -234,6 +234,10 @@ def parse_changes(filename, signing_rules=0):
     changes_in = open_file(filename)
     content = changes_in.read()
     changes_in.close()
+    try:
+        unicode(content, 'utf-8')
+    except UnicodeError:
+        raise ChangesUnicodeError, "Changes file not proper utf-8"
     return parse_deb822(content, signing_rules)
 
 ################################################################################
@@ -1494,6 +1498,25 @@ def is_email_alias(email):
 
 ################################################################################
 
+def get_changes_files(dir):
+    """
+    Takes a directory and lists all .changes files in it (as well as chdir'ing
+    to the directory; this is due to broken behaviour on the part of p-u/p-a
+    when you're not in the right place)
+
+    Returns a list of filenames
+    """
+    try:
+        # Much of the rest of p-u/p-a depends on being in the right place
+        os.chdir(dir)
+        changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')]
+    except OSError, e:
+        fubar("Failed to read list from directory %s (%s)" % (dir, e))
+
+    return changes_files
+
+################################################################################
+
 apt_pkg.init()
 
 Cnf = apt_pkg.newConfiguration()