marge from master

[dak.git] / dak / clean_suites.py

diff --git a/dak/clean_suites.py b/dak/clean_suites.py
index 3445f5d45c8389fbca6c5726b6a463f85879f5e0..52b2a8ccb1f4504003ff487493305e9577ef707c 100755 (executable)
--- a/dak/clean_suites.py
+++ b/dak/clean_suites.py
@@ -74,7 +74,7 @@ SELECT b.file, f.filename FROM binaries b, files f
 
     # Check for any binaries which are marked for eventual deletion
     # but are now used again.
-      
+
     q = session.execute("""
 SELECT b.file, f.filename FROM binaries b, files f
    WHERE f.last_used IS NOT NULL AND f.id = b.file
@@ -86,7 +86,7 @@ SELECT b.file, f.filename FROM binaries b, files f
     session.commit()
 
 ########################################
-  
+
 def check_sources(now_date, delete_date, max_delete, session):
     print "Checking for orphaned source packages..."
 
@@ -142,7 +142,7 @@ SELECT f.id, f.filename FROM source s, files f, dsc_files df
     ####      reinstate sources because of them
 
     for i in q.fetchall():
-        Logger.log(["unset lastused", i[1]]) 
+        Logger.log(["unset lastused", i[1]])
         session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid",
                         {'fileid': i[0]})
 
@@ -163,13 +163,14 @@ def check_files(now_date, delete_date, max_delete, session):
 SELECT id, filename FROM files f
   WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = f.id)
     AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)
+    AND last_used IS NULL
     ORDER BY filename""")
 
     ql = q.fetchall()
     if len(ql) > 0:
-        print "WARNING: check_files found something it shouldn't"
+        utils.warn("check_files found something it shouldn't")
         for x in ql:
-            print x
+            utils.warn("orphaned file: %s" % x)
             Logger.log(["set lastused", x[1], "ORPHANED FILE"])
             session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid",
                             {'lastused': now_date, 'fileid': x[0]})
@@ -209,21 +210,20 @@ def clean(now_date, delete_date, max_delete, session):
     # Delete from source
     print "Deleting from source table... "
     q = session.execute("""
-SELECT df.id, s.id, f.filename FROM source s, files f, dsc_files df
+SELECT s.id, f.filename FROM source s, files f
   WHERE f.last_used <= :deletedate
-        AND s.file = f.id AND s.id = df.source
-        AND df.id = dsc_files.id)""", {'deletedate': delete_date})
+        AND s.file = f.id""", {'deletedate': delete_date})
     for s in q.fetchall():
-        Logger.log(["delete source", s[2]])
+        Logger.log(["delete source", s[1], s[0]])
         if not Options["No-Action"]:
-            session.execute("DELETE FROM dsc_files WHERE id = :dsc_id", {"dscid":s[0]})
-            session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[1]})
+            session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]})
+            session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]})
 
     if not Options["No-Action"]:
         session.commit()
 
     # Delete files from the pool
-    old_files = session.query(PoolFile).filter(last_used <= delete_date)
+    old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date)
     if max_delete is not None:
         old_files = old_files.limit(max_delete)
         print "Limiting removals to %d" % max_delete
@@ -255,7 +255,7 @@ SELECT df.id, s.id, f.filename FROM source s, files f, dsc_files df
 
             if not Options["No-Action"]:
                 session.delete(pf)
-            
+
         else:
             utils.fubar("%s is neither symlink nor file?!" % (filename))
 
@@ -263,6 +263,7 @@ SELECT df.id, s.id, f.filename FROM source s, files f, dsc_files df
         session.commit()
 
     if count > 0:
+        Logger.log(["total", count, utils.size_type(size)])
         print "Cleaned %d files, %s." % (count, utils.size_type(size))
 
 ################################################################################
@@ -290,6 +291,7 @@ SELECT m.id, m.name FROM maintainer m
         session.commit()
 
     if count > 0:
+        Logger.log(["total", count])
         print "Cleared out %d maintainer entries." % (count)
 
 ################################################################################
@@ -317,6 +319,7 @@ SELECT f.id, f.fingerprint FROM fingerprint f
         session.commit()
 
     if count > 0:
+        Logger.log(["total", count])
         print "Cleared out %d fingerprint entries." % (count)
 
 ################################################################################
@@ -333,7 +336,7 @@ def clean_queue_build(now_date, delete_date, max_delete, session):
     our_delete_date = now_date - timedelta(seconds = int(cnf["Clean-Suites::QueueBuildStayOfExecution"]))
     count = 0
 
-    for qf in session.query(QueueBuild).filter(last_used <= our_delete_date):
+    for qf in session.query(QueueBuild).filter(QueueBuild.last_used <= our_delete_date):
         if not os.path.exists(qf.filename):
             utils.warn("%s (from queue_build) doesn't exist." % (qf.filename))
             continue
@@ -351,10 +354,38 @@ def clean_queue_build(now_date, delete_date, max_delete, session):
         session.commit()
 
     if count:
+        Logger.log(["total", count])
         print "Cleaned %d queue_build files." % (count)
 
 ################################################################################
 
+def clean_empty_directories(session):
+    """
+    Removes empty directories from pool directories.
+    """
+
+    count = 0
+
+    cursor = session.execute(
+        "SELECT DISTINCT(path) FROM location WHERE type = :type",
+        {'type': 'pool'},
+    )
+    bases = [x[0] for x in cursor.fetchall()]
+
+    for base in bases:
+        for dirpath, dirnames, filenames in os.walk(base, topdown=False):
+            if not filenames and not dirnames:
+                to_remove = os.path.join(base, dirpath)
+                if not Options["No-Action"]:
+                    Logger.log(["removing directory", to_remove])
+                    os.removedirs(to_remove)
+                count += 1
+
+    if count:
+        Logger.log(["total removed directories", count])
+
+################################################################################
+
 def main():
     global Options, Logger
 
@@ -400,6 +431,7 @@ def main():
     clean_maintainers(now_date, delete_date, max_delete, session)
     clean_fingerprints(now_date, delete_date, max_delete, session)
     clean_queue_build(now_date, delete_date, max_delete, session)
+    clean_empty_directories(session)
 
     Logger.close()