3 """ Cleans up unassociated binary and source packages """
4 # Copyright (C) 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # 07:05|<elmo> well.. *shrug*.. no, probably not.. but to fix it,
23 # | we're going to have to implement reference counting
24 # | through dependencies.. do we really want to go down
27 # 07:05|<Culus> elmo: Augh! <brain jumps out of skull>
29 ################################################################################
31 import os, stat, sys, time
33 from datetime import datetime, timedelta
35 from daklib.config import Config
36 from daklib.dbconn import *
37 from daklib import utils
38 from daklib import daklog
40 ################################################################################
45 ################################################################################
47 def usage (exit_code=0):
48 print """Usage: dak clean-suites [OPTIONS]
49 Clean old packages from suites.
51 -n, --no-action don't do anything
52 -h, --help show this help and exit
53 -m, --maximum maximum number of files to remove"""
56 ################################################################################
58 def check_binaries(now_date, delete_date, max_delete, session):
59 print "Checking for orphaned binary packages..."
61 # Get the list of binary packages not in a suite and mark them for
64 q = session.execute("""
65 SELECT b.file, f.filename FROM binaries b, files f
66 WHERE f.last_used IS NULL AND b.file = f.id
67 AND NOT EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
69 for i in q.fetchall():
70 Logger.log(["set lastused", i[1]])
71 session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid AND last_used IS NULL",
72 {'lastused': now_date, 'fileid': i[0]})
75 # Check for any binaries which are marked for eventual deletion
76 # but are now used again.
78 q = session.execute("""
79 SELECT b.file, f.filename FROM binaries b, files f
80 WHERE f.last_used IS NOT NULL AND f.id = b.file
81 AND EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
83 for i in q.fetchall():
84 Logger.log(["unset lastused", i[1]])
85 session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid", {'fileid': i[0]})
88 ########################################
90 def check_sources(now_date, delete_date, max_delete, session):
91 print "Checking for orphaned source packages..."
93 # Get the list of source packages not in a suite and not used by
95 q = session.execute("""
96 SELECT s.id, s.file, f.filename FROM source s, files f
97 WHERE f.last_used IS NULL AND s.file = f.id
98 AND NOT EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id)
99 AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)""")
101 #### XXX: this should ignore cases where the files for the binary b
102 #### have been marked for deletion (so the delay between bins go
103 #### byebye and sources go byebye is 0 instead of StayOfExecution)
105 for i in q.fetchall():
110 # Mark the .dsc file for deletion
111 Logger.log(["set lastused", dsc_fname])
112 session.execute("""UPDATE files SET last_used = :last_used
113 WHERE id = :dscfileid AND last_used IS NULL""",
114 {'last_used': now_date, 'dscfileid': dsc_file_id})
116 # Mark all other files references by .dsc too if they're not used by anyone else
117 x = session.execute("""SELECT f.id, f.filename FROM files f, dsc_files d
118 WHERE d.source = :sourceid AND d.file = f.id""",
119 {'sourceid': source_id})
120 for j in x.fetchall():
123 y = session.execute("SELECT id FROM dsc_files d WHERE d.file = :fileid", {'fileid': file_id})
124 if len(y.fetchall()) == 1:
125 Logger.log(["set lastused", file_name])
126 session.execute("""UPDATE files SET last_used = :lastused
127 WHERE id = :fileid AND last_used IS NULL""",
128 {'lastused': now_date, 'fileid': file_id})
132 # Check for any sources which are marked for deletion but which
133 # are now used again.
135 q = session.execute("""
136 SELECT f.id, f.filename FROM source s, files f, dsc_files df
137 WHERE f.last_used IS NOT NULL AND s.id = df.source AND df.file = f.id
138 AND ((EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id))
139 OR (EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)))""")
141 #### XXX: this should also handle deleted binaries specially (ie, not
142 #### reinstate sources because of them
144 for i in q.fetchall():
145 Logger.log(["unset lastused", i[1]])
146 session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid",
151 ########################################
153 def check_files(now_date, delete_date, max_delete, session):
154 # FIXME: this is evil; nothing should ever be in this state. if
155 # they are, it's a bug.
157 # However, we've discovered it happens sometimes so we print a huge warning
158 # and then mark the file for deletion. This probably masks a bug somwhere
159 # else but is better than collecting cruft forever
161 print "Checking for unused files..."
162 q = session.execute("""
163 SELECT id, filename FROM files f
164 WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = f.id)
165 AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)
166 AND NOT EXISTS (SELECT 1 FROM changes_pool_files cpf WHERE cpf.fileid = f.id)
167 AND last_used IS NULL
168 ORDER BY filename""")
172 utils.warn("check_files found something it shouldn't")
174 utils.warn("orphaned file: %s" % x)
175 Logger.log(["set lastused", x[1], "ORPHANED FILE"])
176 session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid",
177 {'lastused': now_date, 'fileid': x[0]})
181 def clean_binaries(now_date, delete_date, max_delete, session):
182 # We do this here so that the binaries we remove will have their
183 # source also removed (if possible).
185 # XXX: why doesn't this remove the files here as well? I don't think it
186 # buys anything keeping this separate
187 print "Cleaning binaries from the DB..."
188 print "Deleting from binaries table... "
189 for bin in session.query(DBBinary).join(DBBinary.poolfile).filter(PoolFile.last_used <= delete_date):
190 Logger.log(["delete binary", bin.poolfile.filename])
191 if not Options["No-Action"]:
193 if not Options["No-Action"]:
196 ########################################
198 def clean(now_date, delete_date, max_delete, session):
204 print "Cleaning out packages..."
206 cur_date = now_date.strftime("%Y-%m-%d")
207 dest = os.path.join(cnf["Dir::Morgue"], cnf["Clean-Suites::MorgueSubDir"], cur_date)
208 if not os.path.exists(dest):
212 print "Deleting from source table... "
213 q = session.execute("""
214 SELECT s.id, f.filename FROM source s, files f
215 WHERE f.last_used <= :deletedate
216 AND s.file = f.id""", {'deletedate': delete_date})
217 for s in q.fetchall():
218 Logger.log(["delete source", s[1], s[0]])
219 if not Options["No-Action"]:
220 session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]})
221 session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]})
223 if not Options["No-Action"]:
226 # Delete files from the pool
227 old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date)
228 if max_delete is not None:
229 old_files = old_files.limit(max_delete)
230 print "Limiting removals to %d" % max_delete
233 filename = os.path.join(pf.location.path, pf.filename)
234 if not os.path.exists(filename):
235 utils.warn("can not find '%s'." % (filename))
237 Logger.log(["delete pool file", filename])
238 if os.path.isfile(filename):
239 if os.path.islink(filename):
241 Logger.log(["delete symlink", filename])
242 if not Options["No-Action"]:
245 size += os.stat(filename)[stat.ST_SIZE]
248 dest_filename = dest + '/' + os.path.basename(filename)
249 # If the destination file exists; try to find another filename to use
250 if os.path.exists(dest_filename):
251 dest_filename = utils.find_next_free(dest_filename)
253 Logger.log(["move to morgue", filename, dest_filename])
254 if not Options["No-Action"]:
255 utils.move(filename, dest_filename)
257 if not Options["No-Action"]:
261 utils.fubar("%s is neither symlink nor file?!" % (filename))
263 if not Options["No-Action"]:
267 Logger.log(["total", count, utils.size_type(size)])
268 print "Cleaned %d files, %s." % (count, utils.size_type(size))
270 ################################################################################
272 def clean_maintainers(now_date, delete_date, max_delete, session):
273 print "Cleaning out unused Maintainer entries..."
275 # TODO Replace this whole thing with one SQL statement
276 q = session.execute("""
277 SELECT m.id, m.name FROM maintainer m
278 WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id)
279 AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id OR s.changedby = m.id)
280 AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)""")
284 for i in q.fetchall():
286 Logger.log(["delete maintainer", i[1]])
287 if not Options["No-Action"]:
288 session.execute("DELETE FROM maintainer WHERE id = :maint", {'maint': maintainer_id})
291 if not Options["No-Action"]:
295 Logger.log(["total", count])
296 print "Cleared out %d maintainer entries." % (count)
298 ################################################################################
300 def clean_fingerprints(now_date, delete_date, max_delete, session):
301 print "Cleaning out unused fingerprint entries..."
303 # TODO Replace this whole thing with one SQL statement
304 q = session.execute("""
305 SELECT f.id, f.fingerprint FROM fingerprint f
306 WHERE f.keyring IS NULL
307 AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
308 AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""")
312 for i in q.fetchall():
313 fingerprint_id = i[0]
314 Logger.log(["delete fingerprint", i[1]])
315 if not Options["No-Action"]:
316 session.execute("DELETE FROM fingerprint WHERE id = :fpr", {'fpr': fingerprint_id})
319 if not Options["No-Action"]:
323 Logger.log(["total", count])
324 print "Cleared out %d fingerprint entries." % (count)
326 ################################################################################
328 def clean_queue_build(now_date, delete_date, max_delete, session):
332 if not cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]:
335 print "Cleaning out queue build symlinks..."
337 our_delete_date = now_date - timedelta(seconds = int(cnf["Clean-Suites::QueueBuildStayOfExecution"]))
340 for qf in session.query(QueueBuild).filter(QueueBuild.last_used <= our_delete_date):
341 if not os.path.exists(qf.filename):
342 utils.warn("%s (from queue_build) doesn't exist." % (qf.filename))
345 if not cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(qf.filename):
346 utils.fubar("%s (from queue_build) should be a symlink but isn't." % (qf.filename))
348 Logger.log(["delete queue build", qf.filename])
349 if not Options["No-Action"]:
350 os.unlink(qf.filename)
354 if not Options["No-Action"]:
358 Logger.log(["total", count])
359 print "Cleaned %d queue_build files." % (count)
361 ################################################################################
363 def clean_empty_directories(session):
365 Removes empty directories from pool directories.
370 cursor = session.execute(
371 "SELECT DISTINCT(path) FROM location WHERE type = :type",
374 bases = [x[0] for x in cursor.fetchall()]
377 for dirpath, dirnames, filenames in os.walk(base, topdown=False):
378 if not filenames and not dirnames:
379 to_remove = os.path.join(base, dirpath)
380 if not Options["No-Action"]:
381 Logger.log(["removing directory", to_remove])
382 os.removedirs(to_remove)
386 Logger.log(["total removed directories", count])
388 ################################################################################
391 global Options, Logger
395 for i in ["Help", "No-Action", "Maximum" ]:
396 if not cnf.has_key("Clean-Suites::Options::%s" % (i)):
397 cnf["Clean-Suites::Options::%s" % (i)] = ""
399 Arguments = [('h',"help","Clean-Suites::Options::Help"),
400 ('n',"no-action","Clean-Suites::Options::No-Action"),
401 ('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")]
403 apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
404 Options = cnf.SubTree("Clean-Suites::Options")
406 if cnf["Clean-Suites::Options::Maximum"] != "":
408 # Only use Maximum if it's an integer
409 max_delete = int(cnf["Clean-Suites::Options::Maximum"])
411 utils.fubar("If given, Maximum must be at least 1")
412 except ValueError, e:
413 utils.fubar("If given, Maximum must be an integer")
420 Logger = daklog.Logger(cnf, "clean-suites", debug=Options["No-Action"])
422 session = DBConn().session()
424 now_date = datetime.now()
425 delete_date = now_date - timedelta(seconds=int(cnf['Clean-Suites::StayOfExecution']))
427 check_binaries(now_date, delete_date, max_delete, session)
428 clean_binaries(now_date, delete_date, max_delete, session)
429 check_sources(now_date, delete_date, max_delete, session)
430 check_files(now_date, delete_date, max_delete, session)
431 clean(now_date, delete_date, max_delete, session)
432 clean_maintainers(now_date, delete_date, max_delete, session)
433 clean_fingerprints(now_date, delete_date, max_delete, session)
434 clean_queue_build(now_date, delete_date, max_delete, session)
435 clean_empty_directories(session)
439 ################################################################################
441 if __name__ == '__main__':