3 """ Cleans up unassociated binary and source packages """
4 # Copyright (C) 2000, 2001, 2002, 2003, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # 07:05|<elmo> well.. *shrug*.. no, probably not.. but to fix it,
23 # | we're going to have to implement reference counting
24 # | through dependencies.. do we really want to go down
27 # 07:05|<Culus> elmo: Augh! <brain jumps out of skull>
29 ################################################################################
31 import os, stat, sys, time
33 from datetime import datetime, timedelta
35 from daklib.config import Config
36 from daklib.dbconn import *
37 from daklib import utils
38 from daklib import daklog
40 ################################################################################
45 ################################################################################
47 def usage (exit_code=0):
48 print """Usage: dak clean-suites [OPTIONS]
49 Clean old packages from suites.
51 -n, --no-action don't do anything
52 -h, --help show this help and exit
53 -m, --maximum maximum number of files to remove"""
56 ################################################################################
58 def check_binaries(now_date, delete_date, max_delete, session):
59 print "Checking for orphaned binary packages..."
61 # Get the list of binary packages not in a suite and mark them for
64 q = session.execute("""
65 SELECT b.file, f.filename FROM binaries b, files f
66 WHERE f.last_used IS NULL AND b.file = f.id
67 AND NOT EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
69 for i in q.fetchall():
70 Logger.log(["set lastused", i[1]])
71 session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid AND last_used IS NULL",
72 {'lastused': now_date, 'fileid': i[0]})
75 # Check for any binaries which are marked for eventual deletion
76 # but are now used again.
78 q = session.execute("""
79 SELECT b.file, f.filename FROM binaries b, files f
80 WHERE f.last_used IS NOT NULL AND f.id = b.file
81 AND EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""")
83 for i in q.fetchall():
84 Logger.log(["unset lastused", i[1]])
85 session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid", {'fileid': i[0]})
88 ########################################
90 def check_sources(now_date, delete_date, max_delete, session):
91 print "Checking for orphaned source packages..."
93 # Get the list of source packages not in a suite and not used by
95 q = session.execute("""
96 SELECT s.id, s.file, f.filename FROM source s, files f
97 WHERE f.last_used IS NULL AND s.file = f.id
98 AND NOT EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id)
99 AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)""")
101 #### XXX: this should ignore cases where the files for the binary b
102 #### have been marked for deletion (so the delay between bins go
103 #### byebye and sources go byebye is 0 instead of StayOfExecution)
105 for i in q.fetchall():
110 # Mark the .dsc file for deletion
111 Logger.log(["set lastused", dsc_fname])
112 session.execute("""UPDATE files SET last_used = :last_used
113 WHERE id = :dscfileid AND last_used IS NULL""",
114 {'last_used': now_date, 'dscfileid': dsc_file_id})
116 # Mark all other files references by .dsc too if they're not used by anyone else
117 x = session.execute("""SELECT f.id, f.filename FROM files f, dsc_files d
118 WHERE d.source = :sourceid AND d.file = f.id""",
119 {'sourceid': source_id})
120 for j in x.fetchall():
123 y = session.execute("SELECT id FROM dsc_files d WHERE d.file = :fileid", {'fileid': file_id})
124 if len(y.fetchall()) == 1:
125 Logger.log(["set lastused", file_name])
126 session.execute("""UPDATE files SET last_used = :lastused
127 WHERE id = :fileid AND last_used IS NULL""",
128 {'lastused': now_date, 'fileid': file_id})
132 # Check for any sources which are marked for deletion but which
133 # are now used again.
135 q = session.execute("""
136 SELECT f.id, f.filename FROM source s, files f, dsc_files df
137 WHERE f.last_used IS NOT NULL AND s.id = df.source AND df.file = f.id
138 AND ((EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id))
139 OR (EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)))""")
141 #### XXX: this should also handle deleted binaries specially (ie, not
142 #### reinstate sources because of them
144 for i in q.fetchall():
145 Logger.log(["unset lastused", i[1]])
146 session.execute("UPDATE files SET last_used = NULL WHERE id = :fileid",
151 ########################################
153 def check_files(now_date, delete_date, max_delete, session):
154 # FIXME: this is evil; nothing should ever be in this state. if
155 # they are, it's a bug.
157 # However, we've discovered it happens sometimes so we print a huge warning
158 # and then mark the file for deletion. This probably masks a bug somwhere
159 # else but is better than collecting cruft forever
161 print "Checking for unused files..."
162 q = session.execute("""
163 SELECT id, filename FROM files f
164 WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = f.id)
165 AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)
166 ORDER BY filename""")
170 utils.warn("check_files found something it shouldn't")
172 utils.warn("orphaned file: %s" % x)
173 Logger.log(["set lastused", x[1], "ORPHANED FILE"])
174 session.execute("UPDATE files SET last_used = :lastused WHERE id = :fileid",
175 {'lastused': now_date, 'fileid': x[0]})
179 def clean_binaries(now_date, delete_date, max_delete, session):
180 # We do this here so that the binaries we remove will have their
181 # source also removed (if possible).
183 # XXX: why doesn't this remove the files here as well? I don't think it
184 # buys anything keeping this separate
185 print "Cleaning binaries from the DB..."
186 print "Deleting from binaries table... "
187 for bin in session.query(DBBinary).join(DBBinary.poolfile).filter(PoolFile.last_used <= delete_date):
188 Logger.log(["delete binary", bin.poolfile.filename])
189 if not Options["No-Action"]:
191 if not Options["No-Action"]:
194 ########################################
196 def clean(now_date, delete_date, max_delete, session):
202 print "Cleaning out packages..."
204 cur_date = now_date.strftime("%Y-%m-%d")
205 dest = os.path.join(cnf["Dir::Morgue"], cnf["Clean-Suites::MorgueSubDir"], cur_date)
206 if not os.path.exists(dest):
210 print "Deleting from source table... "
211 q = session.execute("""
212 SELECT s.id, f.filename FROM source s, files f
213 WHERE f.last_used <= :deletedate
214 AND s.file = f.id""", {'deletedate': delete_date})
215 for s in q.fetchall():
216 Logger.log(["delete source", s[1], s[0]])
217 if not Options["No-Action"]:
218 session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]})
219 session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]})
221 if not Options["No-Action"]:
224 # Delete files from the pool
225 old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date)
226 if max_delete is not None:
227 old_files = old_files.limit(max_delete)
228 print "Limiting removals to %d" % max_delete
231 filename = os.path.join(pf.location.path, pf.filename)
232 if not os.path.exists(filename):
233 utils.warn("can not find '%s'." % (filename))
235 Logger.log(["delete pool file", filename])
236 if os.path.isfile(filename):
237 if os.path.islink(filename):
239 Logger.log(["delete symlink", filename])
240 if not Options["No-Action"]:
243 size += os.stat(filename)[stat.ST_SIZE]
246 dest_filename = dest + '/' + os.path.basename(filename)
247 # If the destination file exists; try to find another filename to use
248 if os.path.exists(dest_filename):
249 dest_filename = utils.find_next_free(dest_filename)
251 Logger.log(["move to morgue", filename, dest_filename])
252 if not Options["No-Action"]:
253 utils.move(filename, dest_filename)
255 if not Options["No-Action"]:
259 utils.fubar("%s is neither symlink nor file?!" % (filename))
261 if not Options["No-Action"]:
265 print "Cleaned %d files, %s." % (count, utils.size_type(size))
267 ################################################################################
269 def clean_maintainers(now_date, delete_date, max_delete, session):
270 print "Cleaning out unused Maintainer entries..."
272 # TODO Replace this whole thing with one SQL statement
273 q = session.execute("""
274 SELECT m.id, m.name FROM maintainer m
275 WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id)
276 AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id OR s.changedby = m.id)
277 AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)""")
281 for i in q.fetchall():
283 Logger.log(["delete maintainer", i[1]])
284 if not Options["No-Action"]:
285 session.execute("DELETE FROM maintainer WHERE id = :maint", {'maint': maintainer_id})
288 if not Options["No-Action"]:
292 print "Cleared out %d maintainer entries." % (count)
294 ################################################################################
296 def clean_fingerprints(now_date, delete_date, max_delete, session):
297 print "Cleaning out unused fingerprint entries..."
299 # TODO Replace this whole thing with one SQL statement
300 q = session.execute("""
301 SELECT f.id, f.fingerprint FROM fingerprint f
302 WHERE f.keyring IS NULL
303 AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
304 AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""")
308 for i in q.fetchall():
309 fingerprint_id = i[0]
310 Logger.log(["delete fingerprint", i[1]])
311 if not Options["No-Action"]:
312 session.execute("DELETE FROM fingerprint WHERE id = :fpr", {'fpr': fingerprint_id})
315 if not Options["No-Action"]:
319 print "Cleared out %d fingerprint entries." % (count)
321 ################################################################################
323 def clean_queue_build(now_date, delete_date, max_delete, session):
327 if not cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]:
330 print "Cleaning out queue build symlinks..."
332 our_delete_date = now_date - timedelta(seconds = int(cnf["Clean-Suites::QueueBuildStayOfExecution"]))
335 for qf in session.query(QueueBuild).filter(QueueBuild.last_used <= our_delete_date):
336 if not os.path.exists(qf.filename):
337 utils.warn("%s (from queue_build) doesn't exist." % (qf.filename))
340 if not cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(qf.filename):
341 utils.fubar("%s (from queue_build) should be a symlink but isn't." % (qf.filename))
343 Logger.log(["delete queue build", qf.filename])
344 if not Options["No-Action"]:
345 os.unlink(qf.filename)
349 if not Options["No-Action"]:
353 print "Cleaned %d queue_build files." % (count)
355 ################################################################################
358 global Options, Logger
362 for i in ["Help", "No-Action", "Maximum" ]:
363 if not cnf.has_key("Clean-Suites::Options::%s" % (i)):
364 cnf["Clean-Suites::Options::%s" % (i)] = ""
366 Arguments = [('h',"help","Clean-Suites::Options::Help"),
367 ('n',"no-action","Clean-Suites::Options::No-Action"),
368 ('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")]
370 apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
371 Options = cnf.SubTree("Clean-Suites::Options")
373 if cnf["Clean-Suites::Options::Maximum"] != "":
375 # Only use Maximum if it's an integer
376 max_delete = int(cnf["Clean-Suites::Options::Maximum"])
378 utils.fubar("If given, Maximum must be at least 1")
379 except ValueError, e:
380 utils.fubar("If given, Maximum must be an integer")
387 Logger = daklog.Logger(cnf, "clean-suites", debug=Options["No-Action"])
389 session = DBConn().session()
391 now_date = datetime.now()
392 delete_date = now_date - timedelta(seconds=int(cnf['Clean-Suites::StayOfExecution']))
394 check_binaries(now_date, delete_date, max_delete, session)
395 clean_binaries(now_date, delete_date, max_delete, session)
396 check_sources(now_date, delete_date, max_delete, session)
397 check_files(now_date, delete_date, max_delete, session)
398 clean(now_date, delete_date, max_delete, session)
399 clean_maintainers(now_date, delete_date, max_delete, session)
400 clean_fingerprints(now_date, delete_date, max_delete, session)
401 clean_queue_build(now_date, delete_date, max_delete, session)
405 ################################################################################
407 if __name__ == '__main__':