################################################################################
-def check_binaries(now_date, delete_date, max_delete, session):
- print "Checking for orphaned binary packages..."
+def check_binaries(now_date, session):
+ Logger.log(["Checking for orphaned binary packages..."])
# Get the list of binary packages not in a suite and mark them for
# deletion.
########################################
-def check_sources(now_date, delete_date, max_delete, session):
- print "Checking for orphaned source packages..."
+def check_sources(now_date, session):
+ Logger.log(["Checking for orphaned source packages..."])
# Get the list of source packages not in a suite and not used by
# any binaries.
- #### XXX: this should ignore cases where the files for the binary b
- #### have been marked for deletion (so the delay between bins go
- #### byebye and sources go byebye is 0 instead of StayOfExecution)
-
# Check for any sources which are marked for deletion but which
# are now used again.
- #### XXX: this should also handle deleted binaries specially (ie, not
- #### reinstate sources because of them
-
# TODO: the UPDATE part is the same as in check_binaries. Merge?
query = """
OR EXISTS (SELECT 1 FROM files_archive_map af_bin
JOIN binaries b ON af_bin.file_id = b.file
WHERE b.source = df.source
- AND af_bin.archive_id = af.archive_id)
+ AND af_bin.archive_id = af.archive_id
+ AND (af_bin.last_used IS NULL OR af_bin.last_used > ad.delete_date))
OR EXISTS (SELECT 1 FROM extra_src_references esr
JOIN bin_associations ba ON esr.bin_id = ba.bin
JOIN binaries b ON ba.bin = b.id
AS in_use
FROM files_archive_map af
JOIN dsc_files df ON af.file_id = df.file
+ JOIN archive_delete_date ad ON af.archive_id = ad.archive_id
GROUP BY af.archive_id, af.file_id, af.component_id
)
########################################
-def check_files(now_date, delete_date, max_delete, session):
+def check_files(now_date, session):
# FIXME: this is evil; nothing should ever be in this state. if
# they are, it's a bug.
# and then mark the file for deletion. This probably masks a bug somwhere
# else but is better than collecting cruft forever
- print "Checking for unused files..."
+ Logger.log(["Checking for unused files..."])
q = session.execute("""
UPDATE files_archive_map af
SET last_used = :last_used
if not Options["No-Action"]:
session.commit()
-def clean_binaries(now_date, delete_date, max_delete, session):
+def clean_binaries(now_date, session):
# We do this here so that the binaries we remove will have their
# source also removed (if possible).
# XXX: why doesn't this remove the files here as well? I don't think it
# buys anything keeping this separate
- print "Deleting from binaries table... "
+ Logger.log(["Deleting from binaries table... "])
q = session.execute("""
DELETE FROM binaries b
USING files f
WHERE f.id = b.file
AND NOT EXISTS (SELECT 1 FROM files_archive_map af
+ JOIN archive_delete_date ad ON af.archive_id = ad.archive_id
WHERE af.file_id = b.file
- AND (af.last_used IS NULL OR af.last_used >= :delete_date))
+ AND (af.last_used IS NULL OR af.last_used > ad.delete_date))
RETURNING f.filename
- """, {'delete_date': delete_date})
+ """)
for b in q:
Logger.log(["delete binary", b[0]])
########################################
-def clean(now_date, delete_date, max_delete, session):
+def clean(now_date, archives, max_delete, session):
cnf = Config()
count = 0
size = 0
- print "Cleaning out packages..."
+ Logger.log(["Cleaning out packages..."])
morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue'))
morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool')
os.makedirs(dest)
# Delete from source
- print "Deleting from source table... "
+ Logger.log(["Deleting from source table..."])
q = session.execute("""
WITH
deleted_sources AS (
USING files f
WHERE source.file = f.id
AND NOT EXISTS (SELECT 1 FROM files_archive_map af
+ JOIN archive_delete_date ad ON af.archive_id = ad.archive_id
WHERE af.file_id = source.file
- AND (af.last_used IS NULL OR af.last_used >= :delete_date))
+ AND (af.last_used IS NULL OR af.last_used > ad.delete_date))
RETURNING source.id AS id, f.filename AS filename
),
deleted_dsc_files AS (
),
now_unused_source_files AS (
UPDATE files_archive_map af
- SET last_used = :delete_date -- Kill it now. We waited long enough before removing the .dsc.
+ SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc.
WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files)
AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id)
)
- SELECT filename FROM deleted_sources""",
- {'delete_date': delete_date})
+ SELECT filename FROM deleted_sources""")
for s in q:
Logger.log(["delete source", s[0]])
session.commit()
# Delete files from the pool
- old_files = session.query(ArchiveFile).filter(ArchiveFile.last_used <= delete_date).join(Archive)
+ old_files = session.query(ArchiveFile).filter('files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)').join(Archive)
if max_delete is not None:
old_files = old_files.limit(max_delete)
- print "Limiting removals to %d" % max_delete
+ Logger.log(["Limiting removals to %d" % max_delete])
+
+ if archives is not None:
+ archive_ids = [ a.archive_id for a in archives ]
+ old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids))
for af in old_files:
filename = af.path
dest_filename = dest + '/' + os.path.basename(filename)
# If the destination file exists; try to find another filename to use
- if os.path.exists(dest_filename):
+ if os.path.lexists(dest_filename):
dest_filename = utils.find_next_free(dest_filename)
if not Options["No-Action"]:
if count > 0:
Logger.log(["total", count, utils.size_type(size)])
- print "Cleaned %d files, %s." % (count, utils.size_type(size))
# Delete entries in files no longer referenced by any archive
query = """
################################################################################
-def clean_maintainers(now_date, delete_date, max_delete, session):
- print "Cleaning out unused Maintainer entries..."
+def clean_maintainers(now_date, session):
+ Logger.log(["Cleaning out unused Maintainer entries..."])
# TODO Replace this whole thing with one SQL statement
q = session.execute("""
if count > 0:
Logger.log(["total", count])
- print "Cleared out %d maintainer entries." % (count)
################################################################################
-def clean_fingerprints(now_date, delete_date, max_delete, session):
- print "Cleaning out unused fingerprint entries..."
+def clean_fingerprints(now_date, session):
+ Logger.log(["Cleaning out unused fingerprint entries..."])
# TODO Replace this whole thing with one SQL statement
q = session.execute("""
SELECT f.id, f.fingerprint FROM fingerprint f
WHERE f.keyring IS NULL
AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
- AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""")
+ AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)
+ AND NOT EXISTS (SELECT 1 FROM acl_per_source aps WHERE aps.created_by_id = f.id)""")
count = 0
if count > 0:
Logger.log(["total", count])
- print "Cleared out %d fingerprint entries." % (count)
################################################################################
Removes empty directories from pool directories.
"""
- print "Cleaning out empty directories..."
+ Logger.log(["Cleaning out empty directories..."])
count = 0
################################################################################
+def set_archive_delete_dates(now_date, session):
+ session.execute("""
+ CREATE TEMPORARY TABLE archive_delete_date (
+ archive_id INT NOT NULL,
+ delete_date TIMESTAMP NOT NULL
+ )""")
+
+ session.execute("""
+ INSERT INTO archive_delete_date
+ (archive_id, delete_date)
+ SELECT
+ archive.id, :now_date - archive.stayofexecution
+ FROM archive""", {'now_date': now_date})
+
+ session.flush()
+
+################################################################################
+
def main():
global Options, Logger
cnf["Clean-Suites::Options::%s" % (i)] = ""
Arguments = [('h',"help","Clean-Suites::Options::Help"),
+ ('a','archive','Clean-Suites::Options::Archive','HasArg'),
('n',"no-action","Clean-Suites::Options::No-Action"),
('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")]
if Options["Help"]:
usage()
- Logger = daklog.Logger("clean-suites", debug=Options["No-Action"])
+ program = "clean-suites"
+ if Options['No-Action']:
+ program = "clean-suites (no action)"
+ Logger = daklog.Logger(program, debug=Options["No-Action"])
session = DBConn().session()
- now_date = datetime.now()
+ archives = None
+ if 'Archive' in Options:
+ archive_names = Options['Archive'].split(',')
+ archives = session.query(Archive).filter(Archive.archive_name.in_(archive_names)).all()
+ if len(archives) == 0:
+ utils.fubar('Unknown archive.')
- # Stay of execution; default to 1.5 days
- soe = int(cnf.get('Clean-Suites::StayOfExecution', '129600'))
+ now_date = datetime.now()
- delete_date = now_date - timedelta(seconds=soe)
+ set_archive_delete_dates(now_date, session)
- check_binaries(now_date, delete_date, max_delete, session)
- clean_binaries(now_date, delete_date, max_delete, session)
- check_sources(now_date, delete_date, max_delete, session)
- check_files(now_date, delete_date, max_delete, session)
- clean(now_date, delete_date, max_delete, session)
- clean_maintainers(now_date, delete_date, max_delete, session)
- clean_fingerprints(now_date, delete_date, max_delete, session)
+ check_binaries(now_date, session)
+ clean_binaries(now_date, session)
+ check_sources(now_date, session)
+ check_files(now_date, session)
+ clean(now_date, archives, max_delete, session)
+ clean_maintainers(now_date, session)
+ clean_fingerprints(now_date, session)
clean_empty_directories(session)
session.rollback()
projects / dak.git / blobdiff