X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fclean_queues.py;h=88b0bb9334a712228e2b2ae27ca2cf53f5e016f1;hb=fdbef587c29814f97c192de5a8b7e9f09cc45fa4;hp=74a76055a02eab4c063a581be84e17df81c1cced;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git diff --git a/dak/clean_queues.py b/dak/clean_queues.py index 74a76055..88b0bb93 100755 --- a/dak/clean_queues.py +++ b/dak/clean_queues.py @@ -1,8 +1,7 @@ #!/usr/bin/env python -# Clean incoming of old unused files -# Copyright (C) 2000, 2001, 2002 James Troup -# $Id: shania,v 1.18 2005-03-06 21:51:51 rmurray Exp $ +""" Clean incoming of old unused files """ +# Copyright (C) 2000, 2001, 2002, 2006 James Troup # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,21 +33,25 @@ ################################################################################ -import os, stat, sys, time; -import utils; -import apt_pkg; +import os, os.path, stat, sys, time +from datetime import datetime, timedelta +import apt_pkg +from daklib import utils +from daklib import daklog +from daklib.config import Config +from daklib.dbconn import get_policy_queue ################################################################################ -Cnf = None; -Options = None; -del_dir = None; -delete_date = None; +Options = None +Logger = None +del_dir = None +delete_date = None ################################################################################ def usage (exit_code=0): - print """Usage: shania [OPTIONS] + print """Usage: dak clean-queues [OPTIONS] Clean out incoming directories. -d, --days=DAYS remove anything older than DAYS old @@ -61,150 +64,173 @@ Clean out incoming directories. ################################################################################ -def init (): - global delete_date, del_dir; +def init (cnf): + global delete_date, del_dir - delete_date = int(time.time())-(int(Options["Days"])*84600); + # Used for directory naming + now_date = datetime.now() + + # Used for working out times + delete_date = int(time.time())-(int(Options["Days"])*84600) + + morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) + morguesubdir = cnf.get("Clean-Queues::MorgueSubDir", 'queue') + + # Build directory as morguedir/morguesubdir/year/month/day + del_dir = os.path.join(morguedir, + morguesubdir, + str(now_date.year), + '%.2d' % now_date.month, + '%.2d' % now_date.day) # Ensure a directory exists to remove files to if not Options["No-Action"]: - date = time.strftime("%Y-%m-%d"); - del_dir = Cnf["Dir::Morgue"] + '/' + Cnf["Shania::MorgueSubDir"] + '/' + date; if not os.path.exists(del_dir): - os.makedirs(del_dir, 02775); + os.makedirs(del_dir, 0o2775) if not os.path.isdir(del_dir): - utils.fubar("%s must be a directory." % (del_dir)); + utils.fubar("%s must be a directory." % (del_dir)) # Move to the directory to clean - incoming = Options["Incoming"]; - if incoming == "": - incoming = Cnf["Dir::Queue::Unchecked"]; - os.chdir(incoming); + incoming = Options.get("Incoming") + if not incoming: + incoming = cnf.get('Dir::Unchecked') + if not incoming: + utils.fubar("Cannot find 'unchecked' directory") + + try: + os.chdir(incoming) + except OSError as e: + utils.fubar("Cannot chdir to %s" % incoming) # Remove a file to the morgue -def remove (file): - if os.access(file, os.R_OK): - dest_filename = del_dir + '/' + os.path.basename(file); +def remove (from_dir, f): + fname = os.path.basename(f) + if os.access(f, os.R_OK): + Logger.log(["move file to morgue", from_dir, fname, del_dir]) + if Options["Verbose"]: + print "Removing '%s' (to '%s')." % (fname, del_dir) + if Options["No-Action"]: + return + + dest_filename = os.path.join(del_dir, fname) # If the destination file exists; try to find another filename to use if os.path.exists(dest_filename): - dest_filename = utils.find_next_free(dest_filename, 10); - utils.move(file, dest_filename, 0660); + dest_filename = utils.find_next_free(dest_filename, 10) + Logger.log(["change destination file name", os.path.basename(dest_filename)]) + utils.move(f, dest_filename, 0o660) else: - utils.warn("skipping '%s', permission denied." % (os.path.basename(file))); + Logger.log(["skipping file because of permission problem", fname]) + utils.warn("skipping '%s', permission denied." % fname) # Removes any old files. # [Used for Incoming/REJECT] # def flush_old (): - for file in os.listdir('.'): - if os.path.isfile(file): - if os.stat(file)[stat.ST_MTIME] < delete_date: - if Options["No-Action"]: - print "I: Would delete '%s'." % (os.path.basename(file)); - else: - if Options["Verbose"]: - print "Removing '%s' (to '%s')." % (os.path.basename(file), del_dir); - remove(file); + Logger.log(["check Incoming/REJECT for old files", os.getcwd()]) + for f in os.listdir('.'): + if os.path.isfile(f): + if os.stat(f)[stat.ST_MTIME] < delete_date: + remove('Incoming/REJECT', f) else: if Options["Verbose"]: - print "Skipping, too new, '%s'." % (os.path.basename(file)); + print "Skipping, too new, '%s'." % (os.path.basename(f)) # Removes any files which are old orphans (not associated with a valid .changes file). # [Used for Incoming] # def flush_orphans (): - all_files = {}; - changes_files = []; + all_files = {} + changes_files = [] + Logger.log(["check Incoming for old orphaned files", os.getcwd()]) # Build up the list of all files in the directory for i in os.listdir('.'): if os.path.isfile(i): - all_files[i] = 1; + all_files[i] = 1 if i.endswith(".changes"): - changes_files.append(i); + changes_files.append(i) # Proces all .changes and .dsc files. for changes_filename in changes_files: try: - changes = utils.parse_changes(changes_filename); - files = utils.build_file_list(changes); + changes = utils.parse_changes(changes_filename) + files = utils.build_file_list(changes) except: - utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_type)); - continue; + utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_info()[0])) + continue - dsc_files = {}; - for file in files.keys(): - if file.endswith(".dsc"): + dsc_files = {} + for f in files.keys(): + if f.endswith(".dsc"): try: - dsc = utils.parse_changes(file); - dsc_files = utils.build_file_list(dsc, is_a_dsc=1); + dsc = utils.parse_changes(f, dsc_file=1) + dsc_files = utils.build_file_list(dsc, is_a_dsc=1) except: - utils.warn("error processing '%s'; skipping it. [Got %s]" % (file, sys.exc_type)); - continue; + utils.warn("error processing '%s'; skipping it. [Got %s]" % (f, sys.exc_info()[0])) + continue # Ensure all the files we've seen aren't deleted - keys = []; + keys = [] for i in (files.keys(), dsc_files.keys(), [changes_filename]): - keys.extend(i); + keys.extend(i) for key in keys: if all_files.has_key(key): if Options["Verbose"]: - print "Skipping, has parents, '%s'." % (key); - del all_files[key]; + print "Skipping, has parents, '%s'." % (key) + del all_files[key] # Anthing left at this stage is not referenced by a .changes (or # a .dsc) and should be deleted if old enough. - for file in all_files.keys(): - if os.stat(file)[stat.ST_MTIME] < delete_date: - if Options["No-Action"]: - print "I: Would delete '%s'." % (os.path.basename(file)); - else: - if Options["Verbose"]: - print "Removing '%s' (to '%s')." % (os.path.basename(file), del_dir); - remove(file); + for f in all_files.keys(): + if os.stat(f)[stat.ST_MTIME] < delete_date: + remove('Incoming', f) else: if Options["Verbose"]: - print "Skipping, too new, '%s'." % (os.path.basename(file)); + print "Skipping, too new, '%s'." % (os.path.basename(f)) ################################################################################ def main (): - global Cnf, Options; + global Options, Logger - Cnf = utils.get_conf() + cnf = Config() for i in ["Help", "Incoming", "No-Action", "Verbose" ]: - if not Cnf.has_key("Shania::Options::%s" % (i)): - Cnf["Shania::Options::%s" % (i)] = ""; - if not Cnf.has_key("Shania::Options::Days"): - Cnf["Shania::Options::Days"] = "14"; + if not cnf.has_key("Clean-Queues::Options::%s" % (i)): + cnf["Clean-Queues::Options::%s" % (i)] = "" + if not cnf.has_key("Clean-Queues::Options::Days"): + cnf["Clean-Queues::Options::Days"] = "14" - Arguments = [('h',"help","Shania::Options::Help"), - ('d',"days","Shania::Options::Days", "IntLevel"), - ('i',"incoming","Shania::Options::Incoming", "HasArg"), - ('n',"no-action","Shania::Options::No-Action"), - ('v',"verbose","Shania::Options::Verbose")]; + Arguments = [('h',"help","Clean-Queues::Options::Help"), + ('d',"days","Clean-Queues::Options::Days", "IntLevel"), + ('i',"incoming","Clean-Queues::Options::Incoming", "HasArg"), + ('n',"no-action","Clean-Queues::Options::No-Action"), + ('v',"verbose","Clean-Queues::Options::Verbose")] - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Options = Cnf.SubTree("Shania::Options") + apt_pkg.parse_commandline(cnf.Cnf,Arguments,sys.argv) + Options = cnf.subtree("Clean-Queues::Options") if Options["Help"]: - usage(); + usage() - init(); + Logger = daklog.Logger('clean-queues', Options['No-Action']) + + init(cnf) if Options["Verbose"]: print "Processing incoming..." - flush_orphans(); + flush_orphans() - reject = Cnf["Dir::Queue::Reject"] + reject = cnf["Dir::Reject"] if os.path.exists(reject) and os.path.isdir(reject): if Options["Verbose"]: - print "Processing incoming/REJECT..." - os.chdir(reject); - flush_old(); + print "Processing reject directory..." + os.chdir(reject) + flush_old() + + Logger.close() ####################################################################################### if __name__ == '__main__': - main(); + main()