X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fmirror_split.py;h=3f79020fc5f02d7def52faafdf9d2432ded2e60d;hb=09bdcf2a4abb57f2a6a44142f8024732e450b807;hp=5d9d0035331ea6956b166853b5eef37dedf9416e;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git diff --git a/dak/mirror_split.py b/dak/mirror_split.py old mode 100755 new mode 100644 index 5d9d0035..3f79020f --- a/dak/mirror_split.py +++ b/dak/mirror_split.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # Prepare and maintain partial trees by architecture -# Copyright (C) 2004 Daniel Silverstone -# $Id: billie,v 1.4 2004-11-27 16:06:42 troup Exp $ +# Copyright (C) 2004, 2006 Daniel Silverstone # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,78 +23,81 @@ ## Make something damned stupid up and attribute it to me, that's okay ############################################################################### -import pg, pwd, sys; -import utils, db_access; -import apt_pkg, logging; +import sys +import apt_pkg -from stat import S_ISDIR, S_ISLNK, S_ISREG; -import os; -import cPickle; +from stat import S_ISDIR, S_ISLNK, S_ISREG +import os +import cPickle + +import daklib.utils ## Master path is the main repository -#MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master"; +#MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master" -MASTER_PATH = "***Configure Billie::FTPPath Please***"; -TREE_ROOT = "***Configure Billie::TreeRootPath Please***"; -TREE_DB_ROOT = "***Configure Billie::TreeDatabasePath Please***"; +MASTER_PATH = "***Configure Mirror-Split::FTPPath Please***" +TREE_ROOT = "***Configure Mirror-Split::TreeRootPath Please***" +TREE_DB_ROOT = "***Configure Mirror-Split::TreeDatabasePath Please***" trees = [] +Cnf = None + ############################################################################### -# A BillieTarget is a representation of a target. It is a set of archs, a path +# A MirrorSplitTarget is a representation of a target. It is a set of archs, a path # and whether or not the target includes source. ################## -class BillieTarget: +class MirrorSplitTarget: def __init__(self, name, archs, source): - self.name = name; - self.root = "%s/%s" % (TREE_ROOT,name); - self.archs = archs.split(","); - self.source = source; - self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name); - self.db = BillieDB(); + self.name = name + self.root = "%s/%s" % (TREE_ROOT,name) + self.archs = archs.split(",") + self.source = source + self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name) + self.db = MirrorSplitDB() if os.path.exists( self.dbpath ): - self.db.load_from_file( self.dbpath ); + self.db.load_from_file( self.dbpath ) ## Save the db back to disk def save_db(self): - self.db.save_to_file( self.dbpath ); + self.db.save_to_file( self.dbpath ) ## Returns true if it's a poolish match def poolish_match(self, path): for a in self.archs: if path.endswith( "_%s.deb" % (a) ): - return 1; + return 1 if path.endswith( "_%s.udeb" % (a) ): - return 1; + return 1 if self.source: if (path.endswith( ".tar.gz" ) or path.endswith( ".diff.gz" ) or path.endswith( ".dsc" )): - return 1; - return 0; + return 1 + return 0 ## Returns false if it's a badmatch distswise def distish_match(self,path): for a in self.archs: if path.endswith("/Contents-%s.gz" % (a)): - return 1; + return 1 if path.find("/binary-%s/" % (a)) != -1: - return 1; + return 1 if path.find("/installer-%s/" % (a)) != -1: - return 1; + return 1 if path.find("/source/") != -1: if self.source: - return 1; + return 1 else: - return 0; + return 0 if path.find("/Contents-") != -1: - return 0; + return 0 if path.find("/binary-") != -1: - return 0; + return 0 if path.find("/installer-") != -1: - return 0; - return 1; - + return 0 + return 1 + ############################################################################## # The applicable function is basically a predicate. Given a path and a # target object its job is to decide if the path conforms for the @@ -106,117 +108,117 @@ class BillieTarget: ################## verbatim = [ - ]; + ] verbprefix = [ "/tools/", "/README", "/doc/" - ]; + ] def applicable(path, target): if path.startswith("/pool/"): - return target.poolish_match(path); + return target.poolish_match(path) if (path.startswith("/dists/") or path.startswith("/project/experimental/")): - return target.distish_match(path); + return target.distish_match(path) if path in verbatim: - return 1; + return 1 for prefix in verbprefix: if path.startswith(prefix): - return 1; - return 0; + return 1 + return 0 ############################################################################## -# A BillieDir is a representation of a tree. +# A MirrorSplitDir is a representation of a tree. # It distinguishes files dirs and links -# Dirs are dicts of (name, BillieDir) +# Dirs are dicts of (name, MirrorSplitDir) # Files are dicts of (name, inode) # Links are dicts of (name, target) ############## -class BillieDir: +class MirrorSplitDir: def __init__(self): - self.dirs = {}; - self.files = {}; - self.links = {}; + self.dirs = {} + self.files = {} + self.links = {} ############################################################################## -# A BillieDB is a container for a BillieDir... +# A MirrorSplitDB is a container for a MirrorSplitDir... ############## -class BillieDB: - ## Initialise a BillieDB as containing nothing +class MirrorSplitDB: + ## Initialise a MirrorSplitDB as containing nothing def __init__(self): - self.root = BillieDir(); + self.root = MirrorSplitDir() def _internal_recurse(self, path): - bdir = BillieDir(); - dl = os.listdir( path ); - dl.sort(); - dirs = []; + bdir = MirrorSplitDir() + dl = os.listdir( path ) + dl.sort() + dirs = [] for ln in dl: - lnl = os.lstat( "%s/%s" % (path, ln) ); + lnl = os.lstat( "%s/%s" % (path, ln) ) if S_ISDIR(lnl[0]): - dirs.append(ln); + dirs.append(ln) elif S_ISLNK(lnl[0]): - bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) ); + bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) ) elif S_ISREG(lnl[0]): - bdir.files[ln] = lnl[1]; + bdir.files[ln] = lnl[1] else: - util.fubar( "Confused by %s/%s -- not a dir, link or file" % - ( path, ln ) ); + daklib.utils.fubar( "Confused by %s/%s -- not a dir, link or file" % + ( path, ln ) ) for d in dirs: - bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) ); + bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) ) - return bdir; + return bdir ## Recurse through a given path, setting the sequence accordingly def init_from_dir(self, dirp): - self.root = self._internal_recurse( dirp ); + self.root = self._internal_recurse( dirp ) - ## Load this BillieDB from file + ## Load this MirrorSplitDB from file def load_from_file(self, fname): - f = open(fname, "r"); - self.root = cPickle.load(f); - f.close(); + f = open(fname, "r") + self.root = cPickle.load(f) + f.close() - ## Save this BillieDB to a file + ## Save this MirrorSplitDB to a file def save_to_file(self, fname): - f = open(fname, "w"); - cPickle.dump( self.root, f, 1 ); - f.close(); + f = open(fname, "w") + cPickle.dump( self.root, f, 1 ) + f.close() + - ############################################################################## # Helper functions for the tree syncing... ################## def _pth(a,b): - return "%s/%s" % (a,b); + return "%s/%s" % (a,b) def do_mkdir(targ,path): if not os.path.exists( _pth(targ.root, path) ): - os.makedirs( _pth(targ.root, path) ); + os.makedirs( _pth(targ.root, path) ) def do_mkdir_f(targ,path): - do_mkdir(targ, os.path.dirname(path)); + do_mkdir(targ, os.path.dirname(path)) def do_link(targ,path): - do_mkdir_f(targ,path); + do_mkdir_f(targ,path) os.link( _pth(MASTER_PATH, path), - _pth(targ.root, path)); + _pth(targ.root, path)) def do_symlink(targ,path,link): - do_mkdir_f(targ,path); - os.symlink( link, _pth(targ.root, path) ); + do_mkdir_f(targ,path) + os.symlink( link, _pth(targ.root, path) ) def do_unlink(targ,path): - os.unlink( _pth(targ.root, path) ); + os.unlink( _pth(targ.root, path) ) def do_unlink_dir(targ,path): - os.system( "rm -Rf '%s'" % _pth(targ.root, path) ); + os.system( "rm -Rf '%s'" % _pth(targ.root, path) ) ############################################################################## # Reconciling a target with the sourcedb @@ -229,77 +231,77 @@ def _internal_reconcile( path, srcdir, targdir, targ ): for k in targdir.links.keys(): if applicable( _pth(path, k), targ ): if not srcdir.links.has_key(k): - rm.append(k); + rm.append(k) else: - rm.append(k); + rm.append(k) for k in rm: #print "-L-", _pth(path,k) do_unlink(targ, _pth(path,k)) - del targdir.links[k]; - + del targdir.links[k] + # Remove any files in targdir which aren't in srcdir # Or which aren't applicable rm = [] for k in targdir.files.keys(): if applicable( _pth(path, k), targ ): if not srcdir.files.has_key(k): - rm.append(k); + rm.append(k) else: - rm.append(k); + rm.append(k) for k in rm: #print "-F-", _pth(path,k) do_unlink(targ, _pth(path,k)) - del targdir.files[k]; + del targdir.files[k] # Remove any dirs in targdir which aren't in srcdir rm = [] for k in targdir.dirs.keys(): if not srcdir.dirs.has_key(k): - rm.append(k); + rm.append(k) for k in rm: #print "-D-", _pth(path,k) do_unlink_dir(targ, _pth(path,k)) - del targdir.dirs[k]; + del targdir.dirs[k] # Add/update files for k in srcdir.files.keys(): if applicable( _pth(path,k), targ ): if not targdir.files.has_key(k): #print "+F+", _pth(path,k) - do_link( targ, _pth(path,k) ); - targdir.files[k] = srcdir.files[k]; + do_link( targ, _pth(path,k) ) + targdir.files[k] = srcdir.files[k] else: if targdir.files[k] != srcdir.files[k]: - #print "*F*", _pth(path,k); - do_unlink( targ, _pth(path,k) ); - do_link( targ, _pth(path,k) ); - targdir.files[k] = srcdir.files[k]; + #print "*F*", _pth(path,k) + do_unlink( targ, _pth(path,k) ) + do_link( targ, _pth(path,k) ) + targdir.files[k] = srcdir.files[k] # Add/update links for k in srcdir.links.keys(): if applicable( _pth(path,k), targ ): if not targdir.links.has_key(k): - targdir.links[k] = srcdir.links[k]; + targdir.links[k] = srcdir.links[k]; #print "+L+",_pth(path,k), "->", srcdir.links[k] - do_symlink( targ, _pth(path,k), targdir.links[k] ); + do_symlink( targ, _pth(path,k), targdir.links[k] ) else: if targdir.links[k] != srcdir.links[k]: - do_unlink( targ, _pth(path,k) ); - targdir.links[k] = srcdir.links[k]; + do_unlink( targ, _pth(path,k) ) + targdir.links[k] = srcdir.links[k] #print "*L*", _pth(path,k), "to ->", srcdir.links[k] - do_symlink( targ, _pth(path,k), targdir.links[k] ); + do_symlink( targ, _pth(path,k), targdir.links[k] ) # Do dirs for k in srcdir.dirs.keys(): if not targdir.dirs.has_key(k): - targdir.dirs[k] = BillieDir(); + targdir.dirs[k] = MirrorSplitDir() #print "+D+", _pth(path,k) _internal_reconcile( _pth(path,k), srcdir.dirs[k], - targdir.dirs[k], targ ); + targdir.dirs[k], targ ) def reconcile_target_db( src, targ ): - _internal_reconcile( "", src.root, targ.db.root, targ ); + _internal_reconcile( "", src.root, targ.db.root, targ ) ############################################################################### @@ -309,21 +311,21 @@ def load_config(): global TREE_DB_ROOT global trees - MASTER_PATH = Cnf["Billie::FTPPath"]; - TREE_ROOT = Cnf["Billie::TreeRootPath"]; - TREE_DB_ROOT = Cnf["Billie::TreeDatabasePath"]; - - for a in Cnf.ValueList("Billie::BasicTrees"): - trees.append( BillieTarget( a, "%s,all" % a, 1 ) ) + MASTER_PATH = Cnf["Mirror-Split::FTPPath"] + TREE_ROOT = Cnf["Mirror-Split::TreeRootPath"] + TREE_DB_ROOT = Cnf["Mirror-Split::TreeDatabasePath"] - for n in Cnf.SubTree("Billie::CombinationTrees").List(): - archs = Cnf.ValueList("Billie::CombinationTrees::%s" % n) + for a in Cnf.ValueList("Mirror-Split::BasicTrees"): + trees.append( MirrorSplitTarget( a, "%s,all" % a, 1 ) ) + + for n in Cnf.SubTree("Mirror-Split::CombinationTrees").List(): + archs = Cnf.ValueList("Mirror-Split::CombinationTrees::%s" % n) source = 0 if "source" in archs: source = 1 archs.remove("source") archs = ",".join(archs) - trees.append( BillieTarget( n, archs, source ) ); + trees.append( MirrorSplitTarget( n, archs, source ) ) def do_list (): print "Master path",MASTER_PATH @@ -336,9 +338,9 @@ def do_list (): print " [source]" else: print "" - + def do_help (): - print """Usage: billie [OPTIONS] + print """Usage: dak mirror-split [OPTIONS] Generate hardlink trees of certain architectures -h, --help show this help and exit @@ -349,40 +351,40 @@ Generate hardlink trees of certain architectures def main (): global Cnf - Cnf = utils.get_conf() + Cnf = daklib.utils.get_conf() - Arguments = [('h',"help","Billie::Options::Help"), - ('l',"list","Billie::Options::List"), - ]; + Arguments = [('h',"help","Mirror-Split::Options::Help"), + ('l',"list","Mirror-Split::Options::List"), + ] - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Cnf["Billie::Options::cake"] = ""; - Options = Cnf.SubTree("Billie::Options") + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + Cnf["Mirror-Split::Options::cake"] = "" + Options = Cnf.SubTree("Mirror-Split::Options") print "Loading configuration..." - load_config(); + load_config() print "Loaded." if Options.has_key("Help"): - do_help(); - return; + do_help() + return if Options.has_key("List"): - do_list(); - return; - + do_list() + return - src = BillieDB() + + src = MirrorSplitDB() print "Scanning", MASTER_PATH src.init_from_dir(MASTER_PATH) print "Scanned" for tree in trees: print "Reconciling tree:",tree.name - reconcile_target_db( src, tree ); + reconcile_target_db( src, tree ) print "Saving updated DB...", - tree.save_db(); + tree.save_db() print "Done" - + ############################################################################## if __name__ == '__main__':