3 # Prepare and maintain partial trees by architecture
4 # Copyright (C) 2004 Daniel Silverstone <dsilvers@digital-scurf.org>
5 # $Id: billie,v 1.2 2004-04-12 12:44:06 dsilvers Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 ###############################################################################
23 ## <kinnison> So Martin, do you have a quote for me yet?
24 ## <tbm> Make something damned stupid up and attribute it to me, that's okay
25 ###############################################################################
28 import utils, db_access;
29 import apt_pkg, logging;
31 from stat import S_ISDIR, S_ISLNK, S_ISREG;
35 ## Master path is the main repository
36 #MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master";
38 MASTER_PATH = "***Configure Billie::FTPPath Please***";
39 TREE_ROOT = "***Configure Billie::TreeRootPath Please***";
40 TREE_DB_ROOT = "***Configure Billie::TreeDatabasePath Please***";
43 ###############################################################################
44 # A BillieTarget is a representation of a target. It is a set of archs, a path
45 # and whether or not the target includes source.
49 def __init__(self, name, archs, source):
51 self.root = "%s/%s" % (TREE_ROOT,name);
52 self.archs = archs.split(",");
54 self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name);
56 if os.path.exists( self.dbpath ):
57 self.db.load_from_file( self.dbpath );
59 ## Save the db back to disk
61 self.db.save_to_file( self.dbpath );
63 ## Returns true if it's a poolish match
64 def poolish_match(self, path):
66 if path.endswith( "_%s.deb" % (a) ):
69 if (path.endswith( ".tar.gz" ) or
70 path.endswith( ".diff.gz" ) or
71 path.endswith( ".dsc" )):
75 ## Returns false if it's a badmatch distswise
76 def distish_match(self,path):
78 if path.endswith("/Contents-%s.gz" % (a)):
80 if path.find("/binary-%s/" % (a)) != -1:
82 if path.find("/installer-%s/" % (a)) != -1:
84 if path.find("/source/") != -1:
89 if path.find("/Contents-") != -1:
91 if path.find("/binary-") != -1:
93 if path.find("/installer-") != -1:
97 ##############################################################################
98 # The applicable function is basically a predicate. Given a path and a
99 # target object its job is to decide if the path conforms for the
100 # target and thus is wanted.
102 # 'verbatim' is a list of files which are copied regardless
103 # it should be loaded from a config file eventually
109 "/README.CD-manufacture",
110 "/README.mirrors.html",
111 "/README.mirrors.txt",
119 def applicable(path, target):
120 if path.startswith("/pool/"):
121 return target.poolish_match(path);
122 if (path.startswith("/dists/") or
123 path.startswith("/project/experimental/")):
124 return target.distish_match(path);
127 for prefix in verbprefix:
128 if path.startswith(prefix):
133 ##############################################################################
134 # A BillieDir is a representation of a tree.
135 # It distinguishes files dirs and links
136 # Dirs are dicts of (name, BillieDir)
137 # Files are dicts of (name, inode)
138 # Links are dicts of (name, target)
147 ##############################################################################
148 # A BillieDB is a container for a BillieDir...
152 ## Initialise a BillieDB as containing nothing
154 self.root = BillieDir();
156 def _internal_recurse(self, path):
158 dl = os.listdir( path );
162 lnl = os.lstat( "%s/%s" % (path, ln) );
165 elif S_ISLNK(lnl[0]):
166 bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) );
167 elif S_ISREG(lnl[0]):
168 bdir.files[ln] = lnl[1];
170 util.fubar( "Confused by %s/%s -- not a dir, link or file" %
173 bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) );
177 ## Recurse through a given path, setting the sequence accordingly
178 def init_from_dir(self, dirp):
179 self.root = self._internal_recurse( dirp );
181 ## Load this BillieDB from file
182 def load_from_file(self, fname):
183 f = open(fname, "r");
184 self.root = cPickle.load(f);
187 ## Save this BillieDB to a file
188 def save_to_file(self, fname):
189 f = open(fname, "w");
190 cPickle.dump( self.root, f, 1 );
194 ##############################################################################
195 # Helper functions for the tree syncing...
199 return "%s/%s" % (a,b);
201 def do_mkdir(targ,path):
202 if not os.path.exists( _pth(targ.root, path) ):
203 os.makedirs( _pth(targ.root, path) );
205 def do_mkdir_f(targ,path):
206 do_mkdir(targ, os.path.dirname(path));
208 def do_link(targ,path):
209 do_mkdir_f(targ,path);
210 os.link( _pth(MASTER_PATH, path),
211 _pth(targ.root, path));
213 def do_symlink(targ,path,link):
214 do_mkdir_f(targ,path);
215 os.symlink( link, _pth(targ.root, path) );
217 def do_unlink(targ,path):
218 os.unlink( _pth(targ.root, path) );
220 def do_unlink_dir(targ,path):
221 os.system( "rm -Rf '%s'" % _pth(targ.root, path) );
223 ##############################################################################
224 # Reconciling a target with the sourcedb
227 def _internal_reconcile( path, srcdir, targdir, targ ):
228 # Remove any links in targdir which aren't in srcdir
229 # Or which aren't applicable
231 for k in targdir.links.keys():
232 if applicable( _pth(path, k), targ ):
233 if not srcdir.links.has_key(k):
238 #print "-L-", _pth(path,k)
239 do_unlink(targ, _pth(path,k))
240 del targdir.links[k];
242 # Remove any files in targdir which aren't in srcdir
243 # Or which aren't applicable
245 for k in targdir.files.keys():
246 if applicable( _pth(path, k), targ ):
247 if not srcdir.files.has_key(k):
252 #print "-F-", _pth(path,k)
253 do_unlink(targ, _pth(path,k))
254 del targdir.files[k];
256 # Remove any dirs in targdir which aren't in srcdir
258 for k in targdir.dirs.keys():
259 if not srcdir.dirs.has_key(k):
262 #print "-D-", _pth(path,k)
263 do_unlink_dir(targ, _pth(path,k))
267 for k in srcdir.files.keys():
268 if applicable( _pth(path,k), targ ):
269 if not targdir.files.has_key(k):
270 #print "+F+", _pth(path,k)
271 do_link( targ, _pth(path,k) );
272 targdir.files[k] = srcdir.files[k];
274 if targdir.files[k] != srcdir.files[k]:
275 #print "*F*", _pth(path,k);
276 do_unlink( targ, _pth(path,k) );
277 do_link( targ, _pth(path,k) );
278 targdir.files[k] = srcdir.files[k];
281 for k in srcdir.links.keys():
282 if applicable( _pth(path,k), targ ):
283 if not targdir.links.has_key(k):
284 targdir.links[k] = srcdir.links[k];
285 #print "+L+",_pth(path,k), "->", srcdir.links[k]
286 do_symlink( targ, _pth(path,k), targdir.links[k] );
288 if targdir.links[k] != srcdir.links[k]:
289 do_unlink( targ, _pth(path,k) );
290 targdir.links[k] = srcdir.links[k];
291 #print "*L*", _pth(path,k), "to ->", srcdir.links[k]
292 do_symlink( targ, _pth(path,k), targdir.links[k] );
295 for k in srcdir.dirs.keys():
296 if not targdir.dirs.has_key(k):
297 targdir.dirs[k] = BillieDir();
298 #print "+D+", _pth(path,k)
299 _internal_reconcile( _pth(path,k), srcdir.dirs[k],
300 targdir.dirs[k], targ );
303 def reconcile_target_db( src, targ ):
304 _internal_reconcile( "", src.root, targ.db.root, targ );
306 ###############################################################################
314 MASTER_PATH = Cnf["Billie::FTPPath"];
315 TREE_ROOT = Cnf["Billie::TreeRootPath"];
316 TREE_DB_ROOT = Cnf["Billie::TreeDatabasePath"];
318 for a in Cnf.ValueList("Billie::BasicTrees"):
319 trees.append( BillieTarget( a, "%s,all" % a, 1 ) )
321 for n in Cnf.SubTree("Billie::CombinationTrees").List():
322 archs = Cnf.ValueList("Billie::CombinationTrees::%s" % n)
324 if "source" in archs:
326 archs.remove("source")
327 archs = ",".join(archs)
328 trees.append( BillieTarget( n, archs, source ) );
331 print "Master path",MASTER_PATH
332 print "Trees at",TREE_ROOT
333 print "DBs at",TREE_DB_ROOT
336 print tree.name,"contains",", ".join(tree.archs),
343 print """Usage: billie [OPTIONS]
344 Generate hardlink trees of certain architectures
346 -h, --help show this help and exit
347 -l, --list list the configuration and exit
354 Cnf = utils.get_conf()
356 Arguments = [('h',"help","Billie::Options::Help"),
357 ('l',"list","Billie::Options::List"),
360 arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
361 Cnf["Billie::Options::cake"] = "";
362 Options = Cnf.SubTree("Billie::Options")
364 print "Loading configuration..."
368 if Options.has_key("Help"):
371 if Options.has_key("List"):
377 print "Scanning", MASTER_PATH
378 src.init_from_dir(MASTER_PATH)
382 print "Reconciling tree:",tree.name
383 reconcile_target_db( src, tree );
384 print "Saving updated DB...",
388 ##############################################################################
390 if __name__ == '__main__':