3 # Prepare and maintain partial trees by architecture
4 # Copyright (C) 2004 Daniel Silverstone <dsilvers@digital-scurf.org>
5 # $Id: billie,v 1.4 2004-11-27 16:06:42 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 ###############################################################################
23 ## <kinnison> So Martin, do you have a quote for me yet?
24 ## <tbm> Make something damned stupid up and attribute it to me, that's okay
25 ###############################################################################
28 import utils, db_access;
29 import apt_pkg, logging;
31 from stat import S_ISDIR, S_ISLNK, S_ISREG;
35 ## Master path is the main repository
36 #MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master";
38 MASTER_PATH = "***Configure Billie::FTPPath Please***";
39 TREE_ROOT = "***Configure Billie::TreeRootPath Please***";
40 TREE_DB_ROOT = "***Configure Billie::TreeDatabasePath Please***";
43 ###############################################################################
44 # A BillieTarget is a representation of a target. It is a set of archs, a path
45 # and whether or not the target includes source.
49 def __init__(self, name, archs, source):
51 self.root = "%s/%s" % (TREE_ROOT,name);
52 self.archs = archs.split(",");
54 self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name);
56 if os.path.exists( self.dbpath ):
57 self.db.load_from_file( self.dbpath );
59 ## Save the db back to disk
61 self.db.save_to_file( self.dbpath );
63 ## Returns true if it's a poolish match
64 def poolish_match(self, path):
66 if path.endswith( "_%s.deb" % (a) ):
68 if path.endswith( "_%s.udeb" % (a) ):
71 if (path.endswith( ".tar.gz" ) or
72 path.endswith( ".diff.gz" ) or
73 path.endswith( ".dsc" )):
77 ## Returns false if it's a badmatch distswise
78 def distish_match(self,path):
80 if path.endswith("/Contents-%s.gz" % (a)):
82 if path.find("/binary-%s/" % (a)) != -1:
84 if path.find("/installer-%s/" % (a)) != -1:
86 if path.find("/source/") != -1:
91 if path.find("/Contents-") != -1:
93 if path.find("/binary-") != -1:
95 if path.find("/installer-") != -1:
99 ##############################################################################
100 # The applicable function is basically a predicate. Given a path and a
101 # target object its job is to decide if the path conforms for the
102 # target and thus is wanted.
104 # 'verbatim' is a list of files which are copied regardless
105 # it should be loaded from a config file eventually
117 def applicable(path, target):
118 if path.startswith("/pool/"):
119 return target.poolish_match(path);
120 if (path.startswith("/dists/") or
121 path.startswith("/project/experimental/")):
122 return target.distish_match(path);
125 for prefix in verbprefix:
126 if path.startswith(prefix):
131 ##############################################################################
132 # A BillieDir is a representation of a tree.
133 # It distinguishes files dirs and links
134 # Dirs are dicts of (name, BillieDir)
135 # Files are dicts of (name, inode)
136 # Links are dicts of (name, target)
145 ##############################################################################
146 # A BillieDB is a container for a BillieDir...
150 ## Initialise a BillieDB as containing nothing
152 self.root = BillieDir();
154 def _internal_recurse(self, path):
156 dl = os.listdir( path );
160 lnl = os.lstat( "%s/%s" % (path, ln) );
163 elif S_ISLNK(lnl[0]):
164 bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) );
165 elif S_ISREG(lnl[0]):
166 bdir.files[ln] = lnl[1];
168 util.fubar( "Confused by %s/%s -- not a dir, link or file" %
171 bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) );
175 ## Recurse through a given path, setting the sequence accordingly
176 def init_from_dir(self, dirp):
177 self.root = self._internal_recurse( dirp );
179 ## Load this BillieDB from file
180 def load_from_file(self, fname):
181 f = open(fname, "r");
182 self.root = cPickle.load(f);
185 ## Save this BillieDB to a file
186 def save_to_file(self, fname):
187 f = open(fname, "w");
188 cPickle.dump( self.root, f, 1 );
192 ##############################################################################
193 # Helper functions for the tree syncing...
197 return "%s/%s" % (a,b);
199 def do_mkdir(targ,path):
200 if not os.path.exists( _pth(targ.root, path) ):
201 os.makedirs( _pth(targ.root, path) );
203 def do_mkdir_f(targ,path):
204 do_mkdir(targ, os.path.dirname(path));
206 def do_link(targ,path):
207 do_mkdir_f(targ,path);
208 os.link( _pth(MASTER_PATH, path),
209 _pth(targ.root, path));
211 def do_symlink(targ,path,link):
212 do_mkdir_f(targ,path);
213 os.symlink( link, _pth(targ.root, path) );
215 def do_unlink(targ,path):
216 os.unlink( _pth(targ.root, path) );
218 def do_unlink_dir(targ,path):
219 os.system( "rm -Rf '%s'" % _pth(targ.root, path) );
221 ##############################################################################
222 # Reconciling a target with the sourcedb
225 def _internal_reconcile( path, srcdir, targdir, targ ):
226 # Remove any links in targdir which aren't in srcdir
227 # Or which aren't applicable
229 for k in targdir.links.keys():
230 if applicable( _pth(path, k), targ ):
231 if not srcdir.links.has_key(k):
236 #print "-L-", _pth(path,k)
237 do_unlink(targ, _pth(path,k))
238 del targdir.links[k];
240 # Remove any files in targdir which aren't in srcdir
241 # Or which aren't applicable
243 for k in targdir.files.keys():
244 if applicable( _pth(path, k), targ ):
245 if not srcdir.files.has_key(k):
250 #print "-F-", _pth(path,k)
251 do_unlink(targ, _pth(path,k))
252 del targdir.files[k];
254 # Remove any dirs in targdir which aren't in srcdir
256 for k in targdir.dirs.keys():
257 if not srcdir.dirs.has_key(k):
260 #print "-D-", _pth(path,k)
261 do_unlink_dir(targ, _pth(path,k))
265 for k in srcdir.files.keys():
266 if applicable( _pth(path,k), targ ):
267 if not targdir.files.has_key(k):
268 #print "+F+", _pth(path,k)
269 do_link( targ, _pth(path,k) );
270 targdir.files[k] = srcdir.files[k];
272 if targdir.files[k] != srcdir.files[k]:
273 #print "*F*", _pth(path,k);
274 do_unlink( targ, _pth(path,k) );
275 do_link( targ, _pth(path,k) );
276 targdir.files[k] = srcdir.files[k];
279 for k in srcdir.links.keys():
280 if applicable( _pth(path,k), targ ):
281 if not targdir.links.has_key(k):
282 targdir.links[k] = srcdir.links[k];
283 #print "+L+",_pth(path,k), "->", srcdir.links[k]
284 do_symlink( targ, _pth(path,k), targdir.links[k] );
286 if targdir.links[k] != srcdir.links[k]:
287 do_unlink( targ, _pth(path,k) );
288 targdir.links[k] = srcdir.links[k];
289 #print "*L*", _pth(path,k), "to ->", srcdir.links[k]
290 do_symlink( targ, _pth(path,k), targdir.links[k] );
293 for k in srcdir.dirs.keys():
294 if not targdir.dirs.has_key(k):
295 targdir.dirs[k] = BillieDir();
296 #print "+D+", _pth(path,k)
297 _internal_reconcile( _pth(path,k), srcdir.dirs[k],
298 targdir.dirs[k], targ );
301 def reconcile_target_db( src, targ ):
302 _internal_reconcile( "", src.root, targ.db.root, targ );
304 ###############################################################################
312 MASTER_PATH = Cnf["Billie::FTPPath"];
313 TREE_ROOT = Cnf["Billie::TreeRootPath"];
314 TREE_DB_ROOT = Cnf["Billie::TreeDatabasePath"];
316 for a in Cnf.ValueList("Billie::BasicTrees"):
317 trees.append( BillieTarget( a, "%s,all" % a, 1 ) )
319 for n in Cnf.SubTree("Billie::CombinationTrees").List():
320 archs = Cnf.ValueList("Billie::CombinationTrees::%s" % n)
322 if "source" in archs:
324 archs.remove("source")
325 archs = ",".join(archs)
326 trees.append( BillieTarget( n, archs, source ) );
329 print "Master path",MASTER_PATH
330 print "Trees at",TREE_ROOT
331 print "DBs at",TREE_DB_ROOT
334 print tree.name,"contains",", ".join(tree.archs),
341 print """Usage: billie [OPTIONS]
342 Generate hardlink trees of certain architectures
344 -h, --help show this help and exit
345 -l, --list list the configuration and exit
352 Cnf = utils.get_conf()
354 Arguments = [('h',"help","Billie::Options::Help"),
355 ('l',"list","Billie::Options::List"),
358 arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
359 Cnf["Billie::Options::cake"] = "";
360 Options = Cnf.SubTree("Billie::Options")
362 print "Loading configuration..."
366 if Options.has_key("Help"):
369 if Options.has_key("List"):
375 print "Scanning", MASTER_PATH
376 src.init_from_dir(MASTER_PATH)
380 print "Reconciling tree:",tree.name
381 reconcile_target_db( src, tree );
382 print "Saving updated DB...",
386 ##############################################################################
388 if __name__ == '__main__':