3 # Prepare and maintain partial trees by architecture
4 # Copyright (C) 2004 Daniel Silverstone <dsilvers@digital-scurf.org>
5 # $Id: billie,v 1.3 2004-07-12 15:44:09 dsilvers Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 ###############################################################################
23 ## <kinnison> So Martin, do you have a quote for me yet?
24 ## <tbm> Make something damned stupid up and attribute it to me, that's okay
25 ###############################################################################
28 import utils, db_access;
29 import apt_pkg, logging;
31 from stat import S_ISDIR, S_ISLNK, S_ISREG;
35 ## Master path is the main repository
36 #MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master";
38 MASTER_PATH = "***Configure Billie::FTPPath Please***";
39 TREE_ROOT = "***Configure Billie::TreeRootPath Please***";
40 TREE_DB_ROOT = "***Configure Billie::TreeDatabasePath Please***";
43 ###############################################################################
44 # A BillieTarget is a representation of a target. It is a set of archs, a path
45 # and whether or not the target includes source.
49 def __init__(self, name, archs, source):
51 self.root = "%s/%s" % (TREE_ROOT,name);
52 self.archs = archs.split(",");
54 self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name);
56 if os.path.exists( self.dbpath ):
57 self.db.load_from_file( self.dbpath );
59 ## Save the db back to disk
61 self.db.save_to_file( self.dbpath );
63 ## Returns true if it's a poolish match
64 def poolish_match(self, path):
66 if path.endswith( "_%s.deb" % (a) ):
69 if (path.endswith( ".tar.gz" ) or
70 path.endswith( ".diff.gz" ) or
71 path.endswith( ".dsc" )):
75 ## Returns false if it's a badmatch distswise
76 def distish_match(self,path):
78 if path.endswith("/Contents-%s.gz" % (a)):
80 if path.find("/binary-%s/" % (a)) != -1:
82 if path.find("/installer-%s/" % (a)) != -1:
84 if path.find("/source/") != -1:
89 if path.find("/Contents-") != -1:
91 if path.find("/binary-") != -1:
93 if path.find("/installer-") != -1:
97 ##############################################################################
98 # The applicable function is basically a predicate. Given a path and a
99 # target object its job is to decide if the path conforms for the
100 # target and thus is wanted.
102 # 'verbatim' is a list of files which are copied regardless
103 # it should be loaded from a config file eventually
115 def applicable(path, target):
116 if path.startswith("/pool/"):
117 return target.poolish_match(path);
118 if (path.startswith("/dists/") or
119 path.startswith("/project/experimental/")):
120 return target.distish_match(path);
123 for prefix in verbprefix:
124 if path.startswith(prefix):
129 ##############################################################################
130 # A BillieDir is a representation of a tree.
131 # It distinguishes files dirs and links
132 # Dirs are dicts of (name, BillieDir)
133 # Files are dicts of (name, inode)
134 # Links are dicts of (name, target)
143 ##############################################################################
144 # A BillieDB is a container for a BillieDir...
148 ## Initialise a BillieDB as containing nothing
150 self.root = BillieDir();
152 def _internal_recurse(self, path):
154 dl = os.listdir( path );
158 lnl = os.lstat( "%s/%s" % (path, ln) );
161 elif S_ISLNK(lnl[0]):
162 bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) );
163 elif S_ISREG(lnl[0]):
164 bdir.files[ln] = lnl[1];
166 util.fubar( "Confused by %s/%s -- not a dir, link or file" %
169 bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) );
173 ## Recurse through a given path, setting the sequence accordingly
174 def init_from_dir(self, dirp):
175 self.root = self._internal_recurse( dirp );
177 ## Load this BillieDB from file
178 def load_from_file(self, fname):
179 f = open(fname, "r");
180 self.root = cPickle.load(f);
183 ## Save this BillieDB to a file
184 def save_to_file(self, fname):
185 f = open(fname, "w");
186 cPickle.dump( self.root, f, 1 );
190 ##############################################################################
191 # Helper functions for the tree syncing...
195 return "%s/%s" % (a,b);
197 def do_mkdir(targ,path):
198 if not os.path.exists( _pth(targ.root, path) ):
199 os.makedirs( _pth(targ.root, path) );
201 def do_mkdir_f(targ,path):
202 do_mkdir(targ, os.path.dirname(path));
204 def do_link(targ,path):
205 do_mkdir_f(targ,path);
206 os.link( _pth(MASTER_PATH, path),
207 _pth(targ.root, path));
209 def do_symlink(targ,path,link):
210 do_mkdir_f(targ,path);
211 os.symlink( link, _pth(targ.root, path) );
213 def do_unlink(targ,path):
214 os.unlink( _pth(targ.root, path) );
216 def do_unlink_dir(targ,path):
217 os.system( "rm -Rf '%s'" % _pth(targ.root, path) );
219 ##############################################################################
220 # Reconciling a target with the sourcedb
223 def _internal_reconcile( path, srcdir, targdir, targ ):
224 # Remove any links in targdir which aren't in srcdir
225 # Or which aren't applicable
227 for k in targdir.links.keys():
228 if applicable( _pth(path, k), targ ):
229 if not srcdir.links.has_key(k):
234 #print "-L-", _pth(path,k)
235 do_unlink(targ, _pth(path,k))
236 del targdir.links[k];
238 # Remove any files in targdir which aren't in srcdir
239 # Or which aren't applicable
241 for k in targdir.files.keys():
242 if applicable( _pth(path, k), targ ):
243 if not srcdir.files.has_key(k):
248 #print "-F-", _pth(path,k)
249 do_unlink(targ, _pth(path,k))
250 del targdir.files[k];
252 # Remove any dirs in targdir which aren't in srcdir
254 for k in targdir.dirs.keys():
255 if not srcdir.dirs.has_key(k):
258 #print "-D-", _pth(path,k)
259 do_unlink_dir(targ, _pth(path,k))
263 for k in srcdir.files.keys():
264 if applicable( _pth(path,k), targ ):
265 if not targdir.files.has_key(k):
266 #print "+F+", _pth(path,k)
267 do_link( targ, _pth(path,k) );
268 targdir.files[k] = srcdir.files[k];
270 if targdir.files[k] != srcdir.files[k]:
271 #print "*F*", _pth(path,k);
272 do_unlink( targ, _pth(path,k) );
273 do_link( targ, _pth(path,k) );
274 targdir.files[k] = srcdir.files[k];
277 for k in srcdir.links.keys():
278 if applicable( _pth(path,k), targ ):
279 if not targdir.links.has_key(k):
280 targdir.links[k] = srcdir.links[k];
281 #print "+L+",_pth(path,k), "->", srcdir.links[k]
282 do_symlink( targ, _pth(path,k), targdir.links[k] );
284 if targdir.links[k] != srcdir.links[k]:
285 do_unlink( targ, _pth(path,k) );
286 targdir.links[k] = srcdir.links[k];
287 #print "*L*", _pth(path,k), "to ->", srcdir.links[k]
288 do_symlink( targ, _pth(path,k), targdir.links[k] );
291 for k in srcdir.dirs.keys():
292 if not targdir.dirs.has_key(k):
293 targdir.dirs[k] = BillieDir();
294 #print "+D+", _pth(path,k)
295 _internal_reconcile( _pth(path,k), srcdir.dirs[k],
296 targdir.dirs[k], targ );
299 def reconcile_target_db( src, targ ):
300 _internal_reconcile( "", src.root, targ.db.root, targ );
302 ###############################################################################
310 MASTER_PATH = Cnf["Billie::FTPPath"];
311 TREE_ROOT = Cnf["Billie::TreeRootPath"];
312 TREE_DB_ROOT = Cnf["Billie::TreeDatabasePath"];
314 for a in Cnf.ValueList("Billie::BasicTrees"):
315 trees.append( BillieTarget( a, "%s,all" % a, 1 ) )
317 for n in Cnf.SubTree("Billie::CombinationTrees").List():
318 archs = Cnf.ValueList("Billie::CombinationTrees::%s" % n)
320 if "source" in archs:
322 archs.remove("source")
323 archs = ",".join(archs)
324 trees.append( BillieTarget( n, archs, source ) );
327 print "Master path",MASTER_PATH
328 print "Trees at",TREE_ROOT
329 print "DBs at",TREE_DB_ROOT
332 print tree.name,"contains",", ".join(tree.archs),
339 print """Usage: billie [OPTIONS]
340 Generate hardlink trees of certain architectures
342 -h, --help show this help and exit
343 -l, --list list the configuration and exit
350 Cnf = utils.get_conf()
352 Arguments = [('h',"help","Billie::Options::Help"),
353 ('l',"list","Billie::Options::List"),
356 arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
357 Cnf["Billie::Options::cake"] = "";
358 Options = Cnf.SubTree("Billie::Options")
360 print "Loading configuration..."
364 if Options.has_key("Help"):
367 if Options.has_key("List"):
373 print "Scanning", MASTER_PATH
374 src.init_from_dir(MASTER_PATH)
378 print "Reconciling tree:",tree.name
379 reconcile_target_db( src, tree );
380 print "Saving updated DB...",
384 ##############################################################################
386 if __name__ == '__main__':