3 # Prepare and maintain partial trees by architecture
4 # Copyright (C) 2004, 2006 Daniel Silverstone <dsilvers@digital-scurf.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ###############################################################################
22 ## <kinnison> So Martin, do you have a quote for me yet?
23 ## <tbm> Make something damned stupid up and attribute it to me, that's okay
24 ###############################################################################
30 from stat import S_ISDIR, S_ISLNK, S_ISREG
34 ## Master path is the main repository
35 #MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master"
37 MASTER_PATH = "***Configure Mirror-Split::FTPPath Please***"
38 TREE_ROOT = "***Configure Mirror-Split::TreeRootPath Please***"
39 TREE_DB_ROOT = "***Configure Mirror-Split::TreeDatabasePath Please***"
44 ###############################################################################
45 # A MirrorSplitTarget is a representation of a target. It is a set of archs, a path
46 # and whether or not the target includes source.
49 class MirrorSplitTarget:
50 def __init__(self, name, archs, source):
52 self.root = "%s/%s" % (TREE_ROOT,name)
53 self.archs = archs.split(",")
55 self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name)
56 self.db = MirrorSplitDB()
57 if os.path.exists( self.dbpath ):
58 self.db.load_from_file( self.dbpath )
60 ## Save the db back to disk
62 self.db.save_to_file( self.dbpath )
64 ## Returns true if it's a poolish match
65 def poolish_match(self, path):
67 if path.endswith( "_%s.deb" % (a) ):
69 if path.endswith( "_%s.udeb" % (a) ):
72 if (path.endswith( ".tar.gz" ) or
73 path.endswith( ".diff.gz" ) or
74 path.endswith( ".dsc" )):
78 ## Returns false if it's a badmatch distswise
79 def distish_match(self,path):
81 if path.endswith("/Contents-%s.gz" % (a)):
83 if path.find("/binary-%s/" % (a)) != -1:
85 if path.find("/installer-%s/" % (a)) != -1:
87 if path.find("/source/") != -1:
92 if path.find("/Contents-") != -1:
94 if path.find("/binary-") != -1:
96 if path.find("/installer-") != -1:
100 ##############################################################################
101 # The applicable function is basically a predicate. Given a path and a
102 # target object its job is to decide if the path conforms for the
103 # target and thus is wanted.
105 # 'verbatim' is a list of files which are copied regardless
106 # it should be loaded from a config file eventually
118 def applicable(path, target):
119 if path.startswith("/pool/"):
120 return target.poolish_match(path)
121 if (path.startswith("/dists/") or
122 path.startswith("/project/experimental/")):
123 return target.distish_match(path)
126 for prefix in verbprefix:
127 if path.startswith(prefix):
132 ##############################################################################
133 # A MirrorSplitDir is a representation of a tree.
134 # It distinguishes files dirs and links
135 # Dirs are dicts of (name, MirrorSplitDir)
136 # Files are dicts of (name, inode)
137 # Links are dicts of (name, target)
140 class MirrorSplitDir:
146 ##############################################################################
147 # A MirrorSplitDB is a container for a MirrorSplitDir...
151 ## Initialise a MirrorSplitDB as containing nothing
153 self.root = MirrorSplitDir()
155 def _internal_recurse(self, path):
156 bdir = MirrorSplitDir()
157 dl = os.listdir( path )
161 lnl = os.lstat( "%s/%s" % (path, ln) )
164 elif S_ISLNK(lnl[0]):
165 bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) )
166 elif S_ISREG(lnl[0]):
167 bdir.files[ln] = lnl[1]
169 dak.lib.utils.fubar( "Confused by %s/%s -- not a dir, link or file" %
172 bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) )
176 ## Recurse through a given path, setting the sequence accordingly
177 def init_from_dir(self, dirp):
178 self.root = self._internal_recurse( dirp )
180 ## Load this MirrorSplitDB from file
181 def load_from_file(self, fname):
183 self.root = cPickle.load(f)
186 ## Save this MirrorSplitDB to a file
187 def save_to_file(self, fname):
189 cPickle.dump( self.root, f, 1 )
193 ##############################################################################
194 # Helper functions for the tree syncing...
198 return "%s/%s" % (a,b)
200 def do_mkdir(targ,path):
201 if not os.path.exists( _pth(targ.root, path) ):
202 os.makedirs( _pth(targ.root, path) )
204 def do_mkdir_f(targ,path):
205 do_mkdir(targ, os.path.dirname(path))
207 def do_link(targ,path):
208 do_mkdir_f(targ,path)
209 os.link( _pth(MASTER_PATH, path),
210 _pth(targ.root, path))
212 def do_symlink(targ,path,link):
213 do_mkdir_f(targ,path)
214 os.symlink( link, _pth(targ.root, path) )
216 def do_unlink(targ,path):
217 os.unlink( _pth(targ.root, path) )
219 def do_unlink_dir(targ,path):
220 os.system( "rm -Rf '%s'" % _pth(targ.root, path) )
222 ##############################################################################
223 # Reconciling a target with the sourcedb
226 def _internal_reconcile( path, srcdir, targdir, targ ):
227 # Remove any links in targdir which aren't in srcdir
228 # Or which aren't applicable
230 for k in targdir.links.keys():
231 if applicable( _pth(path, k), targ ):
232 if not srcdir.links.has_key(k):
237 #print "-L-", _pth(path,k)
238 do_unlink(targ, _pth(path,k))
241 # Remove any files in targdir which aren't in srcdir
242 # Or which aren't applicable
244 for k in targdir.files.keys():
245 if applicable( _pth(path, k), targ ):
246 if not srcdir.files.has_key(k):
251 #print "-F-", _pth(path,k)
252 do_unlink(targ, _pth(path,k))
255 # Remove any dirs in targdir which aren't in srcdir
257 for k in targdir.dirs.keys():
258 if not srcdir.dirs.has_key(k):
261 #print "-D-", _pth(path,k)
262 do_unlink_dir(targ, _pth(path,k))
266 for k in srcdir.files.keys():
267 if applicable( _pth(path,k), targ ):
268 if not targdir.files.has_key(k):
269 #print "+F+", _pth(path,k)
270 do_link( targ, _pth(path,k) )
271 targdir.files[k] = srcdir.files[k]
273 if targdir.files[k] != srcdir.files[k]:
274 #print "*F*", _pth(path,k)
275 do_unlink( targ, _pth(path,k) )
276 do_link( targ, _pth(path,k) )
277 targdir.files[k] = srcdir.files[k]
280 for k in srcdir.links.keys():
281 if applicable( _pth(path,k), targ ):
282 if not targdir.links.has_key(k):
283 targdir.links[k] = srcdir.links[k];
284 #print "+L+",_pth(path,k), "->", srcdir.links[k]
285 do_symlink( targ, _pth(path,k), targdir.links[k] )
287 if targdir.links[k] != srcdir.links[k]:
288 do_unlink( targ, _pth(path,k) )
289 targdir.links[k] = srcdir.links[k]
290 #print "*L*", _pth(path,k), "to ->", srcdir.links[k]
291 do_symlink( targ, _pth(path,k), targdir.links[k] )
294 for k in srcdir.dirs.keys():
295 if not targdir.dirs.has_key(k):
296 targdir.dirs[k] = MirrorSplitDir()
297 #print "+D+", _pth(path,k)
298 _internal_reconcile( _pth(path,k), srcdir.dirs[k],
299 targdir.dirs[k], targ )
302 def reconcile_target_db( src, targ ):
303 _internal_reconcile( "", src.root, targ.db.root, targ )
305 ###############################################################################
313 MASTER_PATH = Cnf["Mirror-Split::FTPPath"]
314 TREE_ROOT = Cnf["Mirror-Split::TreeRootPath"]
315 TREE_DB_ROOT = Cnf["Mirror-Split::TreeDatabasePath"]
317 for a in Cnf.ValueList("Mirror-Split::BasicTrees"):
318 trees.append( MirrorSplitTarget( a, "%s,all" % a, 1 ) )
320 for n in Cnf.SubTree("Mirror-Split::CombinationTrees").List():
321 archs = Cnf.ValueList("Mirror-Split::CombinationTrees::%s" % n)
323 if "source" in archs:
325 archs.remove("source")
326 archs = ",".join(archs)
327 trees.append( MirrorSplitTarget( n, archs, source ) )
330 print "Master path",MASTER_PATH
331 print "Trees at",TREE_ROOT
332 print "DBs at",TREE_DB_ROOT
335 print tree.name,"contains",", ".join(tree.archs),
342 print """Usage: dak mirror-split [OPTIONS]
343 Generate hardlink trees of certain architectures
345 -h, --help show this help and exit
346 -l, --list list the configuration and exit
353 Cnf = dak.lib.utils.get_conf()
355 Arguments = [('h',"help","Mirror-Split::Options::Help"),
356 ('l',"list","Mirror-Split::Options::List"),
359 arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
360 Cnf["Mirror-Split::Options::cake"] = ""
361 Options = Cnf.SubTree("Mirror-Split::Options")
363 print "Loading configuration..."
367 if Options.has_key("Help"):
370 if Options.has_key("List"):
375 src = MirrorSplitDB()
376 print "Scanning", MASTER_PATH
377 src.init_from_dir(MASTER_PATH)
381 print "Reconciling tree:",tree.name
382 reconcile_target_db( src, tree )
383 print "Saving updated DB...",
387 ##############################################################################
389 if __name__ == '__main__':