3 # Generate file lists used by apt-ftparchive to generate Packages and Sources files
4 # Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup <james@nocrew.org>
5 # $Id: jenna,v 1.29 2004-11-27 17:58:47 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 # <elmo> I'm doing it in python btw.. nothing against your monster
24 # SQL, but the python wins in terms of speed and readiblity
29 # <aj> you mock my SQL!!!!
30 # <elmo> you want have contest of skillz??????
31 # <aj> all your skillz are belong to my sql!!!!
32 # <elmo> yo momma are belong to my python!!!!
33 # <aj> yo momma was SQLin' like a pig last night!
35 ################################################################################
37 import copy, os, pg, string, sys;
39 import claire, db_access, logging, utils;
41 ################################################################################
48 ################################################################################
50 def Dict(**dict): return dict
52 ################################################################################
54 def usage (exit_code=0):
55 print """Usage: jenna [OPTION]
56 Write out file lists suitable for use with apt-ftparchive.
58 -a, --architecture=ARCH only write file lists for this architecture
59 -c, --component=COMPONENT only write file lists for this component
60 -h, --help show this help and exit
61 -n, --no-delete don't delete older versions
62 -s, --suite=SUITE only write file lists for this suite
64 ARCH, COMPONENT and SUITE can be space separated lists, e.g.
65 --architecture=\"m68k i386\"""";
68 ################################################################################
70 def version_cmp(a, b):
71 return -apt_pkg.VersionCompare(a[0], b[0]);
73 #####################################################
75 def delete_packages(delete_versions, pkg, dominant_arch, suite,
76 dominant_version, delete_table, delete_col, packages):
77 suite_id = db_access.get_suite_id(suite);
78 for version in delete_versions:
79 delete_unique_id = version[1];
80 if not packages.has_key(delete_unique_id):
82 delete_version = version[0];
83 delete_id = packages[delete_unique_id]["id"];
84 delete_arch = packages[delete_unique_id]["arch"];
85 if not Cnf.Find("Suite::%s::Untouchable" % (suite)):
86 if Options["No-Delete"]:
87 print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch);
89 Logger.log(["dominated", pkg, delete_arch, delete_version, dominant_version, dominant_arch]);
90 projectB.query("DELETE FROM %s WHERE suite = %s AND %s = %s" % (delete_table, suite_id, delete_col, delete_id));
91 del packages[delete_unique_id];
93 if Options["No-Delete"]:
94 print "Would delete %s_%s_%s in favour of %s_%s, but %s is untouchable" % (pkg, delete_arch, delete_version, dominant_version, dominant_arch, suite);
96 Logger.log(["dominated but untouchable", pkg, delete_arch, delete_version, dominant_version, dominant_arch]);
98 #####################################################
100 # Per-suite&pkg: resolve arch-all, vs. arch-any, assumes only one arch-all
101 def resolve_arch_all_vs_any(versions, packages):
102 arch_all_version = None;
103 arch_any_versions = copy.copy(versions);
104 for i in arch_any_versions:
106 arch = packages[unique_id]["arch"];
108 arch_all_versions = [i];
109 arch_all_version = i[0];
110 arch_any_versions.remove(i);
111 # Sort arch: any versions into descending order
112 arch_any_versions.sort(version_cmp);
113 highest_arch_any_version = arch_any_versions[0][0];
115 pkg = packages[unique_id]["pkg"];
116 suite = packages[unique_id]["suite"];
117 delete_table = "bin_associations";
120 if apt_pkg.VersionCompare(highest_arch_any_version, arch_all_version) < 1:
121 # arch: all dominates
122 delete_packages(arch_any_versions, pkg, "all", suite,
123 arch_all_version, delete_table, delete_col, packages);
125 # arch: any dominates
126 delete_packages(arch_all_versions, pkg, "any", suite,
127 highest_arch_any_version, delete_table, delete_col,
130 #####################################################
132 # Per-suite&pkg&arch: resolve duplicate versions
133 def remove_duplicate_versions(versions, packages):
134 # Sort versions into descending order
135 versions.sort(version_cmp);
136 dominant_versions = versions[0];
137 dominated_versions = versions[1:];
138 (dominant_version, dominant_unqiue_id) = dominant_versions;
139 pkg = packages[dominant_unqiue_id]["pkg"];
140 arch = packages[dominant_unqiue_id]["arch"];
141 suite = packages[dominant_unqiue_id]["suite"];
143 delete_table = "src_associations";
144 delete_col = "source";
146 delete_table = "bin_associations";
148 # Remove all but the highest
149 delete_packages(dominated_versions, pkg, arch, suite,
150 dominant_version, delete_table, delete_col, packages);
151 return [dominant_versions];
153 ################################################################################
155 def cleanup(packages):
156 # Build up the index used by the clean up functions
158 for unique_id in packages.keys():
159 suite = packages[unique_id]["suite"];
160 pkg = packages[unique_id]["pkg"];
161 arch = packages[unique_id]["arch"];
162 version = packages[unique_id]["version"];
163 d.setdefault(suite, {});
164 d[suite].setdefault(pkg, {});
165 d[suite][pkg].setdefault(arch, []);
166 d[suite][pkg][arch].append([version, unique_id]);
167 # Clean up old versions
168 for suite in d.keys():
169 for pkg in d[suite].keys():
170 for arch in d[suite][pkg].keys():
171 versions = d[suite][pkg][arch];
172 if len(versions) > 1:
173 d[suite][pkg][arch] = remove_duplicate_versions(versions, packages);
175 # Arch: all -> any and vice versa
176 for suite in d.keys():
177 for pkg in d[suite].keys():
178 arches = d[suite][pkg];
179 # If we don't have any arch: all; we've nothing to do
180 if not arches.has_key("all"):
182 # Check to see if we have arch: all and arch: !all (ignoring source)
183 num_arches = len(arches.keys());
184 if arches.has_key("source"):
186 # If we do, remove the duplicates
189 for arch in arches.keys():
191 versions.extend(d[suite][pkg][arch]);
192 resolve_arch_all_vs_any(versions, packages);
194 ################################################################################
196 def write_legacy_mixed_filelist(suite, list, packages, dislocated_files):
197 # Work out the filename
198 filename = os.path.join(Cnf["Dir::Lists"], "%s_-_all.list" % (suite));
199 output = utils.open_file(filename, "w");
200 # Generate the final list of files
203 path = packages[id]["path"];
204 filename = packages[id]["filename"];
205 file_id = packages[id]["file_id"];
206 if suite == "stable" and dislocated_files.has_key(file_id):
207 filename = dislocated_files[file_id];
209 filename = path + filename;
210 if files.has_key(filename):
211 utils.warn("%s (in %s) is duplicated." % (filename, suite));
213 files[filename] = "";
214 # Sort the files since apt-ftparchive doesn't
217 # Write the list of files out
219 output.write(file+'\n')
222 ############################################################
224 def write_filelist(suite, component, arch, type, list, packages, dislocated_files):
225 # Work out the filename
228 arch = "debian-installer_binary-%s" % (arch);
230 arch = "binary-%s" % (arch);
231 filename = os.path.join(Cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch));
232 output = utils.open_file(filename, "w");
233 # Generate the final list of files
236 path = packages[id]["path"];
237 filename = packages[id]["filename"];
238 file_id = packages[id]["file_id"];
239 pkg = packages[id]["pkg"];
240 if suite == "stable" and dislocated_files.has_key(file_id):
241 filename = dislocated_files[file_id];
243 filename = path + filename;
244 if files.has_key(pkg):
245 utils.warn("%s (in %s/%s, %s) is duplicated." % (pkg, suite, component, filename));
247 files[pkg] = filename;
248 # Sort the files since apt-ftparchive doesn't
251 # Write the list of files out
253 output.write(files[pkg]+'\n')
256 ################################################################################
258 def write_filelists(packages, dislocated_files):
259 # Build up the index to iterate over
261 for unique_id in packages.keys():
262 suite = packages[unique_id]["suite"];
263 component = packages[unique_id]["component"];
264 arch = packages[unique_id]["arch"];
265 type = packages[unique_id]["type"];
266 d.setdefault(suite, {});
267 d[suite].setdefault(component, {});
268 d[suite][component].setdefault(arch, {});
269 d[suite][component][arch].setdefault(type, []);
270 d[suite][component][arch][type].append(unique_id);
271 # Flesh out the index
272 if not Options["Suite"]:
273 suites = Cnf.SubTree("Suite").List();
275 suites = utils.split_args(Options["Suite"]);
276 for suite in map(string.lower, suites):
277 d.setdefault(suite, {});
278 if not Options["Component"]:
279 components = Cnf.ValueList("Suite::%s::Components" % (suite));
281 components = utils.split_args(Options["Component"]);
282 udeb_components = Cnf.ValueList("Suite::%s::UdebComponents" % (suite));
283 udeb_components = udeb_components;
284 for component in components:
285 d[suite].setdefault(component, {});
286 if component in udeb_components:
287 binary_types = [ "deb", "udeb" ];
289 binary_types = [ "deb" ];
290 if not Options["Architecture"]:
291 architectures = Cnf.ValueList("Suite::%s::Architectures" % (suite));
293 architectures = utils.split_args(Options["Architectures"]);
294 for arch in map(string.lower, architectures):
295 d[suite][component].setdefault(arch, {});
299 types = binary_types;
301 d[suite][component][arch].setdefault(type, []);
303 for suite in d.keys():
304 if Cnf.has_key("Suite::%s::Components" % (suite)):
305 for component in d[suite].keys():
306 for arch in d[suite][component].keys():
309 for type in d[suite][component][arch].keys():
310 list = d[suite][component][arch][type];
311 # If it's a binary, we need to add in the arch: all debs too
313 archall_suite = Cnf.get("Jenna::ArchAllMap::%s" % (suite));
315 list.extend(d[archall_suite][component]["all"][type]);
316 elif d[suite][component].has_key("all") and \
317 d[suite][component]["all"].has_key(type):
318 list.extend(d[suite][component]["all"][type]);
319 write_filelist(suite, component, arch, type, list,
320 packages, dislocated_files);
321 else: # legacy-mixed suite
323 for component in d[suite].keys():
324 for arch in d[suite][component].keys():
325 for type in d[suite][component][arch].keys():
326 list.extend(d[suite][component][arch][type]);
327 write_legacy_mixed_filelist(suite, list, packages, dislocated_files);
329 ################################################################################
331 # Want to use stable dislocation support: True or false?
332 def stable_dislocation_p():
333 # If the support is not explicitly enabled, assume it's disabled
334 if not Cnf.FindB("Dinstall::StableDislocationSupport"):
336 # If we don't have a stable suite, obviously a no-op
337 if not Cnf.has_key("Suite::Stable"):
339 # If the suite(s) weren't explicitly listed, all suites are done
340 if not Options["Suite"]:
342 # Otherwise, look in what suites the user specified
343 suites = utils.split_args(Options["Suite"]);
345 if "stable" in suites:
350 ################################################################################
353 # If we're only doing a subset of suites, ensure we do enough to
354 # be able to do arch: all mapping.
356 suites = utils.split_args(Options["Suite"]);
358 archall_suite = Cnf.get("Jenna::ArchAllMap::%s" % (suite));
359 if archall_suite and archall_suite not in suites:
360 utils.warn("Adding %s as %s maps Arch: all from it." % (archall_suite, suite));
361 suites.append(archall_suite);
362 Options["Suite"] = ",".join(suites);
364 (con_suites, con_architectures, con_components, check_source) = \
365 utils.parse_args(Options);
367 if stable_dislocation_p():
368 dislocated_files = claire.find_dislocated_stable(Cnf, projectB);
370 dislocated_files = {};
373 SELECT b.id, b.package, a.arch_string, b.version, l.path, f.filename, c.name,
374 f.id, su.suite_name, b.type
375 FROM binaries b, bin_associations ba, architecture a, files f, location l,
376 component c, suite su
377 WHERE b.id = ba.bin AND b.file = f.id AND b.architecture = a.id
378 AND f.location = l.id AND l.component = c.id AND ba.suite = su.id
379 %s %s %s""" % (con_suites, con_architectures, con_components);
383 SELECT s.id, s.source, 'source', s.version, l.path, f.filename, c.name, f.id,
385 FROM source s, src_associations sa, files f, location l, component c, suite su
386 WHERE s.id = sa.source AND s.file = f.id AND f.location = l.id
387 AND l.component = c.id AND sa.suite = su.id %s %s""" % (con_suites, con_components);
388 q = projectB.query(query);
390 # Build up the main index of packages
394 (id, pkg, arch, version, path, filename, component, file_id, suite, type) = i;
395 # 'id' comes from either 'binaries' or 'source', so it's not unique
397 packages[unique_id] = Dict(id=id, pkg=pkg, arch=arch, version=version,
398 path=path, filename=filename,
399 component=component, file_id=file_id,
400 suite=suite, type = type);
402 write_filelists(packages, dislocated_files);
404 ################################################################################
407 global Cnf, projectB, Options, Logger;
409 Cnf = utils.get_conf();
410 Arguments = [('a', "architecture", "Jenna::Options::Architecture", "HasArg"),
411 ('c', "component", "Jenna::Options::Component", "HasArg"),
412 ('h', "help", "Jenna::Options::Help"),
413 ('n', "no-delete", "Jenna::Options::No-Delete"),
414 ('s', "suite", "Jenna::Options::Suite", "HasArg")];
415 for i in ["architecture", "component", "help", "no-delete", "suite" ]:
416 if not Cnf.has_key("Jenna::Options::%s" % (i)):
417 Cnf["Jenna::Options::%s" % (i)] = "";
418 apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
419 Options = Cnf.SubTree("Jenna::Options");
423 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
424 db_access.init(Cnf, projectB);
425 Logger = logging.Logger(Cnf, "jenna");
429 #########################################################################################
431 if __name__ == '__main__':