3 """ Generate Packages/Sources files
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2010 Joerg Jaspert <joerg@debian.org>
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 ################################################################################
32 from tempfile import mkstemp, mkdtemp
34 from multiprocessing import Pool, TimeoutError
36 from daklib import daklog
37 from daklib.dbconn import *
38 from daklib.config import Config
40 ################################################################################
42 Options = None #: Commandline arguments parsed into this
43 Logger = None #: Our logging object
44 results = [] #: Results of the subprocesses
46 ################################################################################
48 def usage (exit_code=0):
49 print """Usage: dak generate-packages-sources [OPTIONS]
50 Generate the Packages/Sources files
52 -s, --suite=SUITE(s) process this suite
53 Default: All suites not marked 'untouchable'
54 -f, --force Allow processing of untouchable suites
55 CAREFUL: Only to be used at point release time!
56 -h, --help show this help and exit
58 SUITE can be a space seperated list, e.g.
59 --suite=unstable testing
64 ################################################################################
66 def generate_packages_sources(arch, suite, tmppath):
68 Generate Packages/Sources files with apt-ftparchive for the given suite/arch
71 @param suite: Suite name
74 @param arch: Architecture name
77 @param tmppath: The temporary path to work ing
83 ArchiveDir "/srv/ftp-master.debian.org/ftp/";
84 OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
85 CacheDir "/srv/ftp-master.debian.org/database/";
90 Packages::Compress "bzip2 gzip";
91 Sources::Compress "bzip2 gzip";
92 Contents::Compress "gzip";
94 MaxContentsChange 25000;
100 Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
107 apt_trees["testing"]="""
110 FakeDI "dists/unstable";
111 FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
112 SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
113 Sections "main contrib non-free";
114 Architectures "%(arch)s";
115 BinOverride "override.wheezy.$(SECTION)";
116 ExtraOverride "override.wheezy.extra.$(SECTION)";
117 SrcOverride "override.wheezy.$(SECTION).src";
121 apt_trees["squeeze-updates"]="""
122 tree "dists/squeeze-updates"
124 FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
125 SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
126 Sections "main contrib non-free";
127 Architectures "%(arch)s";
128 BinOverride "override.squeeze.$(SECTION)";
129 ExtraOverride "override.squeeze.extra.$(SECTION)";
130 SrcOverride "override.squeeze.$(SECTION).src";
135 apt_trees["di"]["testing"]="""
136 tree "dists/testing/main"
138 FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
139 Sections "debian-installer";
140 Architectures "%(arch)s";
141 BinOverride "override.wheezy.main.$(SECTION)";
142 SrcOverride "override.wheezy.main.src";
143 BinCacheDB "packages-debian-installer-$(ARCH).db";
144 Packages::Extensions ".udeb";
148 tree "dists/testing/non-free"
150 FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
151 Sections "debian-installer";
152 Architectures "%(arch)s";
153 BinOverride "override.wheezy.main.$(SECTION)";
154 SrcOverride "override.wheezy.main.src";
155 BinCacheDB "packages-debian-installer-$(ARCH).db";
156 Packages::Extensions ".udeb";
161 apt_trees["unstable"]="""
162 tree "dists/unstable"
164 FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
165 SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
166 Sections "main contrib non-free";
167 Architectures "%(arch)s";
168 BinOverride "override.sid.$(SECTION)";
169 ExtraOverride "override.sid.extra.$(SECTION)";
170 SrcOverride "override.sid.$(SECTION).src";
173 apt_trees["di"]["unstable"]="""
174 tree "dists/unstable/main"
176 FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
177 Sections "debian-installer";
178 Architectures "%(arch)s";
179 BinOverride "override.sid.main.$(SECTION)";
180 SrcOverride "override.sid.main.src";
181 BinCacheDB "packages-debian-installer-$(ARCH).db";
182 Packages::Extensions ".udeb";
186 tree "dists/unstable/non-free"
188 FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
189 Sections "debian-installer";
190 Architectures "%(arch)s";
191 BinOverride "override.sid.main.$(SECTION)";
192 SrcOverride "override.sid.main.src";
193 BinCacheDB "packages-debian-installer-$(ARCH).db";
194 Packages::Extensions ".udeb";
199 apt_trees["experimental"]="""
200 tree "dists/experimental"
202 FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
203 SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
204 Sections "main contrib non-free";
205 Architectures "%(arch)s";
206 BinOverride "override.sid.$(SECTION)";
207 SrcOverride "override.sid.$(SECTION).src";
210 apt_trees["di"]["experimental"]="""
211 tree "dists/experimental/main"
213 FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
214 Sections "debian-installer";
215 Architectures "%(arch)s";
216 BinOverride "override.sid.main.$(SECTION)";
217 SrcOverride "override.sid.main.src";
218 BinCacheDB "packages-debian-installer-$(ARCH).db";
219 Packages::Extensions ".udeb";
223 tree "dists/experimental/non-free"
225 FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
226 Sections "debian-installer";
227 Architectures "%(arch)s";
228 BinOverride "override.sid.main.$(SECTION)";
229 SrcOverride "override.sid.main.src";
230 BinCacheDB "packages-debian-installer-$(ARCH).db";
231 Packages::Extensions ".udeb";
236 apt_trees["testing-proposed-updates"]="""
237 tree "dists/testing-proposed-updates"
239 FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
240 SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
241 Sections "main contrib non-free";
242 Architectures "%(arch)s";
243 BinOverride "override.wheezy.$(SECTION)";
244 ExtraOverride "override.wheezy.extra.$(SECTION)";
245 SrcOverride "override.wheezy.$(SECTION).src";
249 apt_trees["di"]["testing-proposed-updates"]="""
250 tree "dists/testing-proposed-updates/main"
252 FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
253 Sections "debian-installer";
254 Architectures "%(arch)s";
255 BinOverride "override.wheezy.main.$(SECTION)";
256 SrcOverride "override.wheezy.main.src";
257 BinCacheDB "packages-debian-installer-$(ARCH).db";
258 Packages::Extensions ".udeb";
263 apt_trees["proposed-updates"]="""
264 tree "dists/proposed-updates"
266 FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
267 SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
268 Sections "main contrib non-free";
269 Architectures "%(arch)s";
270 BinOverride "override.squeeze.$(SECTION)";
271 ExtraOverride "override.squeeze.extra.$(SECTION)";
272 SrcOverride "override.squeeze.$(SECTION).src";
276 apt_trees["di"]["proposed-updates"]="""
277 tree "dists/proposed-updates/main"
279 FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
280 Sections "debian-installer";
281 Architectures "%(arch)s";
282 BinOverride "override.squeeze.main.$(SECTION)";
283 SrcOverride "override.squeeze.main.src";
284 BinCacheDB "packages-debian-installer-$(ARCH).db";
285 Packages::Extensions ".udeb";
289 apt_trees["oldstable-proposed-updates"]="""
290 tree "dists/oldstable-proposed-updates"
292 FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
293 SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
294 Sections "main contrib non-free";
295 Architectures "%(arch)s";
296 BinOverride "override.lenny.$(SECTION)";
297 ExtraOverride "override.lenny.extra.$(SECTION)";
298 SrcOverride "override.lenny.$(SECTION).src";
302 apt_trees["di"]["oldstable-proposed-updates"]="""
303 tree "dists/oldstable-proposed-updates/main"
305 FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
306 Sections "debian-installer";
307 Architectures "%(arch)s";
308 BinOverride "override.lenny.main.$(SECTION)";
309 SrcOverride "override.lenny.main.src";
310 BinCacheDB "packages-debian-installer-$(ARCH).db";
311 Packages::Extensions ".udeb";
319 (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
320 os.write(ac_fd, DAILY_APT_CONF)
321 # here we want to generate the tree entries
322 os.write(ac_fd, apt_trees[suite] % {'arch': arch})
323 # this special casing needs to go away, but this whole thing may just want an
324 # aptconfig class anyways
326 if arch == 'hurd-i386' and suite == 'experimental':
328 elif apt_trees["di"].has_key(suite):
330 os.write(ac_fd, apt_trees["di"][suite] %
331 {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
333 os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
336 print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
337 # Run apt-ftparchive generate
338 # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
339 # it has errormessages we like to see
340 os.environ['GZIP'] = '--rsyncable'
342 (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
343 sn="a-f %s,%s: " % (suite, arch)
344 print sn + output.replace('\n', '\n%s' % (sn))
347 # Clean up any left behind files
362 return arch.arch_string
369 ########################################################################
370 ########################################################################
373 global Options, Logger, results
377 for i in ["Help", "Suite", "Force"]:
378 if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
379 cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
381 Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
382 ('s',"suite","Generate-Packages-Sources::Options::Suite"),
383 ('f',"force","Generate-Packages-Sources::Options::Force")]
385 suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
386 Options = cnf.SubTree("Generate-Packages-Sources::Options")
391 Logger = daklog.Logger(cnf, 'generate-packages-sources')
393 session = DBConn().session()
398 for s in suite_names:
399 suite = get_suite(s.lower(), session)
403 print "cannot find suite %s" % s
404 Logger.log(['cannot find suite %s' % s])
406 suites=session.query(Suite).filter(Suite.untouchable == False).all()
408 startdir = os.getcwd()
409 os.chdir(cnf["Dir::TempPath"])
412 # For each given suite, each architecture, run one apt-ftparchive
415 # Setup a multiprocessing Pool. As many workers as we have CPU cores.
417 arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
418 Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
420 pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
422 # No more work will be added to our pool, close it and then wait for all to finish
427 Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
428 print "Trouble, something with a-f broke, resultcodes: %s" % (results)
432 # this script doesn't change the database
436 #######################################################################################
438 if __name__ == '__main__':