3 """ Generate Packages/Sources files
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2010 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
33 from tempfile import mkstemp, mkdtemp
35 from multiprocessing import Pool, TimeoutError
37 from daklib import daklog
38 from daklib.dbconn import *
39 from daklib.config import Config
41 ################################################################################
43 Options = None #: Commandline arguments parsed into this
44 Logger = None #: Our logging object
45 results = [] #: Results of the subprocesses
47 ################################################################################
49 def usage (exit_code=0):
50 print """Usage: dak generate-packages-sources [OPTIONS]
51 Generate the Packages/Sources files
53 -s, --suite=SUITE(s) process this suite
54 Default: All suites not marked 'untouchable'
55 -f, --force Allow processing of untouchable suites
56 CAREFUL: Only to be used at point release time!
57 -h, --help show this help and exit
59 SUITE can be a space seperated list, e.g.
60 --suite=unstable testing
65 ################################################################################
67 def generate_packages_sources(arch, suite, tmppath):
69 Generate Packages/Sources files with apt-ftparchive for the given suite/arch
72 @param suite: Suite name
75 @param arch: Architecture name
78 @param tmppath: The temporary path to work ing
84 ArchiveDir "/srv/ftp-master.debian.org/ftp/";
85 OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
86 CacheDir "/srv/ftp-master.debian.org/database/";
91 Packages::Compress "bzip2 gzip";
92 Sources::Compress "bzip2 gzip";
93 Contents::Compress "gzip";
95 MaxContentsChange 25000;
101 Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
109 apt_trees["stable"]="""
112 FileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_binary-$(ARCH).list";
113 SourceFileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_source.list";
114 Sections "main contrib non-free";
115 Architectures "%(arch)s";
116 BinOverride "override.squeeze.$(SECTION)";
117 ExtraOverride "override.squeeze.extra.$(SECTION)";
118 SrcOverride "override.squeeze.$(SECTION).src";
122 apt_trees["di"]["stable"]="""
123 tree "dists/stable/main"
125 FileList "/srv/ftp-master.debian.org/database/dists/stable_main_$(SECTION)_binary-$(ARCH).list";
126 Sections "debian-installer";
127 Architectures "%(arch)s";
128 BinOverride "override.squeeze.main.$(SECTION)";
129 SrcOverride "override.squeeze.main.src";
130 BinCacheDB "packages-debian-installer-$(ARCH).db";
131 Packages::Extensions ".udeb";
135 tree "dists/stable/non-free"
137 FileList "/srv/ftp-master.debian.org/database/dists/stable_non-free_$(SECTION)_binary-$(ARCH).list";
138 Sections "debian-installer";
139 Architectures "%(arch)s";
140 BinOverride "override.squeeze.main.$(SECTION)";
141 SrcOverride "override.squeeze.main.src";
142 BinCacheDB "packages-debian-installer-$(ARCH).db";
143 Packages::Extensions ".udeb";
148 apt_trees["squeeze-updates"]="""
149 tree "dists/squeeze-updates"
151 FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
152 SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
153 Sections "main contrib non-free";
154 Architectures "%(arch)s";
155 BinOverride "override.squeeze.$(SECTION)";
156 ExtraOverride "override.squeeze.extra.$(SECTION)";
157 SrcOverride "override.squeeze.$(SECTION).src";
162 apt_trees["testing"]="""
165 FakeDI "dists/unstable";
166 FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
167 SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
168 Sections "main contrib non-free";
169 Architectures "%(arch)s";
170 BinOverride "override.wheezy.$(SECTION)";
171 ExtraOverride "override.wheezy.extra.$(SECTION)";
172 SrcOverride "override.wheezy.$(SECTION).src";
176 apt_trees["di"]["testing"]="""
177 tree "dists/testing/main"
179 FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
180 Sections "debian-installer";
181 Architectures "%(arch)s";
182 BinOverride "override.wheezy.main.$(SECTION)";
183 SrcOverride "override.wheezy.main.src";
184 BinCacheDB "packages-debian-installer-$(ARCH).db";
185 Packages::Extensions ".udeb";
189 tree "dists/testing/non-free"
191 FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
192 Sections "debian-installer";
193 Architectures "%(arch)s";
194 BinOverride "override.wheezy.main.$(SECTION)";
195 SrcOverride "override.wheezy.main.src";
196 BinCacheDB "packages-debian-installer-$(ARCH).db";
197 Packages::Extensions ".udeb";
202 apt_trees["unstable"]="""
203 tree "dists/unstable"
205 FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
206 SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
207 Sections "main contrib non-free";
208 Architectures "%(arch)s";
209 BinOverride "override.sid.$(SECTION)";
210 ExtraOverride "override.sid.extra.$(SECTION)";
211 SrcOverride "override.sid.$(SECTION).src";
214 apt_trees["di"]["unstable"]="""
215 tree "dists/unstable/main"
217 FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
218 Sections "debian-installer";
219 Architectures "%(arch)s";
220 BinOverride "override.sid.main.$(SECTION)";
221 SrcOverride "override.sid.main.src";
222 BinCacheDB "packages-debian-installer-$(ARCH).db";
223 Packages::Extensions ".udeb";
227 tree "dists/unstable/non-free"
229 FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
230 Sections "debian-installer";
231 Architectures "%(arch)s";
232 BinOverride "override.sid.main.$(SECTION)";
233 SrcOverride "override.sid.main.src";
234 BinCacheDB "packages-debian-installer-$(ARCH).db";
235 Packages::Extensions ".udeb";
240 apt_trees["experimental"]="""
241 tree "dists/experimental"
243 FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
244 SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
245 Sections "main contrib non-free";
246 Architectures "%(arch)s";
247 BinOverride "override.sid.$(SECTION)";
248 SrcOverride "override.sid.$(SECTION).src";
251 apt_trees["di"]["experimental"]="""
252 tree "dists/experimental/main"
254 FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
255 Sections "debian-installer";
256 Architectures "%(arch)s";
257 BinOverride "override.sid.main.$(SECTION)";
258 SrcOverride "override.sid.main.src";
259 BinCacheDB "packages-debian-installer-$(ARCH).db";
260 Packages::Extensions ".udeb";
264 tree "dists/experimental/non-free"
266 FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
267 Sections "debian-installer";
268 Architectures "%(arch)s";
269 BinOverride "override.sid.main.$(SECTION)";
270 SrcOverride "override.sid.main.src";
271 BinCacheDB "packages-debian-installer-$(ARCH).db";
272 Packages::Extensions ".udeb";
277 apt_trees["testing-proposed-updates"]="""
278 tree "dists/testing-proposed-updates"
280 FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
281 SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
282 Sections "main contrib non-free";
283 Architectures "%(arch)s";
284 BinOverride "override.wheezy.$(SECTION)";
285 ExtraOverride "override.wheezy.extra.$(SECTION)";
286 SrcOverride "override.wheezy.$(SECTION).src";
290 apt_trees["di"]["testing-proposed-updates"]="""
291 tree "dists/testing-proposed-updates/main"
293 FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
294 Sections "debian-installer";
295 Architectures "%(arch)s";
296 BinOverride "override.wheezy.main.$(SECTION)";
297 SrcOverride "override.wheezy.main.src";
298 BinCacheDB "packages-debian-installer-$(ARCH).db";
299 Packages::Extensions ".udeb";
304 apt_trees["proposed-updates"]="""
305 tree "dists/proposed-updates"
307 FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
308 SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
309 Sections "main contrib non-free";
310 Architectures "%(arch)s";
311 BinOverride "override.squeeze.$(SECTION)";
312 ExtraOverride "override.squeeze.extra.$(SECTION)";
313 SrcOverride "override.squeeze.$(SECTION).src";
317 apt_trees["di"]["proposed-updates"]="""
318 tree "dists/proposed-updates/main"
320 FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
321 Sections "debian-installer";
322 Architectures "%(arch)s";
323 BinOverride "override.squeeze.main.$(SECTION)";
324 SrcOverride "override.squeeze.main.src";
325 BinCacheDB "packages-debian-installer-$(ARCH).db";
326 Packages::Extensions ".udeb";
333 (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
334 os.write(ac_fd, DAILY_APT_CONF)
335 # here we want to generate the tree entries
336 os.write(ac_fd, apt_trees[suite] % {'arch': arch})
337 # this special casing needs to go away, but this whole thing may just want an
338 # aptconfig class anyways
340 if arch == 'hurd-i386' and suite == 'experimental':
342 elif apt_trees["di"].has_key(suite):
344 os.write(ac_fd, apt_trees["di"][suite] %
345 {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
347 os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
350 print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
351 # Run apt-ftparchive generate
352 # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
353 # it has errormessages we like to see
354 os.environ['GZIP'] = '--rsyncable'
356 (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
357 sn="a-f %s,%s: " % (suite, arch)
358 print sn + output.replace('\n', '\n%s' % (sn))
361 # Clean up any left behind files
376 return arch.arch_string
383 ########################################################################
384 ########################################################################
387 global Options, Logger, results
391 for i in ["Help", "Suite", "Force"]:
392 if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
393 cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
395 Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
396 ('s',"suite","Generate-Packages-Sources::Options::Suite"),
397 ('f',"force","Generate-Packages-Sources::Options::Force")]
399 suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
400 Options = cnf.SubTree("Generate-Packages-Sources::Options")
405 Logger = daklog.Logger('generate-packages-sources')
407 session = DBConn().session()
412 for s in suite_names:
413 suite = get_suite(s.lower(), session)
417 print "cannot find suite %s" % s
418 Logger.log(['cannot find suite %s' % s])
420 suites=session.query(Suite).filter(Suite.untouchable == False).all()
422 startdir = os.getcwd()
423 os.chdir(cnf["Dir::TempPath"])
426 # For each given suite, each architecture, run one apt-ftparchive
429 # Setup a multiprocessing Pool. As many workers as we have CPU cores.
431 arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
432 Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
434 pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
436 # No more work will be added to our pool, close it and then wait for all to finish
441 Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
442 print "Trouble, something with a-f broke, resultcodes: %s" % (results)
446 # this script doesn't change the database
450 #######################################################################################
452 if __name__ == '__main__':