]> git.decadent.org.uk Git - dak.git/blob - dak/generate_packages_sources.py
Merge remote-tracking branch 'drkranz/lintian' into merge
[dak.git] / dak / generate_packages_sources.py
1 #!/usr/bin/env python
2
3 """ Generate Packages/Sources files
4
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
7 @copyright: 2009  Mark Hymers <mhy@debian.org>
8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
10
11 """
12
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
17
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21 # GNU General Public License for more details.
22
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26
27 ################################################################################
28
29 import os
30 import os.path
31 import sys
32 import apt_pkg
33 from tempfile import mkstemp, mkdtemp
34 import commands
35 from multiprocessing import Pool, TimeoutError
36
37 from daklib import daklog
38 from daklib.dbconn import *
39 from daklib.config import Config
40
41 ################################################################################
42
43 Options = None                 #: Commandline arguments parsed into this
44 Logger = None                  #: Our logging object
45 results = []                   #: Results of the subprocesses
46
47 ################################################################################
48
49 def usage (exit_code=0):
50     print """Usage: dak generate-packages-sources [OPTIONS]
51 Generate the Packages/Sources files
52
53   -s, --suite=SUITE(s)       process this suite
54                              Default: All suites not marked 'untouchable'
55   -f, --force                Allow processing of untouchable suites
56                              CAREFUL: Only to be used at point release time!
57   -h, --help                 show this help and exit
58
59 SUITE can be a space seperated list, e.g.
60    --suite=unstable testing
61   """
62
63     sys.exit(exit_code)
64
65 ################################################################################
66
67 def generate_packages_sources(arch, suite, tmppath):
68     """
69     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
70
71     @type suite: string
72     @param suite: Suite name
73
74     @type arch: string
75     @param arch: Architecture name
76
77     @type tmppath: string
78     @param tmppath: The temporary path to work ing
79     """
80
81     DAILY_APT_CONF="""
82 Dir
83 {
84    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
85    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
86    CacheDir "/srv/ftp-master.debian.org/database/";
87 };
88
89 Default
90 {
91    Packages::Compress "bzip2 gzip";
92    Sources::Compress "bzip2 gzip";
93    Contents::Compress "gzip";
94    DeLinkLimit 0;
95    MaxContentsChange 25000;
96    FileMode 0664;
97 }
98
99 TreeDefault
100 {
101    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
102 };
103
104 """
105
106     apt_trees={}
107     apt_trees["di"]={}
108
109     apt_trees["oldstable"]="""
110 tree "dists/oldstable"
111 {
112    FileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_binary-$(ARCH).list";
113    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_source.list";
114    Sections "main contrib non-free";
115    Architectures "%(arch)s";
116    BinOverride "override.lenny.$(SECTION)";
117    ExtraOverride "override.lenny.extra.$(SECTION)";
118    SrcOverride "override.lenny.$(SECTION).src";
119 };
120 """
121
122     apt_trees["di"]["oldstable"]="""
123 tree "dists/oldstable/main"
124 {
125    FileList "/srv/ftp-master.debian.org/database/dists/oldstable_main_$(SECTION)_binary-$(ARCH).list";
126    Sections "debian-installer";
127    Architectures "%(arch)s";
128    BinOverride "override.lenny.main.$(SECTION)";
129    SrcOverride "override.lenny.main.src";
130    BinCacheDB "packages-debian-installer-$(ARCH).db";
131    Packages::Extensions ".udeb";
132    %(contentsline)s
133 };
134
135 tree "dists/oldstable/non-free"
136 {
137    FileList "/srv/ftp-master.debian.org/database/dists/oldstable_non-free_$(SECTION)_binary-$(ARCH).list";
138    Sections "debian-installer";
139    Architectures "%(arch)s";
140    BinOverride "override.lenny.main.$(SECTION)";
141    SrcOverride "override.lenny.main.src";
142    BinCacheDB "packages-debian-installer-$(ARCH).db";
143    Packages::Extensions ".udeb";
144    %(contentsline)s
145 };
146 """
147
148     apt_trees["stable"]="""
149 tree "dists/stable"
150 {
151    FileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_binary-$(ARCH).list";
152    SourceFileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_source.list";
153    Sections "main contrib non-free";
154    Architectures "%(arch)s";
155    BinOverride "override.squeeze.$(SECTION)";
156    ExtraOverride "override.squeeze.extra.$(SECTION)";
157    SrcOverride "override.squeeze.$(SECTION).src";
158 };
159 """
160
161     apt_trees["di"]["stable"]="""
162 tree "dists/stable/main"
163 {
164    FileList "/srv/ftp-master.debian.org/database/dists/stable_main_$(SECTION)_binary-$(ARCH).list";
165    Sections "debian-installer";
166    Architectures "%(arch)s";
167    BinOverride "override.squeeze.main.$(SECTION)";
168    SrcOverride "override.squeeze.main.src";
169    BinCacheDB "packages-debian-installer-$(ARCH).db";
170    Packages::Extensions ".udeb";
171    %(contentsline)s
172 };
173
174 tree "dists/stable/non-free"
175 {
176    FileList "/srv/ftp-master.debian.org/database/dists/stable_non-free_$(SECTION)_binary-$(ARCH).list";
177    Sections "debian-installer";
178    Architectures "%(arch)s";
179    BinOverride "override.squeeze.main.$(SECTION)";
180    SrcOverride "override.squeeze.main.src";
181    BinCacheDB "packages-debian-installer-$(ARCH).db";
182    Packages::Extensions ".udeb";
183    %(contentsline)s
184 };
185 """
186
187     apt_trees["squeeze-updates"]="""
188 tree "dists/squeeze-updates"
189 {
190    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
191    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
192    Sections "main contrib non-free";
193    Architectures "%(arch)s";
194    BinOverride "override.squeeze.$(SECTION)";
195    ExtraOverride "override.squeeze.extra.$(SECTION)";
196    SrcOverride "override.squeeze.$(SECTION).src";
197    Contents " ";
198 };
199 """
200
201     apt_trees["testing"]="""
202 tree "dists/testing"
203 {
204    FakeDI "dists/unstable";
205    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
206    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
207    Sections "main contrib non-free";
208    Architectures "%(arch)s";
209    BinOverride "override.wheezy.$(SECTION)";
210    ExtraOverride "override.wheezy.extra.$(SECTION)";
211    SrcOverride "override.wheezy.$(SECTION).src";
212 };
213 """
214
215     apt_trees["di"]["testing"]="""
216 tree "dists/testing/main"
217 {
218    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
219    Sections "debian-installer";
220    Architectures "%(arch)s";
221    BinOverride "override.wheezy.main.$(SECTION)";
222    SrcOverride "override.wheezy.main.src";
223    BinCacheDB "packages-debian-installer-$(ARCH).db";
224    Packages::Extensions ".udeb";
225    %(contentsline)s
226 };
227
228 tree "dists/testing/non-free"
229 {
230    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
231    Sections "debian-installer";
232    Architectures "%(arch)s";
233    BinOverride "override.wheezy.main.$(SECTION)";
234    SrcOverride "override.wheezy.main.src";
235    BinCacheDB "packages-debian-installer-$(ARCH).db";
236    Packages::Extensions ".udeb";
237    %(contentsline)s
238 };
239 """
240
241     apt_trees["unstable"]="""
242 tree "dists/unstable"
243 {
244    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
245    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
246    Sections "main contrib non-free";
247    Architectures "%(arch)s";
248    BinOverride "override.sid.$(SECTION)";
249    ExtraOverride "override.sid.extra.$(SECTION)";
250    SrcOverride "override.sid.$(SECTION).src";
251 };
252 """
253     apt_trees["di"]["unstable"]="""
254 tree "dists/unstable/main"
255 {
256    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
257    Sections "debian-installer";
258    Architectures "%(arch)s";
259    BinOverride "override.sid.main.$(SECTION)";
260    SrcOverride "override.sid.main.src";
261    BinCacheDB "packages-debian-installer-$(ARCH).db";
262    Packages::Extensions ".udeb";
263    %(contentsline)s
264 };
265
266 tree "dists/unstable/non-free"
267 {
268    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
269    Sections "debian-installer";
270    Architectures "%(arch)s";
271    BinOverride "override.sid.main.$(SECTION)";
272    SrcOverride "override.sid.main.src";
273    BinCacheDB "packages-debian-installer-$(ARCH).db";
274    Packages::Extensions ".udeb";
275    %(contentsline)s
276 };
277 """
278
279     apt_trees["experimental"]="""
280 tree "dists/experimental"
281 {
282    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
283    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
284    Sections "main contrib non-free";
285    Architectures "%(arch)s";
286    BinOverride "override.sid.$(SECTION)";
287    SrcOverride "override.sid.$(SECTION).src";
288 };
289 """
290     apt_trees["di"]["experimental"]="""
291 tree "dists/experimental/main"
292 {
293    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
294    Sections "debian-installer";
295    Architectures "%(arch)s";
296    BinOverride "override.sid.main.$(SECTION)";
297    SrcOverride "override.sid.main.src";
298    BinCacheDB "packages-debian-installer-$(ARCH).db";
299    Packages::Extensions ".udeb";
300    %(contentsline)s
301 };
302
303 tree "dists/experimental/non-free"
304 {
305    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
306    Sections "debian-installer";
307    Architectures "%(arch)s";
308    BinOverride "override.sid.main.$(SECTION)";
309    SrcOverride "override.sid.main.src";
310    BinCacheDB "packages-debian-installer-$(ARCH).db";
311    Packages::Extensions ".udeb";
312    %(contentsline)s
313 };
314 """
315
316     apt_trees["testing-proposed-updates"]="""
317 tree "dists/testing-proposed-updates"
318 {
319    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
320    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
321    Sections "main contrib non-free";
322    Architectures "%(arch)s";
323    BinOverride "override.wheezy.$(SECTION)";
324    ExtraOverride "override.wheezy.extra.$(SECTION)";
325    SrcOverride "override.wheezy.$(SECTION).src";
326    Contents " ";
327 };
328 """
329     apt_trees["di"]["testing-proposed-updates"]="""
330 tree "dists/testing-proposed-updates/main"
331 {
332    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
333    Sections "debian-installer";
334    Architectures "%(arch)s";
335    BinOverride "override.wheezy.main.$(SECTION)";
336    SrcOverride "override.wheezy.main.src";
337    BinCacheDB "packages-debian-installer-$(ARCH).db";
338    Packages::Extensions ".udeb";
339    Contents " ";
340 };
341 """
342
343     apt_trees["proposed-updates"]="""
344 tree "dists/proposed-updates"
345 {
346    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
347    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
348    Sections "main contrib non-free";
349    Architectures "%(arch)s";
350    BinOverride "override.squeeze.$(SECTION)";
351    ExtraOverride "override.squeeze.extra.$(SECTION)";
352    SrcOverride "override.squeeze.$(SECTION).src";
353    Contents " ";
354 };
355 """
356     apt_trees["di"]["proposed-updates"]="""
357 tree "dists/proposed-updates/main"
358 {
359    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
360    Sections "debian-installer";
361    Architectures "%(arch)s";
362    BinOverride "override.squeeze.main.$(SECTION)";
363    SrcOverride "override.squeeze.main.src";
364    BinCacheDB "packages-debian-installer-$(ARCH).db";
365    Packages::Extensions ".udeb";
366    Contents " ";
367 };
368 """
369     apt_trees["oldstable-proposed-updates"]="""
370 tree "dists/oldstable-proposed-updates"
371 {
372    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
373    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
374    Sections "main contrib non-free";
375    Architectures "%(arch)s";
376    BinOverride "override.lenny.$(SECTION)";
377    ExtraOverride "override.lenny.extra.$(SECTION)";
378    SrcOverride "override.lenny.$(SECTION).src";
379    Contents " ";
380 };
381 """
382     apt_trees["di"]["oldstable-proposed-updates"]="""
383 tree "dists/oldstable-proposed-updates/main"
384 {
385    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
386    Sections "debian-installer";
387    Architectures "%(arch)s";
388    BinOverride "override.lenny.main.$(SECTION)";
389    SrcOverride "override.lenny.main.src";
390    BinCacheDB "packages-debian-installer-$(ARCH).db";
391    Packages::Extensions ".udeb";
392    Contents " ";
393 };
394 """
395
396     cnf = Config()
397     try:
398         # Write apt.conf
399         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
400         os.write(ac_fd, DAILY_APT_CONF)
401         # here we want to generate the tree entries
402         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
403         # this special casing needs to go away, but this whole thing may just want an
404         # aptconfig class anyways
405         if arch != 'source':
406             if arch == 'hurd-i386' and suite == 'experimental':
407                 pass
408             elif apt_trees["di"].has_key(suite):
409                 if arch == "amd64":
410                     os.write(ac_fd, apt_trees["di"][suite] %
411                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
412                 else:
413                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
414         os.close(ac_fd)
415
416         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
417         # Run apt-ftparchive generate
418         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
419         # it has errormessages we like to see
420         os.environ['GZIP'] = '--rsyncable'
421         os.chdir(tmppath)
422         (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
423         sn="a-f %s,%s: " % (suite, arch)
424         print sn + output.replace('\n', '\n%s' % (sn))
425         return result
426
427     # Clean up any left behind files
428     finally:
429         if ac_fd:
430             try:
431                 os.close(ac_fd)
432             except OSError:
433                 pass
434
435         if ac_name:
436             try:
437                 os.unlink(ac_name)
438             except OSError:
439                 pass
440
441 def sname(arch):
442     return arch.arch_string
443
444 def get_result(arg):
445     global results
446     if arg:
447         results.append(arg)
448
449 ########################################################################
450 ########################################################################
451
452 def main ():
453     global Options, Logger, results
454
455     cnf = Config()
456
457     for i in ["Help", "Suite", "Force"]:
458         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
459             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
460
461     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
462                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
463                  ('f',"force","Generate-Packages-Sources::Options::Force")]
464
465     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
466     Options = cnf.SubTree("Generate-Packages-Sources::Options")
467
468     if Options["Help"]:
469         usage()
470
471     Logger = daklog.Logger('generate-packages-sources')
472
473     session = DBConn().session()
474
475     if Options["Suite"]:
476         # Something here
477         suites = []
478         for s in suite_names:
479             suite = get_suite(s.lower(), session)
480             if suite:
481                 suites.append(suite)
482             else:
483                 print "cannot find suite %s" % s
484                 Logger.log(['cannot find suite %s' % s])
485     else:
486         suites=session.query(Suite).filter(Suite.untouchable == False).all()
487
488     startdir = os.getcwd()
489     os.chdir(cnf["Dir::TempPath"])
490
491     broken=[]
492     # For each given suite, each architecture, run one apt-ftparchive
493     for s in suites:
494         results=[]
495         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
496         pool = Pool()
497         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
498         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
499         for a in arch_list:
500             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
501
502         # No more work will be added to our pool, close it and then wait for all to finish
503         pool.close()
504         pool.join()
505
506     if len(results) > 0:
507         Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
508         print "Trouble, something with a-f broke, resultcodes: %s" % (results)
509         sys.exit(1)
510
511     os.chdir(startdir)
512     # this script doesn't change the database
513     session.close()
514     Logger.close()
515
516 #######################################################################################
517
518 if __name__ == '__main__':
519     main()