]> git.decadent.org.uk Git - dak.git/blob - dak/generate_packages_sources.py
Merge remote branch 'drkranz/master' into merge
[dak.git] / dak / generate_packages_sources.py
1 #!/usr/bin/env python
2
3 """ Generate Packages/Sources files
4
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
7 @copyright: 2009  Mark Hymers <mhy@debian.org>
8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
9
10 """
11
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 2 of the License, or
15 # (at your option) any later version.
16
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25
26 ################################################################################
27
28 import os
29 import os.path
30 import sys
31 import apt_pkg
32 from tempfile import mkstemp, mkdtemp
33 import commands
34 from multiprocessing import Pool
35
36 from daklib import daklog
37 from daklib.dbconn import *
38 from daklib.config import Config
39
40 ################################################################################
41
42 Options = None                 #: Commandline arguments parsed into this
43 Logger = None                  #: Our logging object
44
45 ################################################################################
46
47 def usage (exit_code=0):
48     print """Usage: dak generate-packages-sources [OPTIONS]
49 Generate the Packages/Sources files
50
51   -s, --suite=SUITE(s)       process this suite
52                              Default: All suites not marked 'untouchable'
53   -f, --force                Allow processing of untouchable suites
54                              CAREFUL: Only to be used at point release time!
55   -h, --help                 show this help and exit
56
57 SUITE can be a space seperated list, e.g.
58    --suite=unstable testing
59   """
60
61     sys.exit(exit_code)
62
63 ################################################################################
64
65 def generate_packages_sources(arch, suite, tmppath):
66     """
67     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
68
69     @type suite: string
70     @param suite: Suite name
71
72     @type arch: string
73     @param arch: Architecture name
74
75     @type tmppath: string
76     @param tmppath: The temporary path to work ing
77     """
78
79     DAILY_APT_CONF="""
80 Dir
81 {
82    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
83    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
84    CacheDir "/srv/ftp-master.debian.org/database/";
85 };
86
87 Default
88 {
89    Packages::Compress "bzip2 gzip";
90    Sources::Compress "bzip2 gzip";
91    Contents::Compress "gzip";
92    DeLinkLimit 0;
93    MaxContentsChange 25000;
94    FileMode 0664;
95 }
96
97 TreeDefault
98 {
99    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
100 };
101
102 """
103
104     apt_trees={}
105     apt_trees["di"]={}
106     apt_trees["testing"]="""
107 tree "dists/testing"
108 {
109    FakeDI "dists/unstable";
110    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
111    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
112    Sections "main contrib non-free";
113    Architectures "%(arch)s";
114    BinOverride "override.squeeze.$(SECTION)";
115    ExtraOverride "override.squeeze.extra.$(SECTION)";
116    SrcOverride "override.squeeze.$(SECTION).src";
117 };
118 """
119     apt_trees["di"]["testing"]="""
120 tree "dists/testing/main"
121 {
122    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
123    Sections "debian-installer";
124    Architectures "%(arch)s";
125    BinOverride "override.squeeze.main.$(SECTION)";
126    SrcOverride "override.squeeze.main.src";
127    BinCacheDB "packages-debian-installer-$(ARCH).db";
128    Packages::Extensions ".udeb";
129    %(contentsline)s
130 };
131
132 tree "dists/testing/non-free"
133 {
134    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
135    Sections "debian-installer";
136    Architectures "%(arch)s";
137    BinOverride "override.squeeze.main.$(SECTION)";
138    SrcOverride "override.squeeze.main.src";
139    BinCacheDB "packages-debian-installer-$(ARCH).db";
140    Packages::Extensions ".udeb";
141    %(contentsline)s
142 };
143 """
144
145     apt_trees["unstable"]="""
146 tree "dists/unstable"
147 {
148    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
149    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
150    Sections "main contrib non-free";
151    Architectures "%(arch)s";
152    BinOverride "override.sid.$(SECTION)";
153    ExtraOverride "override.sid.extra.$(SECTION)";
154    SrcOverride "override.sid.$(SECTION).src";
155 };
156 """
157     apt_trees["di"]["unstable"]="""
158 tree "dists/unstable/main"
159 {
160    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
161    Sections "debian-installer";
162    Architectures "%(arch)s";
163    BinOverride "override.sid.main.$(SECTION)";
164    SrcOverride "override.sid.main.src";
165    BinCacheDB "packages-debian-installer-$(ARCH).db";
166    Packages::Extensions ".udeb";
167    %(contentsline)s
168 };
169
170 tree "dists/unstable/non-free"
171 {
172    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
173    Sections "debian-installer";
174    Architectures "%(arch)s";
175    BinOverride "override.sid.main.$(SECTION)";
176    SrcOverride "override.sid.main.src";
177    BinCacheDB "packages-debian-installer-$(ARCH).db";
178    Packages::Extensions ".udeb";
179    %(contentsline)s
180 };
181 """
182
183     apt_trees["experimental"]="""
184 tree "dists/experimental"
185 {
186    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
187    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
188    Sections "main contrib non-free";
189    Architectures "%(arch)s";
190    BinOverride "override.sid.$(SECTION)";
191    SrcOverride "override.sid.$(SECTION).src";
192 };
193 """
194     apt_trees["di"]["experimental"]="""
195 tree "dists/experimental/main"
196 {
197    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
198    Sections "debian-installer";
199    Architectures "%(arch)s";
200    BinOverride "override.sid.main.$(SECTION)";
201    SrcOverride "override.sid.main.src";
202    BinCacheDB "packages-debian-installer-$(ARCH).db";
203    Packages::Extensions ".udeb";
204    %(contentsline)s
205 };
206
207 tree "dists/experimental/non-free"
208 {
209    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
210    Sections "debian-installer";
211    Architectures "%(arch)s";
212    BinOverride "override.sid.main.$(SECTION)";
213    SrcOverride "override.sid.main.src";
214    BinCacheDB "packages-debian-installer-$(ARCH).db";
215    Packages::Extensions ".udeb";
216    %(contentsline)s
217 };
218 """
219
220     apt_trees["testing-proposed-updates"]="""
221 tree "dists/testing-proposed-updates"
222 {
223    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
224    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
225    Sections "main contrib non-free";
226    Architectures "%(arch)s";
227    BinOverride "override.squeeze.$(SECTION)";
228    ExtraOverride "override.squeeze.extra.$(SECTION)";
229    SrcOverride "override.squeeze.$(SECTION).src";
230    Contents " ";
231 };
232 """
233     apt_trees["di"]["testing-proposed-updates"]="""
234 tree "dists/testing-proposed-updates/main"
235 {
236    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
237    Sections "debian-installer";
238    Architectures "%(arch)s";
239    BinOverride "override.squeeze.main.$(SECTION)";
240    SrcOverride "override.squeeze.main.src";
241    BinCacheDB "packages-debian-installer-$(ARCH).db";
242    Packages::Extensions ".udeb";
243    Contents " ";
244 };
245 """
246
247     apt_trees["proposed-updates"]="""
248 tree "dists/proposed-updates"
249 {
250    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
251    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
252    Sections "main contrib non-free";
253    Architectures "%(arch)s";
254    BinOverride "override.lenny.$(SECTION)";
255    ExtraOverride "override.lenny.extra.$(SECTION)";
256    SrcOverride "override.lenny.$(SECTION).src";
257    Contents " ";
258 };
259 """
260     apt_trees["di"]["proposed-updates"]="""
261 tree "dists/proposed-updates/main"
262 {
263    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
264    Sections "debian-installer";
265    Architectures "%(arch)s";
266    BinOverride "override.lenny.main.$(SECTION)";
267    SrcOverride "override.lenny.main.src";
268    BinCacheDB "packages-debian-installer-$(ARCH).db";
269    Packages::Extensions ".udeb";
270    Contents " ";
271 };
272 """
273
274     cnf = Config()
275     try:
276         # Write apt.conf
277         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
278         os.write(ac_fd, DAILY_APT_CONF)
279         # here we want to generate the tree entries
280         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
281         # this special casing needs to go away, but this whole thing may just want an
282         # aptconfig class anyways
283         if arch != 'source':
284             if arch == 'hurd-i386' and suite == 'experimental':
285                 pass
286             else:
287                 if arch == "amd64":
288                     os.write(ac_fd, apt_trees["di"][suite] %
289                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
290                 else:
291                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
292         os.close(ac_fd)
293
294         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
295         # Run apt-ftparchive generate
296         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
297         # it has errormessages we like to see
298         os.environ['GZIP'] = '--rsyncable'
299         os.chdir(tmppath)
300         (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
301         sn="a-f %s,%s: " % (suite, arch)
302         print sn + output.replace('\n', '\n%s' % (sn))
303
304     # Clean up any left behind files
305     finally:
306         if ac_fd:
307             try:
308                 os.close(ac_fd)
309             except OSError:
310                 pass
311
312         if ac_name:
313             try:
314                 os.unlink(ac_name)
315             except OSError:
316                 pass
317
318 def sname(arch):
319     return arch.arch_string
320
321 ########################################################################
322 ########################################################################
323
324 def main ():
325     global Options, Logger
326
327     cnf = Config()
328
329     for i in ["Help", "Suite", "Force"]:
330         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
331             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
332
333     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
334                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
335                  ('f',"force","Generate-Packages-Sources::Options::Force")]
336
337     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
338     Options = cnf.SubTree("Generate-Packages-Sources::Options")
339
340     if Options["Help"]:
341         usage()
342
343     Logger = daklog.Logger(cnf, 'generate-packages-sources')
344
345     session = DBConn().session()
346
347     if Options["Suite"]:
348         # Something here
349         suites = []
350         for s in suite_names:
351             suite = get_suite(s.lower(), session)
352             if suite:
353                 suites.append(suite)
354             else:
355                 print "cannot find suite %s" % s
356                 Logger.log(['cannot find suite %s' % s])
357     else:
358         suites=session.query(Suite).filter(Suite.untouchable == False).all()
359
360     startdir = os.getcwd()
361     os.chdir(cnf["Dir::TempPath"])
362
363     # For each given suite, each architecture, run one apt-ftparchive
364     for s in suites:
365         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
366         pool = Pool()
367         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
368         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
369         for a in arch_list:
370             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
371
372         # No more work will be added to our pool, close it and then wait for all to finish
373         pool.close()
374         pool.join()
375
376     os.chdir(startdir)
377     # this script doesn't change the database
378     session.close()
379     Logger.close()
380
381 #######################################################################################
382
383 if __name__ == '__main__':
384     main()