3 """ Generate Packages/Sources files
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
7 @copyright: 2009 Mark Hymers <mhy@debian.org>
8 @copyright: 2010 Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 ################################################################################
33 from tempfile import mkstemp, mkdtemp
35 from multiprocessing import Pool, TimeoutError
37 from daklib import daklog
38 from daklib.dbconn import *
39 from daklib.config import Config
41 ################################################################################
43 Options = None #: Commandline arguments parsed into this
44 Logger = None #: Our logging object
45 results = [] #: Results of the subprocesses
47 ################################################################################
49 def usage (exit_code=0):
50 print """Usage: dak generate-packages-sources [OPTIONS]
51 Generate the Packages/Sources files
53 -s, --suite=SUITE(s) process this suite
54 Default: All suites not marked 'untouchable'
55 -f, --force Allow processing of untouchable suites
56 CAREFUL: Only to be used at point release time!
57 -h, --help show this help and exit
59 SUITE can be a space seperated list, e.g.
60 --suite=unstable testing
65 ################################################################################
67 def generate_packages_sources(arch, suite, tmppath):
69 Generate Packages/Sources files with apt-ftparchive for the given suite/arch
72 @param suite: Suite name
75 @param arch: Architecture name
78 @param tmppath: The temporary path to work ing
84 ArchiveDir "/srv/ftp-master.debian.org/ftp/";
85 OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
86 CacheDir "/srv/ftp-master.debian.org/database/";
91 Packages::Compress "bzip2 gzip";
92 Sources::Compress "bzip2 gzip";
93 Contents::Compress "gzip";
95 MaxContentsChange 25000;
101 Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
109 apt_trees["oldstable"]="""
110 tree "dists/oldstable"
112 FileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_binary-$(ARCH).list";
113 SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_source.list";
114 Sections "main contrib non-free";
115 Architectures "%(arch)s";
116 BinOverride "override.squeeze.$(SECTION)";
117 ExtraOverride "override.squeeze.extra.$(SECTION)";
118 SrcOverride "override.squeeze.$(SECTION).src";
122 apt_trees["di"]["oldstable"]="""
123 tree "dists/oldstable/main"
125 FileList "/srv/ftp-master.debian.org/database/dists/oldstable_main_$(SECTION)_binary-$(ARCH).list";
126 Sections "debian-installer";
127 Architectures "%(arch)s";
128 BinOverride "override.squeeze.main.$(SECTION)";
129 SrcOverride "override.squeeze.main.src";
130 BinCacheDB "packages-debian-installer-$(ARCH).db";
131 Packages::Extensions ".udeb";
135 tree "dists/oldstable/non-free"
137 FileList "/srv/ftp-master.debian.org/database/dists/oldstable_non-free_$(SECTION)_binary-$(ARCH).list";
138 Sections "debian-installer";
139 Architectures "%(arch)s";
140 BinOverride "override.squeeze.main.$(SECTION)";
141 SrcOverride "override.squeeze.main.src";
142 BinCacheDB "packages-debian-installer-$(ARCH).db";
143 Packages::Extensions ".udeb";
152 (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
153 os.write(ac_fd, DAILY_APT_CONF)
154 # here we want to generate the tree entries
155 os.write(ac_fd, apt_trees[suite] % {'arch': arch})
156 # this special casing needs to go away, but this whole thing may just want an
157 # aptconfig class anyways
159 if arch == 'hurd-i386' and suite == 'experimental':
161 elif apt_trees["di"].has_key(suite):
163 os.write(ac_fd, apt_trees["di"][suite] %
164 {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
166 os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
169 print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
170 # Run apt-ftparchive generate
171 # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
172 # it has errormessages we like to see
173 os.environ['GZIP'] = '--rsyncable'
175 (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off -o APT::FTPArchive::SHA512=off generate %s' % os.path.basename(ac_name))
176 sn="a-f %s,%s: " % (suite, arch)
177 print sn + output.replace('\n', '\n%s' % (sn))
180 # Clean up any left behind files
195 return arch.arch_string
202 ########################################################################
203 ########################################################################
206 global Options, Logger, results
210 for i in ["Help", "Suite", "Force"]:
211 if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
212 cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
214 Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
215 ('s',"suite","Generate-Packages-Sources::Options::Suite"),
216 ('f',"force","Generate-Packages-Sources::Options::Force")]
218 suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
219 Options = cnf.subtree("Generate-Packages-Sources::Options")
224 Logger = daklog.Logger('generate-packages-sources')
226 session = DBConn().session()
231 for s in suite_names:
232 suite = get_suite(s.lower(), session)
236 print "cannot find suite %s" % s
237 Logger.log(['cannot find suite %s' % s])
239 suites=session.query(Suite).filter(Suite.untouchable == False).all()
241 startdir = os.getcwd()
242 os.chdir(cnf["Dir::TempPath"])
245 # For each given suite, each architecture, run one apt-ftparchive
248 # Setup a multiprocessing Pool. As many workers as we have CPU cores.
250 arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
251 Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
253 pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
255 # No more work will be added to our pool, close it and then wait for all to finish
260 Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
261 print "Trouble, something with a-f broke, resultcodes: %s" % (results)
265 # this script doesn't change the database
269 #######################################################################################
271 if __name__ == '__main__':