dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9
  10 """
  11
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 import os
  29 import os.path
  30 import sys
  31 import apt_pkg
  32 from tempfile import mkstemp, mkdtemp
  33 import commands
  34 from multiprocessing import Pool
  35
  36 from daklib import daklog
  37 from daklib.dbconn import *
  38 from daklib.config import Config
  39
  40 ################################################################################
  41
  42 Options = None                 #: Commandline arguments parsed into this
  43 Logger = None                  #: Our logging object
  44
  45 ################################################################################
  46
  47 def usage (exit_code=0):
  48     print """Usage: dak generate-packages-sources [OPTIONS]
  49 Generate the Packages/Sources files
  50
  51   -s, --suite=SUITE(s)       process this suite
  52                              Default: All suites not marked 'untouchable'
  53   -f, --force                Allow processing of untouchable suites
  54                              CAREFUL: Only to be used at point release time!
  55   -h, --help                 show this help and exit
  56
  57 SUITE can be a space seperated list, e.g.
  58    --suite=unstable testing
  59   """
  60
  61     sys.exit(exit_code)
  62
  63 ################################################################################
  64
  65 def generate_packages_sources(arch, suite, tmppath):
  66     """
  67     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  68
  69     @type suite: string
  70     @param suite: Suite name
  71
  72     @type arch: string
  73     @param arch: Architecture name
  74
  75     @type tmppath: string
  76     @param tmppath: The temporary path to work ing
  77     """
  78
  79     DAILY_APT_CONF="""
  80 Dir
  81 {
  82    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  83    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  84    CacheDir "/srv/ftp-master.debian.org/database/";
  85 };
  86
  87 Default
  88 {
  89    Packages::Compress "bzip2 gzip";
  90    Sources::Compress "bzip2 gzip";
  91    Contents::Compress "gzip";
  92    DeLinkLimit 0;
  93    MaxContentsChange 25000;
  94    FileMode 0664;
  95 }
  96
  97 TreeDefault
  98 {
  99    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 100 };
 101
 102 """
 103
 104     apt_trees={}
 105     apt_trees["di"]={}
 106     apt_trees["testing"]="""
 107 tree "dists/testing"
 108 {
 109    FakeDI "dists/unstable";
 110    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 111    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 112    Sections "main contrib non-free";
 113    Architectures "%(arch)s";
 114    BinOverride "override.squeeze.$(SECTION)";
 115    ExtraOverride "override.squeeze.extra.$(SECTION)";
 116    SrcOverride "override.squeeze.$(SECTION).src";
 117 };
 118 """
 119     apt_trees["di"]["testing"]="""
 120 tree "dists/testing/main"
 121 {
 122    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 123    Sections "debian-installer";
 124    Architectures "%(arch)s";
 125    BinOverride "override.squeeze.main.$(SECTION)";
 126    SrcOverride "override.squeeze.main.src";
 127    BinCacheDB "packages-debian-installer-$(ARCH).db";
 128    Packages::Extensions ".udeb";
 129    %(contentsline)s
 130 };
 131
 132 tree "dists/testing/non-free"
 133 {
 134    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 135    Sections "debian-installer";
 136    Architectures "%(arch)s";
 137    BinOverride "override.squeeze.main.$(SECTION)";
 138    SrcOverride "override.squeeze.main.src";
 139    BinCacheDB "packages-debian-installer-$(ARCH).db";
 140    Packages::Extensions ".udeb";
 141    %(contentsline)s
 142 };
 143 """
 144
 145     apt_trees["unstable"]="""
 146 tree "dists/unstable"
 147 {
 148    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 149    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 150    Sections "main contrib non-free";
 151    Architectures "%(arch)s";
 152    BinOverride "override.sid.$(SECTION)";
 153    ExtraOverride "override.sid.extra.$(SECTION)";
 154    SrcOverride "override.sid.$(SECTION).src";
 155 };
 156 """
 157     apt_trees["di"]["unstable"]="""
 158 tree "dists/unstable/main"
 159 {
 160    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 161    Sections "debian-installer";
 162    Architectures "%(arch)s";
 163    BinOverride "override.sid.main.$(SECTION)";
 164    SrcOverride "override.sid.main.src";
 165    BinCacheDB "packages-debian-installer-$(ARCH).db";
 166    Packages::Extensions ".udeb";
 167    %(contentsline)s
 168 };
 169
 170 tree "dists/unstable/non-free"
 171 {
 172    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 173    Sections "debian-installer";
 174    Architectures "%(arch)s";
 175    BinOverride "override.sid.main.$(SECTION)";
 176    SrcOverride "override.sid.main.src";
 177    BinCacheDB "packages-debian-installer-$(ARCH).db";
 178    Packages::Extensions ".udeb";
 179    %(contentsline)s
 180 };
 181 """
 182
 183     apt_trees["experimental"]="""
 184 tree "dists/experimental"
 185 {
 186    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 187    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 188    Sections "main contrib non-free";
 189    Architectures "%(arch)s";
 190    BinOverride "override.sid.$(SECTION)";
 191    SrcOverride "override.sid.$(SECTION).src";
 192 };
 193 """
 194     apt_trees["di"]["experimental"]="""
 195 tree "dists/experimental/main"
 196 {
 197    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 198    Sections "debian-installer";
 199    Architectures "%(arch)s";
 200    BinOverride "override.sid.main.$(SECTION)";
 201    SrcOverride "override.sid.main.src";
 202    BinCacheDB "packages-debian-installer-$(ARCH).db";
 203    Packages::Extensions ".udeb";
 204    %(contentsline)s
 205 };
 206
 207 tree "dists/experimental/non-free"
 208 {
 209    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 210    Sections "debian-installer";
 211    Architectures "%(arch)s";
 212    BinOverride "override.sid.main.$(SECTION)";
 213    SrcOverride "override.sid.main.src";
 214    BinCacheDB "packages-debian-installer-$(ARCH).db";
 215    Packages::Extensions ".udeb";
 216    %(contentsline)s
 217 };
 218 """
 219
 220     apt_trees["testing-proposed-updates"]="""
 221 tree "dists/testing-proposed-updates"
 222 {
 223    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 224    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 225    Sections "main contrib non-free";
 226    Architectures "%(arch)s";
 227    BinOverride "override.squeeze.$(SECTION)";
 228    ExtraOverride "override.squeeze.extra.$(SECTION)";
 229    SrcOverride "override.squeeze.$(SECTION).src";
 230    Contents " ";
 231 };
 232 """
 233     apt_trees["di"]["testing-proposed-updates"]="""
 234 tree "dists/testing-proposed-updates/main"
 235 {
 236    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 237    Sections "debian-installer";
 238    Architectures "%(arch)s";
 239    BinOverride "override.squeeze.main.$(SECTION)";
 240    SrcOverride "override.squeeze.main.src";
 241    BinCacheDB "packages-debian-installer-$(ARCH).db";
 242    Packages::Extensions ".udeb";
 243    Contents " ";
 244 };
 245 """
 246
 247     apt_trees["proposed-updates"]="""
 248 tree "dists/proposed-updates"
 249 {
 250    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 251    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 252    Sections "main contrib non-free";
 253    Architectures "%(arch)s";
 254    BinOverride "override.lenny.$(SECTION)";
 255    ExtraOverride "override.lenny.extra.$(SECTION)";
 256    SrcOverride "override.lenny.$(SECTION).src";
 257    Contents " ";
 258 };
 259 """
 260     apt_trees["di"]["proposed-updates"]="""
 261 tree "dists/proposed-updates/main"
 262 {
 263    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 264    Sections "debian-installer";
 265    Architectures "%(arch)s";
 266    BinOverride "override.lenny.main.$(SECTION)";
 267    SrcOverride "override.lenny.main.src";
 268    BinCacheDB "packages-debian-installer-$(ARCH).db";
 269    Packages::Extensions ".udeb";
 270    Contents " ";
 271 };
 272 """
 273
 274     cnf = Config()
 275     try:
 276         # Write apt.conf
 277         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 278         os.write(ac_fd, DAILY_APT_CONF)
 279         # here we want to generate the tree entries
 280         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 281         # this special casing needs to go away, but this whole thing may just want an
 282         # aptconfig class anyways
 283         if arch != 'source':
 284             if arch == 'hurd-i386' and suite == 'experimental':
 285                 pass
 286             else:
 287                 if arch == "amd64":
 288                     os.write(ac_fd, apt_trees["di"][suite] %
 289                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 290                 else:
 291                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 292         os.close(ac_fd)
 293
 294         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 295         # Run apt-ftparchive generate
 296         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 297         # it has errormessages we like to see
 298         os.environ['GZIP'] = '--rsyncable'
 299         os.chdir(tmppath)
 300         (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
 301         sn="a-f %s,%s: " % (suite, arch)
 302         print sn + output.replace('\n', '\n%s' % (sn))
 303
 304     # Clean up any left behind files
 305     finally:
 306         if ac_fd:
 307             try:
 308                 os.close(ac_fd)
 309             except OSError:
 310                 pass
 311
 312         if ac_name:
 313             try:
 314                 os.unlink(ac_name)
 315             except OSError:
 316                 pass
 317
 318 def sname(arch):
 319     return arch.arch_string
 320
 321 ########################################################################
 322 ########################################################################
 323
 324 def main ():
 325     global Options, Logger
 326
 327     cnf = Config()
 328
 329     for i in ["Help", "Suite", "Force"]:
 330         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 331             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 332
 333     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 334                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 335                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 336
 337     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 338     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 339
 340     if Options["Help"]:
 341         usage()
 342
 343     Logger = daklog.Logger(cnf, 'generate-packages-sources')
 344
 345     session = DBConn().session()
 346
 347     if Options["Suite"]:
 348         # Something here
 349         suites = []
 350         for s in suite_names:
 351             suite = get_suite(s.lower(), session)
 352             if suite:
 353                 suites.append(suite)
 354             else:
 355                 print "cannot find suite %s" % s
 356                 Logger.log(['cannot find suite %s' % s])
 357     else:
 358         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 359
 360     startdir = os.getcwd()
 361     os.chdir(cnf["Dir::TempPath"])
 362
 363     # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 364     pool = Pool()
 365
 366     # For each given suite, each architecture, run one apt-ftparchive
 367     for s in suites:
 368         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
 369         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 370         for a in arch_list:
 371             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
 372
 373     # No more work will be added to our pool, close it and then wait for all to finish
 374     pool.close()
 375     pool.join()
 376     os.chdir(startdir)
 377     # this script doesn't change the database
 378     session.close()
 379     Logger.close()
 380
 381 #######################################################################################
 382
 383 if __name__ == '__main__':
 384     main()