dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9
  10 """
  11
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 import os
  29 import os.path
  30 import sys
  31 import apt_pkg
  32 from tempfile import mkstemp, mkdtemp
  33 import commands
  34 from multiprocessing import Pool
  35
  36 from daklib import daklog
  37 from daklib.dbconn import *
  38 from daklib.config import Config
  39
  40 ################################################################################
  41
  42 Options = None                 #: Commandline arguments parsed into this
  43 Logger = None                  #: Our logging object
  44
  45 ################################################################################
  46
  47 def usage (exit_code=0):
  48     print """Usage: dak generate-packages-sources [OPTIONS]
  49 Generate the Packages/Sources files
  50
  51   -s, --suite=SUITE(s)       process this suite
  52                              Default: All suites not marked 'untouchable'
  53   -f, --force                Allow processing of untouchable suites
  54                              CAREFUL: Only to be used at point release time!
  55   -h, --help                 show this help and exit
  56
  57 SUITE can be a space seperated list, e.g.
  58    --suite=unstable testing
  59   """
  60
  61     sys.exit(exit_code)
  62
  63 ################################################################################
  64
  65 def generate_packages_sources(arch, suite, tmppath):
  66     """
  67     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  68
  69     @type suite: string
  70     @param suite: Suite name
  71
  72     @type arch: string
  73     @param arch: Architecture name
  74
  75     @type tmppath: string
  76     @param tmppath: The temporary path to work ing
  77     """
  78
  79     DAILY_APT_CONF="""
  80 Dir
  81 {
  82    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  83    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  84    CacheDir "/srv/ftp-master.debian.org/database/";
  85 };
  86
  87 Default
  88 {
  89    Packages::Compress "bzip2 gzip";
  90    Sources::Compress "bzip2 gzip";
  91    Contents::Compress "gzip";
  92    DeLinkLimit 0;
  93    MaxContentsChange 25000;
  94    FileMode 0664;
  95 }
  96
  97 TreeDefault
  98 {
  99    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 100 };
 101
 102 """
 103
 104     apt_trees={}
 105     apt_trees["di"]={}
 106     apt_trees["testing"]="""
 107 tree "dists/testing"
 108 {
 109    FakeDI "dists/unstable";
 110    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 111    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 112    Sections "main contrib non-free";
 113    Architectures "%(arch)s";
 114    BinOverride "override.squeeze.$(SECTION)";
 115    ExtraOverride "override.squeeze.extra.$(SECTION)";
 116    SrcOverride "override.squeeze.$(SECTION).src";
 117 };
 118 """
 119
 120     apt_trees["squeeze-updates"]="""
 121 tree "dists/squeeze-updates"
 122 {
 123    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
 124    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
 125    Sections "main contrib non-free";
 126    Architectures "%(arch)s";
 127    BinOverride "override.squeeze.$(SECTION)";
 128    ExtraOverride "override.squeeze.extra.$(SECTION)";
 129    SrcOverride "override.squeeze.$(SECTION).src";
 130    Contents " ";
 131 };
 132 """
 133
 134     apt_trees["di"]["testing"]="""
 135 tree "dists/testing/main"
 136 {
 137    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 138    Sections "debian-installer";
 139    Architectures "%(arch)s";
 140    BinOverride "override.squeeze.main.$(SECTION)";
 141    SrcOverride "override.squeeze.main.src";
 142    BinCacheDB "packages-debian-installer-$(ARCH).db";
 143    Packages::Extensions ".udeb";
 144    %(contentsline)s
 145 };
 146
 147 tree "dists/testing/non-free"
 148 {
 149    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 150    Sections "debian-installer";
 151    Architectures "%(arch)s";
 152    BinOverride "override.squeeze.main.$(SECTION)";
 153    SrcOverride "override.squeeze.main.src";
 154    BinCacheDB "packages-debian-installer-$(ARCH).db";
 155    Packages::Extensions ".udeb";
 156    %(contentsline)s
 157 };
 158 """
 159
 160     apt_trees["unstable"]="""
 161 tree "dists/unstable"
 162 {
 163    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 164    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 165    Sections "main contrib non-free";
 166    Architectures "%(arch)s";
 167    BinOverride "override.sid.$(SECTION)";
 168    ExtraOverride "override.sid.extra.$(SECTION)";
 169    SrcOverride "override.sid.$(SECTION).src";
 170 };
 171 """
 172     apt_trees["di"]["unstable"]="""
 173 tree "dists/unstable/main"
 174 {
 175    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 176    Sections "debian-installer";
 177    Architectures "%(arch)s";
 178    BinOverride "override.sid.main.$(SECTION)";
 179    SrcOverride "override.sid.main.src";
 180    BinCacheDB "packages-debian-installer-$(ARCH).db";
 181    Packages::Extensions ".udeb";
 182    %(contentsline)s
 183 };
 184
 185 tree "dists/unstable/non-free"
 186 {
 187    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 188    Sections "debian-installer";
 189    Architectures "%(arch)s";
 190    BinOverride "override.sid.main.$(SECTION)";
 191    SrcOverride "override.sid.main.src";
 192    BinCacheDB "packages-debian-installer-$(ARCH).db";
 193    Packages::Extensions ".udeb";
 194    %(contentsline)s
 195 };
 196 """
 197
 198     apt_trees["experimental"]="""
 199 tree "dists/experimental"
 200 {
 201    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 202    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 203    Sections "main contrib non-free";
 204    Architectures "%(arch)s";
 205    BinOverride "override.sid.$(SECTION)";
 206    SrcOverride "override.sid.$(SECTION).src";
 207 };
 208 """
 209     apt_trees["di"]["experimental"]="""
 210 tree "dists/experimental/main"
 211 {
 212    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 213    Sections "debian-installer";
 214    Architectures "%(arch)s";
 215    BinOverride "override.sid.main.$(SECTION)";
 216    SrcOverride "override.sid.main.src";
 217    BinCacheDB "packages-debian-installer-$(ARCH).db";
 218    Packages::Extensions ".udeb";
 219    %(contentsline)s
 220 };
 221
 222 tree "dists/experimental/non-free"
 223 {
 224    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 225    Sections "debian-installer";
 226    Architectures "%(arch)s";
 227    BinOverride "override.sid.main.$(SECTION)";
 228    SrcOverride "override.sid.main.src";
 229    BinCacheDB "packages-debian-installer-$(ARCH).db";
 230    Packages::Extensions ".udeb";
 231    %(contentsline)s
 232 };
 233 """
 234
 235     apt_trees["testing-proposed-updates"]="""
 236 tree "dists/testing-proposed-updates"
 237 {
 238    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 239    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 240    Sections "main contrib non-free";
 241    Architectures "%(arch)s";
 242    BinOverride "override.squeeze.$(SECTION)";
 243    ExtraOverride "override.squeeze.extra.$(SECTION)";
 244    SrcOverride "override.squeeze.$(SECTION).src";
 245    Contents " ";
 246 };
 247 """
 248     apt_trees["di"]["testing-proposed-updates"]="""
 249 tree "dists/testing-proposed-updates/main"
 250 {
 251    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 252    Sections "debian-installer";
 253    Architectures "%(arch)s";
 254    BinOverride "override.squeeze.main.$(SECTION)";
 255    SrcOverride "override.squeeze.main.src";
 256    BinCacheDB "packages-debian-installer-$(ARCH).db";
 257    Packages::Extensions ".udeb";
 258    Contents " ";
 259 };
 260 """
 261
 262     apt_trees["proposed-updates"]="""
 263 tree "dists/proposed-updates"
 264 {
 265    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 266    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 267    Sections "main contrib non-free";
 268    Architectures "%(arch)s";
 269    BinOverride "override.lenny.$(SECTION)";
 270    ExtraOverride "override.lenny.extra.$(SECTION)";
 271    SrcOverride "override.lenny.$(SECTION).src";
 272    Contents " ";
 273 };
 274 """
 275     apt_trees["di"]["proposed-updates"]="""
 276 tree "dists/proposed-updates/main"
 277 {
 278    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 279    Sections "debian-installer";
 280    Architectures "%(arch)s";
 281    BinOverride "override.lenny.main.$(SECTION)";
 282    SrcOverride "override.lenny.main.src";
 283    BinCacheDB "packages-debian-installer-$(ARCH).db";
 284    Packages::Extensions ".udeb";
 285    Contents " ";
 286 };
 287 """
 288
 289     cnf = Config()
 290     try:
 291         # Write apt.conf
 292         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 293         os.write(ac_fd, DAILY_APT_CONF)
 294         # here we want to generate the tree entries
 295         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 296         # this special casing needs to go away, but this whole thing may just want an
 297         # aptconfig class anyways
 298         if arch != 'source':
 299             if arch == 'hurd-i386' and suite == 'experimental':
 300                 pass
 301             elif apt_trees["di"].has_key(suite):
 302                 if arch == "amd64":
 303                     os.write(ac_fd, apt_trees["di"][suite] %
 304                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 305                 else:
 306                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 307         os.close(ac_fd)
 308
 309         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 310         # Run apt-ftparchive generate
 311         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 312         # it has errormessages we like to see
 313         os.environ['GZIP'] = '--rsyncable'
 314         os.chdir(tmppath)
 315         (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
 316         sn="a-f %s,%s: " % (suite, arch)
 317         print sn + output.replace('\n', '\n%s' % (sn))
 318
 319     # Clean up any left behind files
 320     finally:
 321         if ac_fd:
 322             try:
 323                 os.close(ac_fd)
 324             except OSError:
 325                 pass
 326
 327         if ac_name:
 328             try:
 329                 os.unlink(ac_name)
 330             except OSError:
 331                 pass
 332
 333 def sname(arch):
 334     return arch.arch_string
 335
 336 ########################################################################
 337 ########################################################################
 338
 339 def main ():
 340     global Options, Logger
 341
 342     cnf = Config()
 343
 344     for i in ["Help", "Suite", "Force"]:
 345         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 346             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 347
 348     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 349                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 350                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 351
 352     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 353     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 354
 355     if Options["Help"]:
 356         usage()
 357
 358     Logger = daklog.Logger(cnf, 'generate-packages-sources')
 359
 360     session = DBConn().session()
 361
 362     if Options["Suite"]:
 363         # Something here
 364         suites = []
 365         for s in suite_names:
 366             suite = get_suite(s.lower(), session)
 367             if suite:
 368                 suites.append(suite)
 369             else:
 370                 print "cannot find suite %s" % s
 371                 Logger.log(['cannot find suite %s' % s])
 372     else:
 373         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 374
 375     startdir = os.getcwd()
 376     os.chdir(cnf["Dir::TempPath"])
 377
 378     # For each given suite, each architecture, run one apt-ftparchive
 379     for s in suites:
 380         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 381         pool = Pool()
 382         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
 383         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 384         for a in arch_list:
 385             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
 386
 387         # No more work will be added to our pool, close it and then wait for all to finish
 388         pool.close()
 389         pool.join()
 390
 391     os.chdir(startdir)
 392     # this script doesn't change the database
 393     session.close()
 394     Logger.close()
 395
 396 #######################################################################################
 397
 398 if __name__ == '__main__':
 399     main()