dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9
  10 """
  11
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 import os
  29 import os.path
  30 import sys
  31 import apt_pkg
  32 from tempfile import mkstemp, mkdtemp
  33 import commands
  34 from multiprocessing import Pool
  35
  36 from daklib import daklog
  37 from daklib.dbconn import *
  38 from daklib.config import Config
  39
  40 ################################################################################
  41
  42 Options = None                 #: Commandline arguments parsed into this
  43 Logger = None                  #: Our logging object
  44
  45 ################################################################################
  46
  47 def usage (exit_code=0):
  48     print """Usage: dak generate-packages-sources [OPTIONS]
  49 Generate the Packages/Sources files
  50
  51   -s, --suite=SUITE(s)       process this suite
  52                              Default: All suites not marked 'untouchable'
  53   -f, --force                Allow processing of untouchable suites
  54                              CAREFUL: Only to be used at point release time!
  55   -h, --help                 show this help and exit
  56
  57 SUITE can be a space seperated list, e.g.
  58    --suite=unstable testing
  59   """
  60
  61     sys.exit(exit_code)
  62
  63 ################################################################################
  64
  65 def generate_packages_sources(arch, suite, tmppath):
  66     """
  67     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  68
  69     @type suite: string
  70     @param suite: Suite name
  71
  72     @type arch: string
  73     @param arch: Architecture name
  74
  75     @type tmppath: string
  76     @param tmppath: The temporary path to work ing
  77     """
  78
  79     DAILY_APT_CONF="""
  80 Dir
  81 {
  82    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  83    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  84    CacheDir "/srv/ftp-master.debian.org/database/";
  85 };
  86
  87 Default
  88 {
  89    Packages::Compress "bzip2 gzip";
  90    Sources::Compress "bzip2 gzip";
  91    Contents::Compress "gzip";
  92    DeLinkLimit 0;
  93    MaxContentsChange 25000;
  94    FileMode 0664;
  95 }
  96
  97 TreeDefault
  98 {
  99    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 100 };
 101
 102 """
 103
 104     apt_trees={}
 105     apt_trees["di"]={}
 106     apt_trees["testing"]="""
 107 tree "dists/testing"
 108 {
 109    FakeDI "dists/unstable";
 110    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 111    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 112    Sections "main contrib non-free";
 113    Architectures "%(arch)s";
 114    BinOverride "override.wheezy.$(SECTION)";
 115    ExtraOverride "override.wheezy.extra.$(SECTION)";
 116    SrcOverride "override.wheezy.$(SECTION).src";
 117 };
 118 """
 119
 120     apt_trees["squeeze-updates"]="""
 121 tree "dists/squeeze-updates"
 122 {
 123    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
 124    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
 125    Sections "main contrib non-free";
 126    Architectures "%(arch)s";
 127    BinOverride "override.squeeze.$(SECTION)";
 128    ExtraOverride "override.squeeze.extra.$(SECTION)";
 129    SrcOverride "override.squeeze.$(SECTION).src";
 130    Contents " ";
 131 };
 132 """
 133
 134     apt_trees["di"]["testing"]="""
 135 tree "dists/testing/main"
 136 {
 137    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 138    Sections "debian-installer";
 139    Architectures "%(arch)s";
 140    BinOverride "override.wheezy.main.$(SECTION)";
 141    SrcOverride "override.wheezy.main.src";
 142    BinCacheDB "packages-debian-installer-$(ARCH).db";
 143    Packages::Extensions ".udeb";
 144    %(contentsline)s
 145 };
 146
 147 tree "dists/testing/non-free"
 148 {
 149    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 150    Sections "debian-installer";
 151    Architectures "%(arch)s";
 152    BinOverride "override.wheezy.main.$(SECTION)";
 153    SrcOverride "override.wheezy.main.src";
 154    BinCacheDB "packages-debian-installer-$(ARCH).db";
 155    Packages::Extensions ".udeb";
 156    %(contentsline)s
 157 };
 158 """
 159
 160     apt_trees["unstable"]="""
 161 tree "dists/unstable"
 162 {
 163    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 164    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 165    Sections "main contrib non-free";
 166    Architectures "%(arch)s";
 167    BinOverride "override.sid.$(SECTION)";
 168    ExtraOverride "override.sid.extra.$(SECTION)";
 169    SrcOverride "override.sid.$(SECTION).src";
 170 };
 171 """
 172     apt_trees["di"]["unstable"]="""
 173 tree "dists/unstable/main"
 174 {
 175    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 176    Sections "debian-installer";
 177    Architectures "%(arch)s";
 178    BinOverride "override.sid.main.$(SECTION)";
 179    SrcOverride "override.sid.main.src";
 180    BinCacheDB "packages-debian-installer-$(ARCH).db";
 181    Packages::Extensions ".udeb";
 182    %(contentsline)s
 183 };
 184
 185 tree "dists/unstable/non-free"
 186 {
 187    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 188    Sections "debian-installer";
 189    Architectures "%(arch)s";
 190    BinOverride "override.sid.main.$(SECTION)";
 191    SrcOverride "override.sid.main.src";
 192    BinCacheDB "packages-debian-installer-$(ARCH).db";
 193    Packages::Extensions ".udeb";
 194    %(contentsline)s
 195 };
 196 """
 197
 198     apt_trees["experimental"]="""
 199 tree "dists/experimental"
 200 {
 201    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 202    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 203    Sections "main contrib non-free";
 204    Architectures "%(arch)s";
 205    BinOverride "override.sid.$(SECTION)";
 206    SrcOverride "override.sid.$(SECTION).src";
 207 };
 208 """
 209     apt_trees["di"]["experimental"]="""
 210 tree "dists/experimental/main"
 211 {
 212    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 213    Sections "debian-installer";
 214    Architectures "%(arch)s";
 215    BinOverride "override.sid.main.$(SECTION)";
 216    SrcOverride "override.sid.main.src";
 217    BinCacheDB "packages-debian-installer-$(ARCH).db";
 218    Packages::Extensions ".udeb";
 219    %(contentsline)s
 220 };
 221
 222 tree "dists/experimental/non-free"
 223 {
 224    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 225    Sections "debian-installer";
 226    Architectures "%(arch)s";
 227    BinOverride "override.sid.main.$(SECTION)";
 228    SrcOverride "override.sid.main.src";
 229    BinCacheDB "packages-debian-installer-$(ARCH).db";
 230    Packages::Extensions ".udeb";
 231    %(contentsline)s
 232 };
 233 """
 234
 235     apt_trees["testing-proposed-updates"]="""
 236 tree "dists/testing-proposed-updates"
 237 {
 238    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 239    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 240    Sections "main contrib non-free";
 241    Architectures "%(arch)s";
 242    BinOverride "override.wheezy.$(SECTION)";
 243    ExtraOverride "override.wheezy.extra.$(SECTION)";
 244    SrcOverride "override.wheezy.$(SECTION).src";
 245    Contents " ";
 246 };
 247 """
 248     apt_trees["di"]["testing-proposed-updates"]="""
 249 tree "dists/testing-proposed-updates/main"
 250 {
 251    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 252    Sections "debian-installer";
 253    Architectures "%(arch)s";
 254    BinOverride "override.wheezy.main.$(SECTION)";
 255    SrcOverride "override.wheezy.main.src";
 256    BinCacheDB "packages-debian-installer-$(ARCH).db";
 257    Packages::Extensions ".udeb";
 258    Contents " ";
 259 };
 260 """
 261
 262     apt_trees["proposed-updates"]="""
 263 tree "dists/proposed-updates"
 264 {
 265    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 266    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 267    Sections "main contrib non-free";
 268    Architectures "%(arch)s";
 269    BinOverride "override.squeeze.$(SECTION)";
 270    ExtraOverride "override.squeeze.extra.$(SECTION)";
 271    SrcOverride "override.squeeze.$(SECTION).src";
 272    Contents " ";
 273 };
 274 """
 275     apt_trees["di"]["proposed-updates"]="""
 276 tree "dists/proposed-updates/main"
 277 {
 278    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 279    Sections "debian-installer";
 280    Architectures "%(arch)s";
 281    BinOverride "override.squeeze.main.$(SECTION)";
 282    SrcOverride "override.squeeze.main.src";
 283    BinCacheDB "packages-debian-installer-$(ARCH).db";
 284    Packages::Extensions ".udeb";
 285    Contents " ";
 286 };
 287 """
 288     apt_trees["oldstable-proposed-updates"]="""
 289 tree "dists/oldstable-proposed-updates"
 290 {
 291    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 292    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
 293    Sections "main contrib non-free";
 294    Architectures "%(arch)s";
 295    BinOverride "override.lenny.$(SECTION)";
 296    ExtraOverride "override.lenny.extra.$(SECTION)";
 297    SrcOverride "override.lenny.$(SECTION).src";
 298    Contents " ";
 299 };
 300 """
 301     apt_trees["di"]["oldstable-proposed-updates"]="""
 302 tree "dists/oldstable-proposed-updates/main"
 303 {
 304    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 305    Sections "debian-installer";
 306    Architectures "%(arch)s";
 307    BinOverride "override.lenny.main.$(SECTION)";
 308    SrcOverride "override.lenny.main.src";
 309    BinCacheDB "packages-debian-installer-$(ARCH).db";
 310    Packages::Extensions ".udeb";
 311    Contents " ";
 312 };
 313 """
 314
 315     cnf = Config()
 316     try:
 317         # Write apt.conf
 318         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 319         os.write(ac_fd, DAILY_APT_CONF)
 320         # here we want to generate the tree entries
 321         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 322         # this special casing needs to go away, but this whole thing may just want an
 323         # aptconfig class anyways
 324         if arch != 'source':
 325             if arch == 'hurd-i386' and suite == 'experimental':
 326                 pass
 327             elif apt_trees["di"].has_key(suite):
 328                 if arch == "amd64":
 329                     os.write(ac_fd, apt_trees["di"][suite] %
 330                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 331                 else:
 332                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 333         os.close(ac_fd)
 334
 335         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 336         # Run apt-ftparchive generate
 337         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 338         # it has errormessages we like to see
 339         os.environ['GZIP'] = '--rsyncable'
 340         os.chdir(tmppath)
 341         (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
 342         sn="a-f %s,%s: " % (suite, arch)
 343         print sn + output.replace('\n', '\n%s' % (sn))
 344
 345     # Clean up any left behind files
 346     finally:
 347         if ac_fd:
 348             try:
 349                 os.close(ac_fd)
 350             except OSError:
 351                 pass
 352
 353         if ac_name:
 354             try:
 355                 os.unlink(ac_name)
 356             except OSError:
 357                 pass
 358
 359 def sname(arch):
 360     return arch.arch_string
 361
 362 ########################################################################
 363 ########################################################################
 364
 365 def main ():
 366     global Options, Logger
 367
 368     cnf = Config()
 369
 370     for i in ["Help", "Suite", "Force"]:
 371         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 372             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 373
 374     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 375                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 376                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 377
 378     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 379     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 380
 381     if Options["Help"]:
 382         usage()
 383
 384     Logger = daklog.Logger(cnf, 'generate-packages-sources')
 385
 386     session = DBConn().session()
 387
 388     if Options["Suite"]:
 389         # Something here
 390         suites = []
 391         for s in suite_names:
 392             suite = get_suite(s.lower(), session)
 393             if suite:
 394                 suites.append(suite)
 395             else:
 396                 print "cannot find suite %s" % s
 397                 Logger.log(['cannot find suite %s' % s])
 398     else:
 399         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 400
 401     startdir = os.getcwd()
 402     os.chdir(cnf["Dir::TempPath"])
 403
 404     # For each given suite, each architecture, run one apt-ftparchive
 405     for s in suites:
 406         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 407         pool = Pool()
 408         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
 409         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 410         for a in arch_list:
 411             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
 412
 413         # No more work will be added to our pool, close it and then wait for all to finish
 414         pool.close()
 415         pool.join()
 416
 417     os.chdir(startdir)
 418     # this script doesn't change the database
 419     session.close()
 420     Logger.close()
 421
 422 #######################################################################################
 423
 424 if __name__ == '__main__':
 425     main()