dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9
  10 """
  11
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28 import os
  29 import os.path
  30 import sys
  31 import apt_pkg
  32 from tempfile import mkstemp, mkdtemp
  33 import commands
  34 from multiprocessing import Pool, TimeoutError
  35
  36 from daklib import daklog
  37 from daklib.dbconn import *
  38 from daklib.config import Config
  39
  40 ################################################################################
  41
  42 Options = None                 #: Commandline arguments parsed into this
  43 Logger = None                  #: Our logging object
  44 results = []                   #: Results of the subprocesses
  45
  46 ################################################################################
  47
  48 def usage (exit_code=0):
  49     print """Usage: dak generate-packages-sources [OPTIONS]
  50 Generate the Packages/Sources files
  51
  52   -s, --suite=SUITE(s)       process this suite
  53                              Default: All suites not marked 'untouchable'
  54   -f, --force                Allow processing of untouchable suites
  55                              CAREFUL: Only to be used at point release time!
  56   -h, --help                 show this help and exit
  57
  58 SUITE can be a space seperated list, e.g.
  59    --suite=unstable testing
  60   """
  61
  62     sys.exit(exit_code)
  63
  64 ################################################################################
  65
  66 def generate_packages_sources(arch, suite, tmppath):
  67     """
  68     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  69
  70     @type suite: string
  71     @param suite: Suite name
  72
  73     @type arch: string
  74     @param arch: Architecture name
  75
  76     @type tmppath: string
  77     @param tmppath: The temporary path to work ing
  78     """
  79
  80     DAILY_APT_CONF="""
  81 Dir
  82 {
  83    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  84    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  85    CacheDir "/srv/ftp-master.debian.org/database/";
  86 };
  87
  88 Default
  89 {
  90    Packages::Compress "bzip2 gzip";
  91    Sources::Compress "bzip2 gzip";
  92    Contents::Compress "gzip";
  93    DeLinkLimit 0;
  94    MaxContentsChange 25000;
  95    FileMode 0664;
  96 }
  97
  98 TreeDefault
  99 {
 100    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 101 };
 102
 103 """
 104
 105     apt_trees={}
 106     apt_trees["di"]={}
 107     apt_trees["testing"]="""
 108 tree "dists/testing"
 109 {
 110    FakeDI "dists/unstable";
 111    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 112    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 113    Sections "main contrib non-free";
 114    Architectures "%(arch)s";
 115    BinOverride "override.wheezy.$(SECTION)";
 116    ExtraOverride "override.wheezy.extra.$(SECTION)";
 117    SrcOverride "override.wheezy.$(SECTION).src";
 118 };
 119 """
 120
 121     apt_trees["squeeze-updates"]="""
 122 tree "dists/squeeze-updates"
 123 {
 124    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
 125    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
 126    Sections "main contrib non-free";
 127    Architectures "%(arch)s";
 128    BinOverride "override.squeeze.$(SECTION)";
 129    ExtraOverride "override.squeeze.extra.$(SECTION)";
 130    SrcOverride "override.squeeze.$(SECTION).src";
 131    Contents " ";
 132 };
 133 """
 134
 135     apt_trees["di"]["testing"]="""
 136 tree "dists/testing/main"
 137 {
 138    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 139    Sections "debian-installer";
 140    Architectures "%(arch)s";
 141    BinOverride "override.wheezy.main.$(SECTION)";
 142    SrcOverride "override.wheezy.main.src";
 143    BinCacheDB "packages-debian-installer-$(ARCH).db";
 144    Packages::Extensions ".udeb";
 145    %(contentsline)s
 146 };
 147
 148 tree "dists/testing/non-free"
 149 {
 150    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 151    Sections "debian-installer";
 152    Architectures "%(arch)s";
 153    BinOverride "override.wheezy.main.$(SECTION)";
 154    SrcOverride "override.wheezy.main.src";
 155    BinCacheDB "packages-debian-installer-$(ARCH).db";
 156    Packages::Extensions ".udeb";
 157    %(contentsline)s
 158 };
 159 """
 160
 161     apt_trees["unstable"]="""
 162 tree "dists/unstable"
 163 {
 164    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 165    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 166    Sections "main contrib non-free";
 167    Architectures "%(arch)s";
 168    BinOverride "override.sid.$(SECTION)";
 169    ExtraOverride "override.sid.extra.$(SECTION)";
 170    SrcOverride "override.sid.$(SECTION).src";
 171 };
 172 """
 173     apt_trees["di"]["unstable"]="""
 174 tree "dists/unstable/main"
 175 {
 176    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 177    Sections "debian-installer";
 178    Architectures "%(arch)s";
 179    BinOverride "override.sid.main.$(SECTION)";
 180    SrcOverride "override.sid.main.src";
 181    BinCacheDB "packages-debian-installer-$(ARCH).db";
 182    Packages::Extensions ".udeb";
 183    %(contentsline)s
 184 };
 185
 186 tree "dists/unstable/non-free"
 187 {
 188    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 189    Sections "debian-installer";
 190    Architectures "%(arch)s";
 191    BinOverride "override.sid.main.$(SECTION)";
 192    SrcOverride "override.sid.main.src";
 193    BinCacheDB "packages-debian-installer-$(ARCH).db";
 194    Packages::Extensions ".udeb";
 195    %(contentsline)s
 196 };
 197 """
 198
 199     apt_trees["experimental"]="""
 200 tree "dists/experimental"
 201 {
 202    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 203    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 204    Sections "main contrib non-free";
 205    Architectures "%(arch)s";
 206    BinOverride "override.sid.$(SECTION)";
 207    SrcOverride "override.sid.$(SECTION).src";
 208 };
 209 """
 210     apt_trees["di"]["experimental"]="""
 211 tree "dists/experimental/main"
 212 {
 213    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 214    Sections "debian-installer";
 215    Architectures "%(arch)s";
 216    BinOverride "override.sid.main.$(SECTION)";
 217    SrcOverride "override.sid.main.src";
 218    BinCacheDB "packages-debian-installer-$(ARCH).db";
 219    Packages::Extensions ".udeb";
 220    %(contentsline)s
 221 };
 222
 223 tree "dists/experimental/non-free"
 224 {
 225    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 226    Sections "debian-installer";
 227    Architectures "%(arch)s";
 228    BinOverride "override.sid.main.$(SECTION)";
 229    SrcOverride "override.sid.main.src";
 230    BinCacheDB "packages-debian-installer-$(ARCH).db";
 231    Packages::Extensions ".udeb";
 232    %(contentsline)s
 233 };
 234 """
 235
 236     apt_trees["testing-proposed-updates"]="""
 237 tree "dists/testing-proposed-updates"
 238 {
 239    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 240    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 241    Sections "main contrib non-free";
 242    Architectures "%(arch)s";
 243    BinOverride "override.wheezy.$(SECTION)";
 244    ExtraOverride "override.wheezy.extra.$(SECTION)";
 245    SrcOverride "override.wheezy.$(SECTION).src";
 246    Contents " ";
 247 };
 248 """
 249     apt_trees["di"]["testing-proposed-updates"]="""
 250 tree "dists/testing-proposed-updates/main"
 251 {
 252    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 253    Sections "debian-installer";
 254    Architectures "%(arch)s";
 255    BinOverride "override.wheezy.main.$(SECTION)";
 256    SrcOverride "override.wheezy.main.src";
 257    BinCacheDB "packages-debian-installer-$(ARCH).db";
 258    Packages::Extensions ".udeb";
 259    Contents " ";
 260 };
 261 """
 262
 263     apt_trees["proposed-updates"]="""
 264 tree "dists/proposed-updates"
 265 {
 266    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 267    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 268    Sections "main contrib non-free";
 269    Architectures "%(arch)s";
 270    BinOverride "override.squeeze.$(SECTION)";
 271    ExtraOverride "override.squeeze.extra.$(SECTION)";
 272    SrcOverride "override.squeeze.$(SECTION).src";
 273    Contents " ";
 274 };
 275 """
 276     apt_trees["di"]["proposed-updates"]="""
 277 tree "dists/proposed-updates/main"
 278 {
 279    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 280    Sections "debian-installer";
 281    Architectures "%(arch)s";
 282    BinOverride "override.squeeze.main.$(SECTION)";
 283    SrcOverride "override.squeeze.main.src";
 284    BinCacheDB "packages-debian-installer-$(ARCH).db";
 285    Packages::Extensions ".udeb";
 286    Contents " ";
 287 };
 288 """
 289     apt_trees["oldstable-proposed-updates"]="""
 290 tree "dists/oldstable-proposed-updates"
 291 {
 292    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 293    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
 294    Sections "main contrib non-free";
 295    Architectures "%(arch)s";
 296    BinOverride "override.lenny.$(SECTION)";
 297    ExtraOverride "override.lenny.extra.$(SECTION)";
 298    SrcOverride "override.lenny.$(SECTION).src";
 299    Contents " ";
 300 };
 301 """
 302     apt_trees["di"]["oldstable-proposed-updates"]="""
 303 tree "dists/oldstable-proposed-updates/main"
 304 {
 305    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 306    Sections "debian-installer";
 307    Architectures "%(arch)s";
 308    BinOverride "override.lenny.main.$(SECTION)";
 309    SrcOverride "override.lenny.main.src";
 310    BinCacheDB "packages-debian-installer-$(ARCH).db";
 311    Packages::Extensions ".udeb";
 312    Contents " ";
 313 };
 314 """
 315
 316     cnf = Config()
 317     try:
 318         # Write apt.conf
 319         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 320         os.write(ac_fd, DAILY_APT_CONF)
 321         # here we want to generate the tree entries
 322         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 323         # this special casing needs to go away, but this whole thing may just want an
 324         # aptconfig class anyways
 325         if arch != 'source':
 326             if arch == 'hurd-i386' and suite == 'experimental':
 327                 pass
 328             elif apt_trees["di"].has_key(suite):
 329                 if arch == "amd64":
 330                     os.write(ac_fd, apt_trees["di"][suite] %
 331                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 332                 else:
 333                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 334         os.close(ac_fd)
 335
 336         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 337         # Run apt-ftparchive generate
 338         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 339         # it has errormessages we like to see
 340         os.environ['GZIP'] = '--rsyncable'
 341         os.chdir(tmppath)
 342         (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
 343         sn="a-f %s,%s: " % (suite, arch)
 344         print sn + output.replace('\n', '\n%s' % (sn))
 345         return result
 346
 347     # Clean up any left behind files
 348     finally:
 349         if ac_fd:
 350             try:
 351                 os.close(ac_fd)
 352             except OSError:
 353                 pass
 354
 355         if ac_name:
 356             try:
 357                 os.unlink(ac_name)
 358             except OSError:
 359                 pass
 360
 361 def sname(arch):
 362     return arch.arch_string
 363
 364 def get_result(arg):
 365     global results
 366     if arg:
 367         results.append(arg)
 368
 369 ########################################################################
 370 ########################################################################
 371
 372 def main ():
 373     global Options, Logger, results
 374
 375     cnf = Config()
 376
 377     for i in ["Help", "Suite", "Force"]:
 378         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 379             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 380
 381     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 382                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 383                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 384
 385     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 386     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 387
 388     if Options["Help"]:
 389         usage()
 390
 391     Logger = daklog.Logger(cnf, 'generate-packages-sources')
 392
 393     session = DBConn().session()
 394
 395     if Options["Suite"]:
 396         # Something here
 397         suites = []
 398         for s in suite_names:
 399             suite = get_suite(s.lower(), session)
 400             if suite:
 401                 suites.append(suite)
 402             else:
 403                 print "cannot find suite %s" % s
 404                 Logger.log(['cannot find suite %s' % s])
 405     else:
 406         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 407
 408     startdir = os.getcwd()
 409     os.chdir(cnf["Dir::TempPath"])
 410
 411     broken=[]
 412     # For each given suite, each architecture, run one apt-ftparchive
 413     for s in suites:
 414         results=[]
 415         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 416         pool = Pool()
 417         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
 418         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 419         for a in arch_list:
 420             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
 421
 422         # No more work will be added to our pool, close it and then wait for all to finish
 423         pool.close()
 424         pool.join()
 425
 426     if len(results) > 0:
 427         Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
 428         print "Trouble, something with a-f broke, resultcodes: %s" % (results)
 429         sys.exit(1)
 430
 431     os.chdir(startdir)
 432     # this script doesn't change the database
 433     session.close()
 434     Logger.close()
 435
 436 #######################################################################################
 437
 438 if __name__ == '__main__':
 439     main()