dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ################################################################################
  28
  29 import os
  30 import os.path
  31 import sys
  32 import apt_pkg
  33 from tempfile import mkstemp, mkdtemp
  34 import commands
  35 from multiprocessing import Pool, TimeoutError
  36
  37 from daklib import daklog
  38 from daklib.dbconn import *
  39 from daklib.config import Config
  40
  41 ################################################################################
  42
  43 Options = None                 #: Commandline arguments parsed into this
  44 Logger = None                  #: Our logging object
  45 results = []                   #: Results of the subprocesses
  46
  47 ################################################################################
  48
  49 def usage (exit_code=0):
  50     print """Usage: dak generate-packages-sources [OPTIONS]
  51 Generate the Packages/Sources files
  52
  53   -s, --suite=SUITE(s)       process this suite
  54                              Default: All suites not marked 'untouchable'
  55   -f, --force                Allow processing of untouchable suites
  56                              CAREFUL: Only to be used at point release time!
  57   -h, --help                 show this help and exit
  58
  59 SUITE can be a space seperated list, e.g.
  60    --suite=unstable testing
  61   """
  62
  63     sys.exit(exit_code)
  64
  65 ################################################################################
  66
  67 def generate_packages_sources(arch, suite, tmppath):
  68     """
  69     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  70
  71     @type suite: string
  72     @param suite: Suite name
  73
  74     @type arch: string
  75     @param arch: Architecture name
  76
  77     @type tmppath: string
  78     @param tmppath: The temporary path to work ing
  79     """
  80
  81     DAILY_APT_CONF="""
  82 Dir
  83 {
  84    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  85    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  86    CacheDir "/srv/ftp-master.debian.org/database/";
  87 };
  88
  89 Default
  90 {
  91    Packages::Compress "bzip2 gzip";
  92    Sources::Compress "bzip2 gzip";
  93    Contents::Compress "gzip";
  94    DeLinkLimit 0;
  95    MaxContentsChange 25000;
  96    FileMode 0664;
  97 }
  98
  99 TreeDefault
 100 {
 101    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 102 };
 103
 104 """
 105
 106     apt_trees={}
 107     apt_trees["di"]={}
 108
 109     apt_trees["oldstable"]="""
 110 tree "dists/oldstable"
 111 {
 112    FileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_binary-$(ARCH).list";
 113    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable_$(SECTION)_source.list";
 114    Sections "main contrib non-free";
 115    Architectures "%(arch)s";
 116    BinOverride "override.lenny.$(SECTION)";
 117    ExtraOverride "override.lenny.extra.$(SECTION)";
 118    SrcOverride "override.lenny.$(SECTION).src";
 119 };
 120 """
 121
 122     apt_trees["di"]["oldstable"]="""
 123 tree "dists/oldstable/main"
 124 {
 125    FileList "/srv/ftp-master.debian.org/database/dists/oldstable_main_$(SECTION)_binary-$(ARCH).list";
 126    Sections "debian-installer";
 127    Architectures "%(arch)s";
 128    BinOverride "override.lenny.main.$(SECTION)";
 129    SrcOverride "override.lenny.main.src";
 130    BinCacheDB "packages-debian-installer-$(ARCH).db";
 131    Packages::Extensions ".udeb";
 132    %(contentsline)s
 133 };
 134
 135 tree "dists/oldstable/non-free"
 136 {
 137    FileList "/srv/ftp-master.debian.org/database/dists/oldstable_non-free_$(SECTION)_binary-$(ARCH).list";
 138    Sections "debian-installer";
 139    Architectures "%(arch)s";
 140    BinOverride "override.lenny.main.$(SECTION)";
 141    SrcOverride "override.lenny.main.src";
 142    BinCacheDB "packages-debian-installer-$(ARCH).db";
 143    Packages::Extensions ".udeb";
 144    %(contentsline)s
 145 };
 146 """
 147
 148     apt_trees["stable"]="""
 149 tree "dists/stable"
 150 {
 151    FileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_binary-$(ARCH).list";
 152    SourceFileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_source.list";
 153    Sections "main contrib non-free";
 154    Architectures "%(arch)s";
 155    BinOverride "override.squeeze.$(SECTION)";
 156    ExtraOverride "override.squeeze.extra.$(SECTION)";
 157    SrcOverride "override.squeeze.$(SECTION).src";
 158 };
 159 """
 160
 161     apt_trees["di"]["stable"]="""
 162 tree "dists/stable/main"
 163 {
 164    FileList "/srv/ftp-master.debian.org/database/dists/stable_main_$(SECTION)_binary-$(ARCH).list";
 165    Sections "debian-installer";
 166    Architectures "%(arch)s";
 167    BinOverride "override.squeeze.main.$(SECTION)";
 168    SrcOverride "override.squeeze.main.src";
 169    BinCacheDB "packages-debian-installer-$(ARCH).db";
 170    Packages::Extensions ".udeb";
 171    %(contentsline)s
 172 };
 173
 174 tree "dists/stable/non-free"
 175 {
 176    FileList "/srv/ftp-master.debian.org/database/dists/stable_non-free_$(SECTION)_binary-$(ARCH).list";
 177    Sections "debian-installer";
 178    Architectures "%(arch)s";
 179    BinOverride "override.squeeze.main.$(SECTION)";
 180    SrcOverride "override.squeeze.main.src";
 181    BinCacheDB "packages-debian-installer-$(ARCH).db";
 182    Packages::Extensions ".udeb";
 183    %(contentsline)s
 184 };
 185 """
 186
 187     apt_trees["squeeze-updates"]="""
 188 tree "dists/squeeze-updates"
 189 {
 190    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
 191    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
 192    Sections "main contrib non-free";
 193    Architectures "%(arch)s";
 194    BinOverride "override.squeeze.$(SECTION)";
 195    ExtraOverride "override.squeeze.extra.$(SECTION)";
 196    SrcOverride "override.squeeze.$(SECTION).src";
 197    Contents " ";
 198 };
 199 """
 200
 201     apt_trees["testing"]="""
 202 tree "dists/testing"
 203 {
 204    FakeDI "dists/unstable";
 205    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 206    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 207    Sections "main contrib non-free";
 208    Architectures "%(arch)s";
 209    BinOverride "override.wheezy.$(SECTION)";
 210    ExtraOverride "override.wheezy.extra.$(SECTION)";
 211    SrcOverride "override.wheezy.$(SECTION).src";
 212 };
 213 """
 214
 215     apt_trees["di"]["testing"]="""
 216 tree "dists/testing/main"
 217 {
 218    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 219    Sections "debian-installer";
 220    Architectures "%(arch)s";
 221    BinOverride "override.wheezy.main.$(SECTION)";
 222    SrcOverride "override.wheezy.main.src";
 223    BinCacheDB "packages-debian-installer-$(ARCH).db";
 224    Packages::Extensions ".udeb";
 225    %(contentsline)s
 226 };
 227
 228 tree "dists/testing/non-free"
 229 {
 230    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 231    Sections "debian-installer";
 232    Architectures "%(arch)s";
 233    BinOverride "override.wheezy.main.$(SECTION)";
 234    SrcOverride "override.wheezy.main.src";
 235    BinCacheDB "packages-debian-installer-$(ARCH).db";
 236    Packages::Extensions ".udeb";
 237    %(contentsline)s
 238 };
 239 """
 240
 241     apt_trees["unstable"]="""
 242 tree "dists/unstable"
 243 {
 244    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 245    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 246    Sections "main contrib non-free";
 247    Architectures "%(arch)s";
 248    BinOverride "override.sid.$(SECTION)";
 249    ExtraOverride "override.sid.extra.$(SECTION)";
 250    SrcOverride "override.sid.$(SECTION).src";
 251 };
 252 """
 253     apt_trees["di"]["unstable"]="""
 254 tree "dists/unstable/main"
 255 {
 256    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 257    Sections "debian-installer";
 258    Architectures "%(arch)s";
 259    BinOverride "override.sid.main.$(SECTION)";
 260    SrcOverride "override.sid.main.src";
 261    BinCacheDB "packages-debian-installer-$(ARCH).db";
 262    Packages::Extensions ".udeb";
 263    %(contentsline)s
 264 };
 265
 266 tree "dists/unstable/non-free"
 267 {
 268    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 269    Sections "debian-installer";
 270    Architectures "%(arch)s";
 271    BinOverride "override.sid.main.$(SECTION)";
 272    SrcOverride "override.sid.main.src";
 273    BinCacheDB "packages-debian-installer-$(ARCH).db";
 274    Packages::Extensions ".udeb";
 275    %(contentsline)s
 276 };
 277 """
 278
 279     apt_trees["experimental"]="""
 280 tree "dists/experimental"
 281 {
 282    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 283    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 284    Sections "main contrib non-free";
 285    Architectures "%(arch)s";
 286    BinOverride "override.sid.$(SECTION)";
 287    SrcOverride "override.sid.$(SECTION).src";
 288 };
 289 """
 290     apt_trees["di"]["experimental"]="""
 291 tree "dists/experimental/main"
 292 {
 293    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 294    Sections "debian-installer";
 295    Architectures "%(arch)s";
 296    BinOverride "override.sid.main.$(SECTION)";
 297    SrcOverride "override.sid.main.src";
 298    BinCacheDB "packages-debian-installer-$(ARCH).db";
 299    Packages::Extensions ".udeb";
 300    %(contentsline)s
 301 };
 302
 303 tree "dists/experimental/non-free"
 304 {
 305    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 306    Sections "debian-installer";
 307    Architectures "%(arch)s";
 308    BinOverride "override.sid.main.$(SECTION)";
 309    SrcOverride "override.sid.main.src";
 310    BinCacheDB "packages-debian-installer-$(ARCH).db";
 311    Packages::Extensions ".udeb";
 312    %(contentsline)s
 313 };
 314 """
 315
 316     apt_trees["testing-proposed-updates"]="""
 317 tree "dists/testing-proposed-updates"
 318 {
 319    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 320    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 321    Sections "main contrib non-free";
 322    Architectures "%(arch)s";
 323    BinOverride "override.wheezy.$(SECTION)";
 324    ExtraOverride "override.wheezy.extra.$(SECTION)";
 325    SrcOverride "override.wheezy.$(SECTION).src";
 326    Contents " ";
 327 };
 328 """
 329     apt_trees["di"]["testing-proposed-updates"]="""
 330 tree "dists/testing-proposed-updates/main"
 331 {
 332    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 333    Sections "debian-installer";
 334    Architectures "%(arch)s";
 335    BinOverride "override.wheezy.main.$(SECTION)";
 336    SrcOverride "override.wheezy.main.src";
 337    BinCacheDB "packages-debian-installer-$(ARCH).db";
 338    Packages::Extensions ".udeb";
 339    Contents " ";
 340 };
 341 """
 342
 343     apt_trees["proposed-updates"]="""
 344 tree "dists/proposed-updates"
 345 {
 346    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 347    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 348    Sections "main contrib non-free";
 349    Architectures "%(arch)s";
 350    BinOverride "override.squeeze.$(SECTION)";
 351    ExtraOverride "override.squeeze.extra.$(SECTION)";
 352    SrcOverride "override.squeeze.$(SECTION).src";
 353    Contents " ";
 354 };
 355 """
 356     apt_trees["di"]["proposed-updates"]="""
 357 tree "dists/proposed-updates/main"
 358 {
 359    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 360    Sections "debian-installer";
 361    Architectures "%(arch)s";
 362    BinOverride "override.squeeze.main.$(SECTION)";
 363    SrcOverride "override.squeeze.main.src";
 364    BinCacheDB "packages-debian-installer-$(ARCH).db";
 365    Packages::Extensions ".udeb";
 366    Contents " ";
 367 };
 368 """
 369     apt_trees["oldstable-proposed-updates"]="""
 370 tree "dists/oldstable-proposed-updates"
 371 {
 372    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 373    SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
 374    Sections "main contrib non-free";
 375    Architectures "%(arch)s";
 376    BinOverride "override.lenny.$(SECTION)";
 377    ExtraOverride "override.lenny.extra.$(SECTION)";
 378    SrcOverride "override.lenny.$(SECTION).src";
 379    Contents " ";
 380 };
 381 """
 382     apt_trees["di"]["oldstable-proposed-updates"]="""
 383 tree "dists/oldstable-proposed-updates/main"
 384 {
 385    FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 386    Sections "debian-installer";
 387    Architectures "%(arch)s";
 388    BinOverride "override.lenny.main.$(SECTION)";
 389    SrcOverride "override.lenny.main.src";
 390    BinCacheDB "packages-debian-installer-$(ARCH).db";
 391    Packages::Extensions ".udeb";
 392    Contents " ";
 393 };
 394 """
 395
 396     cnf = Config()
 397     try:
 398         # Write apt.conf
 399         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 400         os.write(ac_fd, DAILY_APT_CONF)
 401         # here we want to generate the tree entries
 402         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 403         # this special casing needs to go away, but this whole thing may just want an
 404         # aptconfig class anyways
 405         if arch != 'source':
 406             if arch == 'hurd-i386' and suite == 'experimental':
 407                 pass
 408             elif apt_trees["di"].has_key(suite):
 409                 if arch == "amd64":
 410                     os.write(ac_fd, apt_trees["di"][suite] %
 411                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 412                 else:
 413                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 414         os.close(ac_fd)
 415
 416         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 417         # Run apt-ftparchive generate
 418         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 419         # it has errormessages we like to see
 420         os.environ['GZIP'] = '--rsyncable'
 421         os.chdir(tmppath)
 422         (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
 423         sn="a-f %s,%s: " % (suite, arch)
 424         print sn + output.replace('\n', '\n%s' % (sn))
 425         return result
 426
 427     # Clean up any left behind files
 428     finally:
 429         if ac_fd:
 430             try:
 431                 os.close(ac_fd)
 432             except OSError:
 433                 pass
 434
 435         if ac_name:
 436             try:
 437                 os.unlink(ac_name)
 438             except OSError:
 439                 pass
 440
 441 def sname(arch):
 442     return arch.arch_string
 443
 444 def get_result(arg):
 445     global results
 446     if arg:
 447         results.append(arg)
 448
 449 ########################################################################
 450 ########################################################################
 451
 452 def main ():
 453     global Options, Logger, results
 454
 455     cnf = Config()
 456
 457     for i in ["Help", "Suite", "Force"]:
 458         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 459             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 460
 461     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 462                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 463                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 464
 465     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 466     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 467
 468     if Options["Help"]:
 469         usage()
 470
 471     Logger = daklog.Logger(cnf, 'generate-packages-sources')
 472
 473     session = DBConn().session()
 474
 475     if Options["Suite"]:
 476         # Something here
 477         suites = []
 478         for s in suite_names:
 479             suite = get_suite(s.lower(), session)
 480             if suite:
 481                 suites.append(suite)
 482             else:
 483                 print "cannot find suite %s" % s
 484                 Logger.log(['cannot find suite %s' % s])
 485     else:
 486         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 487
 488     startdir = os.getcwd()
 489     os.chdir(cnf["Dir::TempPath"])
 490
 491     broken=[]
 492     # For each given suite, each architecture, run one apt-ftparchive
 493     for s in suites:
 494         results=[]
 495         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 496         pool = Pool()
 497         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
 498         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 499         for a in arch_list:
 500             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
 501
 502         # No more work will be added to our pool, close it and then wait for all to finish
 503         pool.close()
 504         pool.join()
 505
 506     if len(results) > 0:
 507         Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
 508         print "Trouble, something with a-f broke, resultcodes: %s" % (results)
 509         sys.exit(1)
 510
 511     os.chdir(startdir)
 512     # this script doesn't change the database
 513     session.close()
 514     Logger.close()
 515
 516 #######################################################################################
 517
 518 if __name__ == '__main__':
 519     main()