dak/generate_packages_sources.py

   1 #!/usr/bin/env python
   2
   3 """ Generate Packages/Sources files
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Mark Hymers <mhy@debian.org>
   8 @copyright: 2010  Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ################################################################################
  28
  29 import os
  30 import os.path
  31 import sys
  32 import apt_pkg
  33 from tempfile import mkstemp, mkdtemp
  34 import commands
  35 from multiprocessing import Pool, TimeoutError
  36
  37 from daklib import daklog
  38 from daklib.dbconn import *
  39 from daklib.config import Config
  40
  41 ################################################################################
  42
  43 Options = None                 #: Commandline arguments parsed into this
  44 Logger = None                  #: Our logging object
  45 results = []                   #: Results of the subprocesses
  46
  47 ################################################################################
  48
  49 def usage (exit_code=0):
  50     print """Usage: dak generate-packages-sources [OPTIONS]
  51 Generate the Packages/Sources files
  52
  53   -s, --suite=SUITE(s)       process this suite
  54                              Default: All suites not marked 'untouchable'
  55   -f, --force                Allow processing of untouchable suites
  56                              CAREFUL: Only to be used at point release time!
  57   -h, --help                 show this help and exit
  58
  59 SUITE can be a space seperated list, e.g.
  60    --suite=unstable testing
  61   """
  62
  63     sys.exit(exit_code)
  64
  65 ################################################################################
  66
  67 def generate_packages_sources(arch, suite, tmppath):
  68     """
  69     Generate Packages/Sources files with apt-ftparchive for the given suite/arch
  70
  71     @type suite: string
  72     @param suite: Suite name
  73
  74     @type arch: string
  75     @param arch: Architecture name
  76
  77     @type tmppath: string
  78     @param tmppath: The temporary path to work ing
  79     """
  80
  81     DAILY_APT_CONF="""
  82 Dir
  83 {
  84    ArchiveDir "/srv/ftp-master.debian.org/ftp/";
  85    OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
  86    CacheDir "/srv/ftp-master.debian.org/database/";
  87 };
  88
  89 Default
  90 {
  91    Packages::Compress "bzip2 gzip";
  92    Sources::Compress "bzip2 gzip";
  93    Contents::Compress "gzip";
  94    DeLinkLimit 0;
  95    MaxContentsChange 25000;
  96    FileMode 0664;
  97 }
  98
  99 TreeDefault
 100 {
 101    Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
 102 };
 103
 104 """
 105
 106     apt_trees={}
 107     apt_trees["di"]={}
 108
 109     apt_trees["stable"]="""
 110 tree "dists/stable"
 111 {
 112    FileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_binary-$(ARCH).list";
 113    SourceFileList "/srv/ftp-master.debian.org/database/dists/stable_$(SECTION)_source.list";
 114    Sections "main contrib non-free";
 115    Architectures "%(arch)s";
 116    BinOverride "override.squeeze.$(SECTION)";
 117    ExtraOverride "override.squeeze.extra.$(SECTION)";
 118    SrcOverride "override.squeeze.$(SECTION).src";
 119 };
 120 """
 121
 122     apt_trees["di"]["stable"]="""
 123 tree "dists/stable/main"
 124 {
 125    FileList "/srv/ftp-master.debian.org/database/dists/stable_main_$(SECTION)_binary-$(ARCH).list";
 126    Sections "debian-installer";
 127    Architectures "%(arch)s";
 128    BinOverride "override.squeeze.main.$(SECTION)";
 129    SrcOverride "override.squeeze.main.src";
 130    BinCacheDB "packages-debian-installer-$(ARCH).db";
 131    Packages::Extensions ".udeb";
 132    %(contentsline)s
 133 };
 134
 135 tree "dists/stable/non-free"
 136 {
 137    FileList "/srv/ftp-master.debian.org/database/dists/stable_non-free_$(SECTION)_binary-$(ARCH).list";
 138    Sections "debian-installer";
 139    Architectures "%(arch)s";
 140    BinOverride "override.squeeze.main.$(SECTION)";
 141    SrcOverride "override.squeeze.main.src";
 142    BinCacheDB "packages-debian-installer-$(ARCH).db";
 143    Packages::Extensions ".udeb";
 144    %(contentsline)s
 145 };
 146 """
 147
 148     apt_trees["squeeze-updates"]="""
 149 tree "dists/squeeze-updates"
 150 {
 151    FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
 152    SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
 153    Sections "main contrib non-free";
 154    Architectures "%(arch)s";
 155    BinOverride "override.squeeze.$(SECTION)";
 156    ExtraOverride "override.squeeze.extra.$(SECTION)";
 157    SrcOverride "override.squeeze.$(SECTION).src";
 158    Contents " ";
 159 };
 160 """
 161
 162     apt_trees["testing"]="""
 163 tree "dists/testing"
 164 {
 165    FakeDI "dists/unstable";
 166    FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
 167    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
 168    Sections "main contrib non-free";
 169    Architectures "%(arch)s";
 170    BinOverride "override.wheezy.$(SECTION)";
 171    ExtraOverride "override.wheezy.extra.$(SECTION)";
 172    SrcOverride "override.wheezy.$(SECTION).src";
 173 };
 174 """
 175
 176     apt_trees["di"]["testing"]="""
 177 tree "dists/testing/main"
 178 {
 179    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
 180    Sections "debian-installer";
 181    Architectures "%(arch)s";
 182    BinOverride "override.wheezy.main.$(SECTION)";
 183    SrcOverride "override.wheezy.main.src";
 184    BinCacheDB "packages-debian-installer-$(ARCH).db";
 185    Packages::Extensions ".udeb";
 186    %(contentsline)s
 187 };
 188
 189 tree "dists/testing/non-free"
 190 {
 191    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
 192    Sections "debian-installer";
 193    Architectures "%(arch)s";
 194    BinOverride "override.wheezy.main.$(SECTION)";
 195    SrcOverride "override.wheezy.main.src";
 196    BinCacheDB "packages-debian-installer-$(ARCH).db";
 197    Packages::Extensions ".udeb";
 198    %(contentsline)s
 199 };
 200 """
 201
 202     apt_trees["unstable"]="""
 203 tree "dists/unstable"
 204 {
 205    FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
 206    SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
 207    Sections "main contrib non-free";
 208    Architectures "%(arch)s";
 209    BinOverride "override.sid.$(SECTION)";
 210    ExtraOverride "override.sid.extra.$(SECTION)";
 211    SrcOverride "override.sid.$(SECTION).src";
 212 };
 213 """
 214     apt_trees["di"]["unstable"]="""
 215 tree "dists/unstable/main"
 216 {
 217    FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
 218    Sections "debian-installer";
 219    Architectures "%(arch)s";
 220    BinOverride "override.sid.main.$(SECTION)";
 221    SrcOverride "override.sid.main.src";
 222    BinCacheDB "packages-debian-installer-$(ARCH).db";
 223    Packages::Extensions ".udeb";
 224    %(contentsline)s
 225 };
 226
 227 tree "dists/unstable/non-free"
 228 {
 229    FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
 230    Sections "debian-installer";
 231    Architectures "%(arch)s";
 232    BinOverride "override.sid.main.$(SECTION)";
 233    SrcOverride "override.sid.main.src";
 234    BinCacheDB "packages-debian-installer-$(ARCH).db";
 235    Packages::Extensions ".udeb";
 236    %(contentsline)s
 237 };
 238 """
 239
 240     apt_trees["experimental"]="""
 241 tree "dists/experimental"
 242 {
 243    FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
 244    SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
 245    Sections "main contrib non-free";
 246    Architectures "%(arch)s";
 247    BinOverride "override.sid.$(SECTION)";
 248    SrcOverride "override.sid.$(SECTION).src";
 249 };
 250 """
 251     apt_trees["di"]["experimental"]="""
 252 tree "dists/experimental/main"
 253 {
 254    FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
 255    Sections "debian-installer";
 256    Architectures "%(arch)s";
 257    BinOverride "override.sid.main.$(SECTION)";
 258    SrcOverride "override.sid.main.src";
 259    BinCacheDB "packages-debian-installer-$(ARCH).db";
 260    Packages::Extensions ".udeb";
 261    %(contentsline)s
 262 };
 263
 264 tree "dists/experimental/non-free"
 265 {
 266    FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
 267    Sections "debian-installer";
 268    Architectures "%(arch)s";
 269    BinOverride "override.sid.main.$(SECTION)";
 270    SrcOverride "override.sid.main.src";
 271    BinCacheDB "packages-debian-installer-$(ARCH).db";
 272    Packages::Extensions ".udeb";
 273    %(contentsline)s
 274 };
 275 """
 276
 277     apt_trees["testing-proposed-updates"]="""
 278 tree "dists/testing-proposed-updates"
 279 {
 280    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
 281    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
 282    Sections "main contrib non-free";
 283    Architectures "%(arch)s";
 284    BinOverride "override.wheezy.$(SECTION)";
 285    ExtraOverride "override.wheezy.extra.$(SECTION)";
 286    SrcOverride "override.wheezy.$(SECTION).src";
 287    Contents " ";
 288 };
 289 """
 290     apt_trees["di"]["testing-proposed-updates"]="""
 291 tree "dists/testing-proposed-updates/main"
 292 {
 293    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 294    Sections "debian-installer";
 295    Architectures "%(arch)s";
 296    BinOverride "override.wheezy.main.$(SECTION)";
 297    SrcOverride "override.wheezy.main.src";
 298    BinCacheDB "packages-debian-installer-$(ARCH).db";
 299    Packages::Extensions ".udeb";
 300    Contents " ";
 301 };
 302 """
 303
 304     apt_trees["proposed-updates"]="""
 305 tree "dists/proposed-updates"
 306 {
 307    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
 308    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
 309    Sections "main contrib non-free";
 310    Architectures "%(arch)s";
 311    BinOverride "override.squeeze.$(SECTION)";
 312    ExtraOverride "override.squeeze.extra.$(SECTION)";
 313    SrcOverride "override.squeeze.$(SECTION).src";
 314    Contents " ";
 315 };
 316 """
 317     apt_trees["di"]["proposed-updates"]="""
 318 tree "dists/proposed-updates/main"
 319 {
 320    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
 321    Sections "debian-installer";
 322    Architectures "%(arch)s";
 323    BinOverride "override.squeeze.main.$(SECTION)";
 324    SrcOverride "override.squeeze.main.src";
 325    BinCacheDB "packages-debian-installer-$(ARCH).db";
 326    Packages::Extensions ".udeb";
 327    Contents " ";
 328 };
 329 """
 330     cnf = Config()
 331     try:
 332         # Write apt.conf
 333         (ac_fd, ac_name) = mkstemp(dir=tmppath, suffix=suite, prefix=arch)
 334         os.write(ac_fd, DAILY_APT_CONF)
 335         # here we want to generate the tree entries
 336         os.write(ac_fd, apt_trees[suite] % {'arch': arch})
 337         # this special casing needs to go away, but this whole thing may just want an
 338         # aptconfig class anyways
 339         if arch != 'source':
 340             if arch == 'hurd-i386' and suite == 'experimental':
 341                 pass
 342             elif apt_trees["di"].has_key(suite):
 343                 if arch == "amd64":
 344                     os.write(ac_fd, apt_trees["di"][suite] %
 345                              {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
 346                 else:
 347                     os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
 348         os.close(ac_fd)
 349
 350         print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
 351         # Run apt-ftparchive generate
 352         # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
 353         # it has errormessages we like to see
 354         os.environ['GZIP'] = '--rsyncable'
 355         os.chdir(tmppath)
 356         (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
 357         sn="a-f %s,%s: " % (suite, arch)
 358         print sn + output.replace('\n', '\n%s' % (sn))
 359         return result
 360
 361     # Clean up any left behind files
 362     finally:
 363         if ac_fd:
 364             try:
 365                 os.close(ac_fd)
 366             except OSError:
 367                 pass
 368
 369         if ac_name:
 370             try:
 371                 os.unlink(ac_name)
 372             except OSError:
 373                 pass
 374
 375 def sname(arch):
 376     return arch.arch_string
 377
 378 def get_result(arg):
 379     global results
 380     if arg:
 381         results.append(arg)
 382
 383 ########################################################################
 384 ########################################################################
 385
 386 def main ():
 387     global Options, Logger, results
 388
 389     cnf = Config()
 390
 391     for i in ["Help", "Suite", "Force"]:
 392         if not cnf.has_key("Generate-Packages-Sources::Options::%s" % (i)):
 393             cnf["Generate-Packages-Sources::Options::%s" % (i)] = ""
 394
 395     Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
 396                  ('s',"suite","Generate-Packages-Sources::Options::Suite"),
 397                  ('f',"force","Generate-Packages-Sources::Options::Force")]
 398
 399     suite_names = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 400     Options = cnf.SubTree("Generate-Packages-Sources::Options")
 401
 402     if Options["Help"]:
 403         usage()
 404
 405     Logger = daklog.Logger('generate-packages-sources')
 406
 407     session = DBConn().session()
 408
 409     if Options["Suite"]:
 410         # Something here
 411         suites = []
 412         for s in suite_names:
 413             suite = get_suite(s.lower(), session)
 414             if suite:
 415                 suites.append(suite)
 416             else:
 417                 print "cannot find suite %s" % s
 418                 Logger.log(['cannot find suite %s' % s])
 419     else:
 420         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 421
 422     startdir = os.getcwd()
 423     os.chdir(cnf["Dir::TempPath"])
 424
 425     broken=[]
 426     # For each given suite, each architecture, run one apt-ftparchive
 427     for s in suites:
 428         results=[]
 429         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
 430         pool = Pool()
 431         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
 432         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
 433         for a in arch_list:
 434             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
 435
 436         # No more work will be added to our pool, close it and then wait for all to finish
 437         pool.close()
 438         pool.join()
 439
 440     if len(results) > 0:
 441         Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
 442         print "Trouble, something with a-f broke, resultcodes: %s" % (results)
 443         sys.exit(1)
 444
 445     os.chdir(startdir)
 446     # this script doesn't change the database
 447     session.close()
 448     Logger.close()
 449
 450 #######################################################################################
 451
 452 if __name__ == '__main__':
 453     main()