dak/process_upload.py

   1 #!/usr/bin/env python
   2
   3 """
   4 Checks Debian packages from Incoming
   5 @contact: Debian FTP Master <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
   8 @copyright: 2009  Mark Hymers <mhy@debian.org>
   9 @copyright: 2009  Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 # based on process-unchecked and process-accepted
  28
  29 ## pu|pa: locking (daily.lock)
  30 ## pu|pa: parse arguments -> list of changes files
  31 ## pa: initialize urgency log
  32 ## pu|pa: sort changes list
  33
  34 ## foreach changes:
  35 ###  pa: load dak file
  36 ##   pu: copy CHG to tempdir
  37 ##   pu: check CHG signature
  38 ##   pu: parse changes file
  39 ##   pu: checks:
  40 ##     pu: check distribution (mappings, rejects)
  41 ##     pu: copy FILES to tempdir
  42 ##     pu: check whether CHG already exists in CopyChanges
  43 ##     pu: check whether FILES already exist in one of the policy queues
  44 ##     for deb in FILES:
  45 ##       pu: extract control information
  46 ##       pu: various checks on control information
  47 ##       pu|pa: search for source (in CHG, projectb, policy queues)
  48 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  49 ##       pu|pa: check whether deb already exists in the pool
  50 ##     for src in FILES:
  51 ##       pu: various checks on filenames and CHG consistency
  52 ##       pu: if isdsc: check signature
  53 ##     for file in FILES:
  54 ##       pu: various checks
  55 ##       pu: NEW?
  56 ##       //pu: check whether file already exists in the pool
  57 ##       pu: store what "Component" the package is currently in
  58 ##     pu: check whether we found everything we were looking for in CHG
  59 ##     pu: check the DSC:
  60 ##       pu: check whether we need and have ONE DSC
  61 ##       pu: parse the DSC
  62 ##       pu: various checks //maybe drop some of the in favor of lintian
  63 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  64 ##       pu: check whether DSC_FILES is consistent with "Format"
  65 ##       for src in DSC_FILES:
  66 ##         pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
  67 ##     pu: create new tempdir
  68 ##     pu: create symlink mirror of source
  69 ##     pu: unpack source
  70 ##     pu: extract changelog information for BTS
  71 ##     //pu: create missing .orig symlink
  72 ##     pu: check with lintian
  73 ##     for file in FILES:
  74 ##       pu: check checksums and sizes
  75 ##     for file in DSC_FILES:
  76 ##       pu: check checksums and sizes
  77 ##     pu: CHG: check urgency
  78 ##     for deb in FILES:
  79 ##       pu: extract contents list and check for dubious timestamps
  80 ##     pu: check that the uploader is actually allowed to upload the package
  81 ###  pa: install:
  82 ###    if stable_install:
  83 ###      pa: remove from p-u
  84 ###      pa: add to stable
  85 ###      pa: move CHG to morgue
  86 ###      pa: append data to ChangeLog
  87 ###      pa: send mail
  88 ###      pa: remove .dak file
  89 ###    else:
  90 ###      pa: add dsc to db:
  91 ###        for file in DSC_FILES:
  92 ###          pa: add file to file
  93 ###          pa: add file to dsc_files
  94 ###        pa: create source entry
  95 ###        pa: update source associations
  96 ###        pa: update src_uploaders
  97 ###      for deb in FILES:
  98 ###        pa: add deb to db:
  99 ###          pa: add file to file
 100 ###          pa: find source entry
 101 ###          pa: create binaries entry
 102 ###          pa: update binary associations
 103 ###      pa: .orig component move
 104 ###      pa: move files to pool
 105 ###      pa: save CHG
 106 ###      pa: move CHG to done/
 107 ###      pa: change entry in queue_build
 108 ##   pu: use dispatch table to choose target queue:
 109 ##     if NEW:
 110 ##       pu: write .dak file
 111 ##       pu: move to NEW
 112 ##       pu: send mail
 113 ##     elsif AUTOBYHAND:
 114 ##       pu: run autobyhand script
 115 ##       pu: if stuff left, do byhand or accept
 116 ##     elsif targetqueue in (oldstable, stable, embargo, unembargo):
 117 ##       pu: write .dak file
 118 ##       pu: check overrides
 119 ##       pu: move to queue
 120 ##       pu: send mail
 121 ##     else:
 122 ##       pu: write .dak file
 123 ##       pu: move to ACCEPTED
 124 ##       pu: send mails
 125 ##       pu: create files for BTS
 126 ##       pu: create entry in queue_build
 127 ##       pu: check overrides
 128
 129 # Integrity checks
 130 ## GPG
 131 ## Parsing changes (check for duplicates)
 132 ## Parse dsc
 133 ## file list checks
 134
 135 # New check layout (TODO: Implement)
 136 ## Permission checks
 137 ### suite mappings
 138 ### ACLs
 139 ### version checks (suite)
 140 ### override checks
 141
 142 ## Source checks
 143 ### copy orig
 144 ### unpack
 145 ### BTS changelog
 146 ### src contents
 147 ### lintian
 148 ### urgency log
 149
 150 ## Binary checks
 151 ### timestamps
 152 ### control checks
 153 ### src relation check
 154 ### contents
 155
 156 ## Database insertion (? copy from stuff)
 157 ### BYHAND / NEW / Policy queues
 158 ### Pool
 159
 160 ## Queue builds
 161
 162 import datetime
 163 import errno
 164 from errno import EACCES, EAGAIN
 165 import fcntl
 166 import os
 167 import sys
 168 import traceback
 169 import apt_pkg
 170 import time
 171 from sqlalchemy.orm.exc import NoResultFound
 172
 173 from daklib import daklog
 174 from daklib.dbconn import *
 175 from daklib.urgencylog import UrgencyLog
 176 from daklib.summarystats import SummaryStats
 177 from daklib.config import Config
 178 import daklib.utils as utils
 179 from daklib.regexes import *
 180
 181 import daklib.announce
 182 import daklib.archive
 183 import daklib.checks
 184 import daklib.upload
 185
 186 ###############################################################################
 187
 188 Options = None
 189 Logger = None
 190
 191 ###############################################################################
 192
 193 def usage (exit_code=0):
 194     print """Usage: dak process-upload [OPTION]... [CHANGES]...
 195   -a, --automatic           automatic run
 196   -d, --directory <DIR>     process uploads in <DIR>
 197   -h, --help                show this help and exit.
 198   -n, --no-action           don't do anything
 199   -p, --no-lock             don't check lockfile !! for cron.daily only !!
 200   -s, --no-mail             don't send any mail
 201   -V, --version             display the version number and exit"""
 202     sys.exit(exit_code)
 203
 204 ###############################################################################
 205
 206 def try_or_reject(function):
 207     """Try to call function or reject the upload if that fails
 208     """
 209     def wrapper(directory, upload, *args, **kwargs):
 210         reason = 'No exception caught. This should not happen.'
 211
 212         try:
 213             return function(directory, upload, *args, **kwargs)
 214         except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
 215             reason = unicode(e)
 216         except Exception as e:
 217             reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
 218
 219         try:
 220             upload.rollback()
 221             return real_reject(directory, upload, reason=reason)
 222         except Exception as e:
 223             reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
 224             upload.rollback()
 225             return real_reject(directory, upload, reason=reason, notify=False)
 226
 227         raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
 228
 229     return wrapper
 230
 231 def get_processed_upload(upload):
 232     changes = upload.changes
 233     control = upload.changes.changes
 234
 235     pu = daklib.announce.ProcessedUpload()
 236
 237     pu.maintainer = control.get('Maintainer')
 238     pu.changed_by = control.get('Changed-By')
 239     pu.fingerprint = changes.primary_fingerprint
 240
 241     pu.suites = upload.final_suites or []
 242     pu.from_policy_suites = []
 243
 244     pu.changes = open(upload.changes.path, 'r').read()
 245     pu.changes_filename = upload.changes.filename
 246     pu.sourceful = upload.changes.source is not None
 247     pu.source = control.get('Source')
 248     pu.version = control.get('Version')
 249     pu.architecture = control.get('Architecture')
 250     pu.bugs = changes.closed_bugs
 251
 252     pu.program = "process-upload"
 253
 254     pu.warnings = upload.warnings
 255
 256     return pu
 257
 258 @try_or_reject
 259 def accept(directory, upload):
 260     cnf = Config()
 261
 262     Logger.log(['ACCEPT', upload.changes.filename])
 263
 264     upload.install()
 265
 266     accepted_to_real_suite = False
 267     for suite in upload.final_suites:
 268         accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
 269
 270     sourceful_upload = 'source' in upload.changes.architectures
 271
 272     control = upload.changes.changes
 273     if sourceful_upload and not Options['No-Action']:
 274         urgency = control.get('Urgency')
 275         if urgency not in cnf.value_list('Urgency::Valid'):
 276             urgency = cnf['Urgency::Default']
 277         UrgencyLog().log(control['Source'], control['Version'], urgency)
 278
 279     pu = get_processed_upload(upload)
 280     daklib.announce.announce_accept(pu)
 281
 282     # Move .changes to done, but only for uploads that were accepted to a
 283     # real suite.  process-policy will handle this for uploads to queues.
 284     if accepted_to_real_suite:
 285         src = os.path.join(upload.directory, upload.changes.filename)
 286
 287         now = datetime.datetime.now()
 288         donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
 289         dst = os.path.join(donedir, upload.changes.filename)
 290         dst = utils.find_next_free(dst)
 291
 292         upload.transaction.fs.copy(src, dst, mode=0o644)
 293
 294     SummaryStats().accept_count += 1
 295     SummaryStats().accept_bytes += upload.changes.bytes
 296
 297 @try_or_reject
 298 def accept_to_new(directory, upload):
 299     cnf = Config()
 300
 301     Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
 302
 303     upload.install_to_new()
 304     # TODO: tag bugs pending
 305
 306     pu = get_processed_upload(upload)
 307     daklib.announce.announce_new(pu)
 308
 309     SummaryStats().accept_count += 1
 310     SummaryStats().accept_bytes += upload.changes.bytes
 311
 312 @try_or_reject
 313 def reject(directory, upload, reason=None, notify=True):
 314     real_reject(directory, upload, reason, notify)
 315
 316 def real_reject(directory, upload, reason=None, notify=True):
 317     # XXX: rejection itself should go to daklib.archive.ArchiveUpload
 318     cnf = Config()
 319
 320     Logger.log(['REJECT', upload.changes.filename])
 321
 322     fs = upload.transaction.fs
 323     rejectdir = cnf['Dir::Reject']
 324
 325     files = [ f.filename for f in upload.changes.files.itervalues() ]
 326     files.append(upload.changes.filename)
 327
 328     for fn in files:
 329         src = os.path.join(upload.directory, fn)
 330         dst = utils.find_next_free(os.path.join(rejectdir, fn))
 331         if not os.path.exists(src):
 332             continue
 333         fs.copy(src, dst)
 334
 335     if upload.reject_reasons is not None:
 336         if reason is None:
 337             reason = ''
 338         reason = reason + '\n' + '\n'.join(upload.reject_reasons)
 339
 340     if reason is None:
 341         reason = '(Unknown reason. Please check logs.)'
 342
 343     dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
 344     fh = fs.create(dst)
 345     fh.write(reason)
 346     fh.close()
 347
 348     if notify:
 349         pu = get_processed_upload(upload)
 350         daklib.announce.announce_reject(pu, reason)
 351
 352     SummaryStats().reject_count += 1
 353
 354 ###############################################################################
 355
 356 def action(directory, upload):
 357     changes = upload.changes
 358     processed = True
 359
 360     global Logger
 361
 362     cnf = Config()
 363
 364     okay = upload.check()
 365
 366     summary = changes.changes.get('Changes', '')
 367
 368     package_info = []
 369     if okay:
 370         if changes.source is not None:
 371             package_info.append("source:{0}".format(changes.source.dsc['Source']))
 372         for binary in changes.binaries:
 373             package_info.append("binary:{0}".format(binary.control['Package']))
 374
 375     (prompt, answer) = ("", "XXX")
 376     if Options["No-Action"] or Options["Automatic"]:
 377         answer = 'S'
 378
 379     queuekey = ''
 380
 381     print summary
 382     print
 383     print "\n".join(package_info)
 384     print
 385
 386     if len(upload.reject_reasons) > 0:
 387         print "Reason:"
 388         print "\n".join(upload.reject_reasons)
 389         print
 390
 391         path = os.path.join(directory, changes.filename)
 392         created = os.stat(path).st_mtime
 393         now = time.time()
 394         too_new = (now - created < int(cnf['Dinstall::SkipTime']))
 395
 396         if too_new:
 397             print "SKIP (too new)"
 398             prompt = "[S]kip, Quit ?"
 399         else:
 400             prompt = "[R]eject, Skip, Quit ?"
 401             if Options["Automatic"]:
 402                 answer = 'R'
 403     elif upload.new:
 404         prompt = "[N]ew, Skip, Quit ?"
 405         if Options['Automatic']:
 406             answer = 'N'
 407     else:
 408         prompt = "[A]ccept, Skip, Quit ?"
 409         if Options['Automatic']:
 410             answer = 'A'
 411
 412     while prompt.find(answer) == -1:
 413         answer = utils.our_raw_input(prompt)
 414         m = re_default_answer.match(prompt)
 415         if answer == "":
 416             answer = m.group(1)
 417         answer = answer[:1].upper()
 418
 419     if answer == 'R':
 420         reject(directory, upload)
 421     elif answer == 'A':
 422         # upload.try_autobyhand must not be run with No-Action.
 423         if Options['No-Action']:
 424             accept(directory, upload)
 425         elif upload.try_autobyhand():
 426             accept(directory, upload)
 427         else:
 428             print "W: redirecting to BYHAND as automatic processing failed."
 429             accept_to_new(directory, upload)
 430     elif answer == 'N':
 431         accept_to_new(directory, upload)
 432     elif answer == 'Q':
 433         sys.exit(0)
 434     elif answer == 'S':
 435         processed = False
 436
 437     if not Options['No-Action']:
 438         upload.commit()
 439
 440     return processed
 441
 442 ###############################################################################
 443
 444 def unlink_if_exists(path):
 445     try:
 446         os.unlink(path)
 447     except OSError as e:
 448         if e.errno != errno.ENOENT:
 449             raise
 450
 451 def process_it(directory, changes, keyrings, session):
 452     global Logger
 453
 454     print "\n{0}\n".format(changes.filename)
 455     Logger.log(["Processing changes file", changes.filename])
 456
 457     with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
 458         processed = action(directory, upload)
 459         if processed and not Options['No-Action']:
 460             unlink_if_exists(os.path.join(directory, changes.filename))
 461             for fn in changes.files:
 462                 unlink_if_exists(os.path.join(directory, fn))
 463
 464 ###############################################################################
 465
 466 def process_changes(changes_filenames):
 467     session = DBConn().session()
 468     keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
 469     keyring_files = [ k.keyring_name for k in keyrings ]
 470
 471     changes = []
 472     for fn in changes_filenames:
 473         try:
 474             directory, filename = os.path.split(fn)
 475             c = daklib.upload.Changes(directory, filename, keyring_files)
 476             changes.append([directory, c])
 477         except Exception as e:
 478             Logger.log([filename, "Error while loading changes: {0}".format(e)])
 479
 480     changes.sort(key=lambda x: x[1])
 481
 482     for directory, c in changes:
 483         process_it(directory, c, keyring_files, session)
 484
 485     session.rollback()
 486
 487 ###############################################################################
 488
 489 def main():
 490     global Options, Logger
 491
 492     cnf = Config()
 493     summarystats = SummaryStats()
 494
 495     Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
 496                  ('h',"help","Dinstall::Options::Help"),
 497                  ('n',"no-action","Dinstall::Options::No-Action"),
 498                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
 499                  ('s',"no-mail", "Dinstall::Options::No-Mail"),
 500                  ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
 501
 502     for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
 503               "version", "directory"]:
 504         if not cnf.has_key("Dinstall::Options::%s" % (i)):
 505             cnf["Dinstall::Options::%s" % (i)] = ""
 506
 507     changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
 508     Options = cnf.subtree("Dinstall::Options")
 509
 510     if Options["Help"]:
 511         usage()
 512
 513     # -n/--dry-run invalidates some other options which would involve things happening
 514     if Options["No-Action"]:
 515         Options["Automatic"] = ""
 516
 517     # Check that we aren't going to clash with the daily cron job
 518     if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
 519         utils.fubar("Archive maintenance in progress.  Try again later.")
 520
 521     # Obtain lock if not in no-action mode and initialize the log
 522     if not Options["No-Action"]:
 523         lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
 524         try:
 525             fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 526         except IOError as e:
 527             if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
 528                 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
 529             else:
 530                 raise
 531
 532         # Initialise UrgencyLog() - it will deal with the case where we don't
 533         # want to log urgencies
 534         urgencylog = UrgencyLog()
 535
 536     Logger = daklog.Logger("process-upload", Options["No-Action"])
 537
 538     # If we have a directory flag, use it to find our files
 539     if cnf["Dinstall::Options::Directory"] != "":
 540         # Note that we clobber the list of files we were given in this case
 541         # so warn if the user has done both
 542         if len(changes_files) > 0:
 543             utils.warn("Directory provided so ignoring files given on command line")
 544
 545         changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
 546         Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
 547     elif not len(changes_files) > 0:
 548         utils.fubar("No changes files given and no directory specified")
 549     else:
 550         Logger.log(["Using changes files from command-line", len(changes_files)])
 551
 552     process_changes(changes_files)
 553
 554     if summarystats.accept_count:
 555         sets = "set"
 556         if summarystats.accept_count > 1:
 557             sets = "sets"
 558         print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
 559                                                 utils.size_type(int(summarystats.accept_bytes)))
 560         Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
 561
 562     if summarystats.reject_count:
 563         sets = "set"
 564         if summarystats.reject_count > 1:
 565             sets = "sets"
 566         print "Rejected %d package %s." % (summarystats.reject_count, sets)
 567         Logger.log(["rejected", summarystats.reject_count])
 568
 569     if not Options["No-Action"]:
 570         urgencylog.close()
 571
 572     Logger.close()
 573
 574 ###############################################################################
 575
 576 if __name__ == '__main__':
 577     main()