dak/process_upload.py

   1 #!/usr/bin/env python
   2
   3 """
   4 Checks Debian packages from Incoming
   5 @contact: Debian FTP Master <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
   8 @copyright: 2009  Mark Hymers <mhy@debian.org>
   9 @copyright: 2009  Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 # based on process-unchecked and process-accepted
  28
  29 ## pu|pa: locking (daily.lock)
  30 ## pu|pa: parse arguments -> list of changes files
  31 ## pa: initialize urgency log
  32 ## pu|pa: sort changes list
  33
  34 ## foreach changes:
  35 ###  pa: load dak file
  36 ##   pu: copy CHG to tempdir
  37 ##   pu: check CHG signature
  38 ##   pu: parse changes file
  39 ##   pu: checks:
  40 ##     pu: check distribution (mappings, rejects)
  41 ##     pu: copy FILES to tempdir
  42 ##     pu: check whether CHG already exists in CopyChanges
  43 ##     pu: check whether FILES already exist in one of the policy queues
  44 ##     for deb in FILES:
  45 ##       pu: extract control information
  46 ##       pu: various checks on control information
  47 ##       pu|pa: search for source (in CHG, projectb, policy queues)
  48 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  49 ##       pu|pa: check whether deb already exists in the pool
  50 ##     for src in FILES:
  51 ##       pu: various checks on filenames and CHG consistency
  52 ##       pu: if isdsc: check signature
  53 ##     for file in FILES:
  54 ##       pu: various checks
  55 ##       pu: NEW?
  56 ##       //pu: check whether file already exists in the pool
  57 ##       pu: store what "Component" the package is currently in
  58 ##     pu: check whether we found everything we were looking for in CHG
  59 ##     pu: check the DSC:
  60 ##       pu: check whether we need and have ONE DSC
  61 ##       pu: parse the DSC
  62 ##       pu: various checks //maybe drop some of the in favor of lintian
  63 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  64 ##       pu: check whether DSC_FILES is consistent with "Format"
  65 ##       for src in DSC_FILES:
  66 ##         pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
  67 ##     pu: create new tempdir
  68 ##     pu: create symlink mirror of source
  69 ##     pu: unpack source
  70 ##     pu: extract changelog information for BTS
  71 ##     //pu: create missing .orig symlink
  72 ##     pu: check with lintian
  73 ##     for file in FILES:
  74 ##       pu: check checksums and sizes
  75 ##     for file in DSC_FILES:
  76 ##       pu: check checksums and sizes
  77 ##     pu: CHG: check urgency
  78 ##     for deb in FILES:
  79 ##       pu: extract contents list and check for dubious timestamps
  80 ##     pu: check that the uploader is actually allowed to upload the package
  81 ###  pa: install:
  82 ###    if stable_install:
  83 ###      pa: remove from p-u
  84 ###      pa: add to stable
  85 ###      pa: move CHG to morgue
  86 ###      pa: append data to ChangeLog
  87 ###      pa: send mail
  88 ###      pa: remove .dak file
  89 ###    else:
  90 ###      pa: add dsc to db:
  91 ###        for file in DSC_FILES:
  92 ###          pa: add file to file
  93 ###          pa: add file to dsc_files
  94 ###        pa: create source entry
  95 ###        pa: update source associations
  96 ###        pa: update src_uploaders
  97 ###      for deb in FILES:
  98 ###        pa: add deb to db:
  99 ###          pa: add file to file
 100 ###          pa: find source entry
 101 ###          pa: create binaries entry
 102 ###          pa: update binary associations
 103 ###      pa: .orig component move
 104 ###      pa: move files to pool
 105 ###      pa: save CHG
 106 ###      pa: move CHG to done/
 107 ###      pa: change entry in queue_build
 108 ##   pu: use dispatch table to choose target queue:
 109 ##     if NEW:
 110 ##       pu: write .dak file
 111 ##       pu: move to NEW
 112 ##       pu: send mail
 113 ##     elsif AUTOBYHAND:
 114 ##       pu: run autobyhand script
 115 ##       pu: if stuff left, do byhand or accept
 116 ##     elsif targetqueue in (oldstable, stable, embargo, unembargo):
 117 ##       pu: write .dak file
 118 ##       pu: check overrides
 119 ##       pu: move to queue
 120 ##       pu: send mail
 121 ##     else:
 122 ##       pu: write .dak file
 123 ##       pu: move to ACCEPTED
 124 ##       pu: send mails
 125 ##       pu: create files for BTS
 126 ##       pu: create entry in queue_build
 127 ##       pu: check overrides
 128
 129 # Integrity checks
 130 ## GPG
 131 ## Parsing changes (check for duplicates)
 132 ## Parse dsc
 133 ## file list checks
 134
 135 # New check layout (TODO: Implement)
 136 ## Permission checks
 137 ### suite mappings
 138 ### ACLs
 139 ### version checks (suite)
 140 ### override checks
 141
 142 ## Source checks
 143 ### copy orig
 144 ### unpack
 145 ### BTS changelog
 146 ### src contents
 147 ### lintian
 148 ### urgency log
 149
 150 ## Binary checks
 151 ### timestamps
 152 ### control checks
 153 ### src relation check
 154 ### contents
 155
 156 ## Database insertion (? copy from stuff)
 157 ### BYHAND / NEW / Policy queues
 158 ### Pool
 159
 160 ## Queue builds
 161
 162 import datetime
 163 import errno
 164 from errno import EACCES, EAGAIN
 165 import fcntl
 166 import os
 167 import sys
 168 import traceback
 169 import apt_pkg
 170 import time
 171 from sqlalchemy.orm.exc import NoResultFound
 172
 173 from daklib import daklog
 174 from daklib.dbconn import *
 175 from daklib.urgencylog import UrgencyLog
 176 from daklib.summarystats import SummaryStats
 177 from daklib.config import Config
 178 import daklib.utils as utils
 179 from daklib.regexes import *
 180
 181 import daklib.announce
 182 import daklib.archive
 183 import daklib.checks
 184 import daklib.upload
 185
 186 ###############################################################################
 187
 188 Options = None
 189 Logger = None
 190
 191 ###############################################################################
 192
 193 def usage (exit_code=0):
 194     print """Usage: dak process-upload [OPTION]... [CHANGES]...
 195   -a, --automatic           automatic run
 196   -d, --directory <DIR>     process uploads in <DIR>
 197   -h, --help                show this help and exit.
 198   -n, --no-action           don't do anything
 199   -p, --no-lock             don't check lockfile !! for cron.daily only !!
 200   -s, --no-mail             don't send any mail
 201   -V, --version             display the version number and exit"""
 202     sys.exit(exit_code)
 203
 204 ###############################################################################
 205
 206 def try_or_reject(function):
 207     """Try to call function or reject the upload if that fails
 208     """
 209     def wrapper(directory, upload, *args, **kwargs):
 210         reason = 'No exception caught. This should not happen.'
 211
 212         try:
 213             return function(directory, upload, *args, **kwargs)
 214         except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
 215             reason = e
 216         except Exception as e:
 217             reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
 218
 219         try:
 220             upload.rollback()
 221             return real_reject(directory, upload, reason=reason)
 222         except Exception as e:
 223             reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
 224             upload.rollback()
 225             return real_reject(directory, upload, reason=reason, notify=False)
 226
 227         raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
 228
 229     return wrapper
 230
 231 def get_processed_upload(upload):
 232     changes = upload.changes
 233     control = upload.changes.changes
 234
 235     pu = daklib.announce.ProcessedUpload()
 236
 237     pu.maintainer = control.get('Maintainer')
 238     pu.changed_by = control.get('Changed-By')
 239     pu.fingerprint = changes.primary_fingerprint
 240
 241     pu.suites = upload.final_suites or []
 242     pu.from_policy_suites = []
 243
 244     pu.changes = open(upload.changes.path, 'r').read()
 245     pu.changes_filename = upload.changes.filename
 246     pu.sourceful = upload.changes.source is not None
 247     pu.source = control.get('Source')
 248     pu.version = control.get('Version')
 249     pu.architecture = control.get('Architecture')
 250     pu.bugs = changes.closed_bugs
 251
 252     pu.program = "process-upload"
 253
 254     return pu
 255
 256 @try_or_reject
 257 def accept(directory, upload):
 258     cnf = Config()
 259
 260     Logger.log(['ACCEPT', upload.changes.filename])
 261
 262     upload.install()
 263
 264     accepted_to_real_suite = False
 265     for suite in upload.final_suites:
 266         accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
 267
 268     sourceful_upload = 'source' in upload.changes.architectures
 269
 270     control = upload.changes.changes
 271     if sourceful_upload and not Options['No-Action']:
 272         urgency = control.get('Urgency')
 273         if urgency not in cnf.value_list('Urgency::Valid'):
 274             urgency = cnf['Urgency::Default']
 275         UrgencyLog().log(control['Source'], control['Version'], urgency)
 276
 277     pu = get_processed_upload(upload)
 278     daklib.announce.announce_accept(pu)
 279
 280     # Move .changes to done, but only for uploads that were accepted to a
 281     # real suite.  process-policy will handle this for uploads to queues.
 282     if accepted_to_real_suite:
 283         src = os.path.join(upload.directory, upload.changes.filename)
 284
 285         now = datetime.datetime.now()
 286         donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
 287         dst = os.path.join(donedir, upload.changes.filename)
 288         dst = utils.find_next_free(dst)
 289
 290         upload.transaction.fs.copy(src, dst, mode=0o644)
 291
 292     SummaryStats().accept_count += 1
 293     SummaryStats().accept_bytes += upload.changes.bytes
 294
 295 @try_or_reject
 296 def accept_to_new(directory, upload):
 297     cnf = Config()
 298
 299     Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
 300
 301     upload.install_to_new()
 302     # TODO: tag bugs pending
 303
 304     pu = get_processed_upload(upload)
 305     daklib.announce.announce_new(pu)
 306
 307     SummaryStats().accept_count += 1
 308     SummaryStats().accept_bytes += upload.changes.bytes
 309
 310 @try_or_reject
 311 def reject(directory, upload, reason=None, notify=True):
 312     real_reject(directory, upload, reason, notify)
 313
 314 def real_reject(directory, upload, reason=None, notify=True):
 315     # XXX: rejection itself should go to daklib.archive.ArchiveUpload
 316     cnf = Config()
 317
 318     Logger.log(['REJECT', upload.changes.filename])
 319
 320     fs = upload.transaction.fs
 321     rejectdir = cnf['Dir::Reject']
 322
 323     files = [ f.filename for f in upload.changes.files.itervalues() ]
 324     files.append(upload.changes.filename)
 325
 326     for fn in files:
 327         src = os.path.join(upload.directory, fn)
 328         dst = utils.find_next_free(os.path.join(rejectdir, fn))
 329         if not os.path.exists(src):
 330             continue
 331         fs.copy(src, dst)
 332
 333     if upload.reject_reasons is not None:
 334         if reason is None:
 335             reason = ''
 336         reason = reason + '\n' + '\n'.join(upload.reject_reasons)
 337
 338     if reason is None:
 339         reason = '(Unknown reason. Please check logs.)'
 340
 341     dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
 342     fh = fs.create(dst)
 343     fh.write(reason)
 344     fh.close()
 345
 346     if notify:
 347         pu = get_processed_upload(upload)
 348         daklib.announce.announce_reject(pu, reason)
 349
 350     SummaryStats().reject_count += 1
 351
 352 ###############################################################################
 353
 354 def action(directory, upload):
 355     changes = upload.changes
 356     processed = True
 357
 358     global Logger
 359
 360     cnf = Config()
 361
 362     okay = upload.check()
 363
 364     summary = changes.changes.get('Changes', '')
 365
 366     package_info = []
 367     if okay:
 368         if changes.source is not None:
 369             package_info.append("source:{0}".format(changes.source.dsc['Source']))
 370         for binary in changes.binaries:
 371             package_info.append("binary:{0}".format(binary.control['Package']))
 372
 373     (prompt, answer) = ("", "XXX")
 374     if Options["No-Action"] or Options["Automatic"]:
 375         answer = 'S'
 376
 377     queuekey = ''
 378
 379     print summary
 380     print
 381     print "\n".join(package_info)
 382     print
 383
 384     if len(upload.reject_reasons) > 0:
 385         print "Reason:"
 386         print "\n".join(upload.reject_reasons)
 387         print
 388
 389         path = os.path.join(directory, changes.filename)
 390         created = os.stat(path).st_mtime
 391         now = time.time()
 392         too_new = (now - created < int(cnf['Dinstall::SkipTime']))
 393
 394         if too_new:
 395             print "SKIP (too new)"
 396             prompt = "[S]kip, Quit ?"
 397         else:
 398             prompt = "[R]eject, Skip, Quit ?"
 399             if Options["Automatic"]:
 400                 answer = 'R'
 401     elif upload.new:
 402         prompt = "[N]ew, Skip, Quit ?"
 403         if Options['Automatic']:
 404             answer = 'N'
 405     else:
 406         prompt = "[A]ccept, Skip, Quit ?"
 407         if Options['Automatic']:
 408             answer = 'A'
 409
 410     while prompt.find(answer) == -1:
 411         answer = utils.our_raw_input(prompt)
 412         m = re_default_answer.match(prompt)
 413         if answer == "":
 414             answer = m.group(1)
 415         answer = answer[:1].upper()
 416
 417     if answer == 'R':
 418         reject(directory, upload)
 419     elif answer == 'A':
 420         # upload.try_autobyhand must not be run with No-Action.
 421         if Options['No-Action']:
 422             accept(directory, upload)
 423         elif upload.try_autobyhand():
 424             accept(directory, upload)
 425         else:
 426             print "W: redirecting to BYHAND as automatic processing failed."
 427             accept_to_new(directory, upload)
 428     elif answer == 'N':
 429         accept_to_new(directory, upload)
 430     elif answer == 'Q':
 431         sys.exit(0)
 432     elif answer == 'S':
 433         processed = False
 434
 435     if not Options['No-Action']:
 436         upload.commit()
 437
 438     return processed
 439
 440 ###############################################################################
 441
 442 def unlink_if_exists(path):
 443     try:
 444         os.unlink(path)
 445     except OSError as e:
 446         if e.errno != errno.ENOENT:
 447             raise
 448
 449 def process_it(directory, changes, keyrings, session):
 450     global Logger
 451
 452     print "\n{0}\n".format(changes.filename)
 453     Logger.log(["Processing changes file", changes.filename])
 454
 455     with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
 456         processed = action(directory, upload)
 457         if processed and not Options['No-Action']:
 458             unlink_if_exists(os.path.join(directory, changes.filename))
 459             for fn in changes.files:
 460                 unlink_if_exists(os.path.join(directory, fn))
 461
 462 ###############################################################################
 463
 464 def process_changes(changes_filenames):
 465     session = DBConn().session()
 466     keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
 467     keyring_files = [ k.keyring_name for k in keyrings ]
 468
 469     changes = []
 470     for fn in changes_filenames:
 471         try:
 472             directory, filename = os.path.split(fn)
 473             c = daklib.upload.Changes(directory, filename, keyring_files)
 474             changes.append([directory, c])
 475         except Exception as e:
 476             Logger.log([filename, "Error while loading changes: {0}".format(e)])
 477
 478     changes.sort(key=lambda x: x[1])
 479
 480     for directory, c in changes:
 481         process_it(directory, c, keyring_files, session)
 482
 483     session.rollback()
 484
 485 ###############################################################################
 486
 487 def main():
 488     global Options, Logger
 489
 490     cnf = Config()
 491     summarystats = SummaryStats()
 492
 493     Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
 494                  ('h',"help","Dinstall::Options::Help"),
 495                  ('n',"no-action","Dinstall::Options::No-Action"),
 496                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
 497                  ('s',"no-mail", "Dinstall::Options::No-Mail"),
 498                  ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
 499
 500     for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
 501               "version", "directory"]:
 502         if not cnf.has_key("Dinstall::Options::%s" % (i)):
 503             cnf["Dinstall::Options::%s" % (i)] = ""
 504
 505     changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
 506     Options = cnf.subtree("Dinstall::Options")
 507
 508     if Options["Help"]:
 509         usage()
 510
 511     # -n/--dry-run invalidates some other options which would involve things happening
 512     if Options["No-Action"]:
 513         Options["Automatic"] = ""
 514
 515     # Check that we aren't going to clash with the daily cron job
 516     if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
 517         utils.fubar("Archive maintenance in progress.  Try again later.")
 518
 519     # Obtain lock if not in no-action mode and initialize the log
 520     if not Options["No-Action"]:
 521         lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
 522         try:
 523             fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 524         except IOError as e:
 525             if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
 526                 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
 527             else:
 528                 raise
 529
 530         # Initialise UrgencyLog() - it will deal with the case where we don't
 531         # want to log urgencies
 532         urgencylog = UrgencyLog()
 533
 534     Logger = daklog.Logger("process-upload", Options["No-Action"])
 535
 536     # If we have a directory flag, use it to find our files
 537     if cnf["Dinstall::Options::Directory"] != "":
 538         # Note that we clobber the list of files we were given in this case
 539         # so warn if the user has done both
 540         if len(changes_files) > 0:
 541             utils.warn("Directory provided so ignoring files given on command line")
 542
 543         changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
 544         Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
 545     elif not len(changes_files) > 0:
 546         utils.fubar("No changes files given and no directory specified")
 547     else:
 548         Logger.log(["Using changes files from command-line", len(changes_files)])
 549
 550     process_changes(changes_files)
 551
 552     if summarystats.accept_count:
 553         sets = "set"
 554         if summarystats.accept_count > 1:
 555             sets = "sets"
 556         print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
 557                                                 utils.size_type(int(summarystats.accept_bytes)))
 558         Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
 559
 560     if summarystats.reject_count:
 561         sets = "set"
 562         if summarystats.reject_count > 1:
 563             sets = "sets"
 564         print "Rejected %d package %s." % (summarystats.reject_count, sets)
 565         Logger.log(["rejected", summarystats.reject_count])
 566
 567     if not Options["No-Action"]:
 568         urgencylog.close()
 569
 570     Logger.close()
 571
 572 ###############################################################################
 573
 574 if __name__ == '__main__':
 575     main()