dak/process_upload.py

   1 #!/usr/bin/env python
   2
   3 """
   4 Checks Debian packages from Incoming
   5 @contact: Debian FTP Master <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
   8 @copyright: 2009  Mark Hymers <mhy@debian.org>
   9 @copyright: 2009  Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 # based on process-unchecked and process-accepted
  28
  29 ## pu|pa: locking (daily.lock)
  30 ## pu|pa: parse arguments -> list of changes files
  31 ## pa: initialize urgency log
  32 ## pu|pa: sort changes list
  33
  34 ## foreach changes:
  35 ###  pa: load dak file
  36 ##   pu: copy CHG to tempdir
  37 ##   pu: check CHG signature
  38 ##   pu: parse changes file
  39 ##   pu: checks:
  40 ##     pu: check distribution (mappings, rejects)
  41 ##     pu: copy FILES to tempdir
  42 ##     pu: check whether CHG already exists in CopyChanges
  43 ##     pu: check whether FILES already exist in one of the policy queues
  44 ##     for deb in FILES:
  45 ##       pu: extract control information
  46 ##       pu: various checks on control information
  47 ##       pu|pa: search for source (in CHG, projectb, policy queues)
  48 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  49 ##       pu|pa: check whether deb already exists in the pool
  50 ##     for src in FILES:
  51 ##       pu: various checks on filenames and CHG consistency
  52 ##       pu: if isdsc: check signature
  53 ##     for file in FILES:
  54 ##       pu: various checks
  55 ##       pu: NEW?
  56 ##       //pu: check whether file already exists in the pool
  57 ##       pu: store what "Component" the package is currently in
  58 ##     pu: check whether we found everything we were looking for in CHG
  59 ##     pu: check the DSC:
  60 ##       pu: check whether we need and have ONE DSC
  61 ##       pu: parse the DSC
  62 ##       pu: various checks //maybe drop some of the in favor of lintian
  63 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  64 ##       pu: check whether DSC_FILES is consistent with "Format"
  65 ##       for src in DSC_FILES:
  66 ##         pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
  67 ##     pu: create new tempdir
  68 ##     pu: create symlink mirror of source
  69 ##     pu: unpack source
  70 ##     pu: extract changelog information for BTS
  71 ##     //pu: create missing .orig symlink
  72 ##     pu: check with lintian
  73 ##     for file in FILES:
  74 ##       pu: check checksums and sizes
  75 ##     for file in DSC_FILES:
  76 ##       pu: check checksums and sizes
  77 ##     pu: CHG: check urgency
  78 ##     for deb in FILES:
  79 ##       pu: extract contents list and check for dubious timestamps
  80 ##     pu: check that the uploader is actually allowed to upload the package
  81 ###  pa: install:
  82 ###    if stable_install:
  83 ###      pa: remove from p-u
  84 ###      pa: add to stable
  85 ###      pa: move CHG to morgue
  86 ###      pa: append data to ChangeLog
  87 ###      pa: send mail
  88 ###      pa: remove .dak file
  89 ###    else:
  90 ###      pa: add dsc to db:
  91 ###        for file in DSC_FILES:
  92 ###          pa: add file to file
  93 ###          pa: add file to dsc_files
  94 ###        pa: create source entry
  95 ###        pa: update source associations
  96 ###        pa: update src_uploaders
  97 ###      for deb in FILES:
  98 ###        pa: add deb to db:
  99 ###          pa: add file to file
 100 ###          pa: find source entry
 101 ###          pa: create binaries entry
 102 ###          pa: update binary associations
 103 ###      pa: .orig component move
 104 ###      pa: move files to pool
 105 ###      pa: save CHG
 106 ###      pa: move CHG to done/
 107 ###      pa: change entry in queue_build
 108 ##   pu: use dispatch table to choose target queue:
 109 ##     if NEW:
 110 ##       pu: write .dak file
 111 ##       pu: move to NEW
 112 ##       pu: send mail
 113 ##     elsif AUTOBYHAND:
 114 ##       pu: run autobyhand script
 115 ##       pu: if stuff left, do byhand or accept
 116 ##     elsif targetqueue in (oldstable, stable, embargo, unembargo):
 117 ##       pu: write .dak file
 118 ##       pu: check overrides
 119 ##       pu: move to queue
 120 ##       pu: send mail
 121 ##     else:
 122 ##       pu: write .dak file
 123 ##       pu: move to ACCEPTED
 124 ##       pu: send mails
 125 ##       pu: create files for BTS
 126 ##       pu: create entry in queue_build
 127 ##       pu: check overrides
 128
 129 # Integrity checks
 130 ## GPG
 131 ## Parsing changes (check for duplicates)
 132 ## Parse dsc
 133 ## file list checks
 134
 135 # New check layout (TODO: Implement)
 136 ## Permission checks
 137 ### suite mappings
 138 ### ACLs
 139 ### version checks (suite)
 140 ### override checks
 141
 142 ## Source checks
 143 ### copy orig
 144 ### unpack
 145 ### BTS changelog
 146 ### src contents
 147 ### lintian
 148 ### urgency log
 149
 150 ## Binary checks
 151 ### timestamps
 152 ### control checks
 153 ### src relation check
 154 ### contents
 155
 156 ## Database insertion (? copy from stuff)
 157 ### BYHAND / NEW / Policy queues
 158 ### Pool
 159
 160 ## Queue builds
 161
 162 import datetime
 163 import errno
 164 from errno import EACCES, EAGAIN
 165 import fcntl
 166 import os
 167 import sys
 168 import traceback
 169 import apt_pkg
 170 import time
 171 from sqlalchemy.orm.exc import NoResultFound
 172
 173 from daklib import daklog
 174 from daklib.dbconn import *
 175 from daklib.urgencylog import UrgencyLog
 176 from daklib.summarystats import SummaryStats
 177 from daklib.config import Config
 178 import daklib.utils as utils
 179 from daklib.regexes import *
 180
 181 import daklib.announce
 182 import daklib.archive
 183 import daklib.checks
 184 import daklib.upload
 185
 186 ###############################################################################
 187
 188 Options = None
 189 Logger = None
 190
 191 ###############################################################################
 192
 193 def usage (exit_code=0):
 194     print """Usage: dak process-upload [OPTION]... [CHANGES]...
 195   -a, --automatic           automatic run
 196   -d, --directory <DIR>     process uploads in <DIR>
 197   -h, --help                show this help and exit.
 198   -n, --no-action           don't do anything
 199   -p, --no-lock             don't check lockfile !! for cron.daily only !!
 200   -s, --no-mail             don't send any mail
 201   -V, --version             display the version number and exit"""
 202     sys.exit(exit_code)
 203
 204 ###############################################################################
 205
 206 def try_or_reject(function):
 207     """Try to call function or reject the upload if that fails
 208     """
 209     def wrapper(directory, upload, *args, **kwargs):
 210         reason = 'No exception caught. This should not happen.'
 211
 212         try:
 213             return function(directory, upload, *args, **kwargs)
 214         except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
 215             reason = unicode(e)
 216         except Exception as e:
 217             reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
 218
 219         try:
 220             upload.rollback()
 221             return real_reject(directory, upload, reason=reason)
 222         except Exception as e:
 223             reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
 224             upload.rollback()
 225             return real_reject(directory, upload, reason=reason, notify=False)
 226
 227         raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
 228
 229     return wrapper
 230
 231 def get_processed_upload(upload):
 232     changes = upload.changes
 233     control = upload.changes.changes
 234
 235     pu = daklib.announce.ProcessedUpload()
 236
 237     pu.maintainer = control.get('Maintainer')
 238     pu.changed_by = control.get('Changed-By')
 239     pu.fingerprint = changes.primary_fingerprint
 240
 241     pu.suites = upload.final_suites or []
 242     pu.from_policy_suites = []
 243
 244     pu.changes = open(upload.changes.path, 'r').read()
 245     pu.changes_filename = upload.changes.filename
 246     pu.sourceful = upload.changes.sourceful
 247     pu.source = control.get('Source')
 248     pu.version = control.get('Version')
 249     pu.architecture = control.get('Architecture')
 250     pu.bugs = changes.closed_bugs
 251
 252     pu.program = "process-upload"
 253
 254     pu.warnings = upload.warnings
 255
 256     return pu
 257
 258 @try_or_reject
 259 def accept(directory, upload):
 260     cnf = Config()
 261
 262     Logger.log(['ACCEPT', upload.changes.filename])
 263     print "ACCEPT"
 264
 265     upload.install()
 266
 267     accepted_to_real_suite = False
 268     for suite in upload.final_suites:
 269         accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
 270
 271     sourceful_upload = 'source' in upload.changes.architectures
 272
 273     control = upload.changes.changes
 274     if sourceful_upload and not Options['No-Action']:
 275         urgency = control.get('Urgency')
 276         if urgency not in cnf.value_list('Urgency::Valid'):
 277             urgency = cnf['Urgency::Default']
 278         UrgencyLog().log(control['Source'], control['Version'], urgency)
 279
 280     pu = get_processed_upload(upload)
 281     daklib.announce.announce_accept(pu)
 282
 283     # Move .changes to done, but only for uploads that were accepted to a
 284     # real suite.  process-policy will handle this for uploads to queues.
 285     if accepted_to_real_suite:
 286         src = os.path.join(upload.directory, upload.changes.filename)
 287
 288         now = datetime.datetime.now()
 289         donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
 290         dst = os.path.join(donedir, upload.changes.filename)
 291         dst = utils.find_next_free(dst)
 292
 293         upload.transaction.fs.copy(src, dst, mode=0o644)
 294
 295     SummaryStats().accept_count += 1
 296     SummaryStats().accept_bytes += upload.changes.bytes
 297
 298 @try_or_reject
 299 def accept_to_new(directory, upload):
 300
 301     Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
 302     print "ACCEPT-TO-NEW"
 303
 304     upload.install_to_new()
 305     # TODO: tag bugs pending
 306
 307     pu = get_processed_upload(upload)
 308     daklib.announce.announce_new(pu)
 309
 310     SummaryStats().accept_count += 1
 311     SummaryStats().accept_bytes += upload.changes.bytes
 312
 313 @try_or_reject
 314 def reject(directory, upload, reason=None, notify=True):
 315     real_reject(directory, upload, reason, notify)
 316
 317 def real_reject(directory, upload, reason=None, notify=True):
 318     # XXX: rejection itself should go to daklib.archive.ArchiveUpload
 319     cnf = Config()
 320
 321     Logger.log(['REJECT', upload.changes.filename])
 322     print "REJECT"
 323
 324     fs = upload.transaction.fs
 325     rejectdir = cnf['Dir::Reject']
 326
 327     files = [ f.filename for f in upload.changes.files.itervalues() ]
 328     files.append(upload.changes.filename)
 329
 330     for fn in files:
 331         src = os.path.join(upload.directory, fn)
 332         dst = utils.find_next_free(os.path.join(rejectdir, fn))
 333         if not os.path.exists(src):
 334             continue
 335         fs.copy(src, dst)
 336
 337     if upload.reject_reasons is not None:
 338         if reason is None:
 339             reason = ''
 340         reason = reason + '\n' + '\n'.join(upload.reject_reasons)
 341
 342     if reason is None:
 343         reason = '(Unknown reason. Please check logs.)'
 344
 345     dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
 346     fh = fs.create(dst)
 347     fh.write(reason)
 348     fh.close()
 349
 350     if notify:
 351         pu = get_processed_upload(upload)
 352         daklib.announce.announce_reject(pu, reason)
 353
 354     SummaryStats().reject_count += 1
 355
 356 ###############################################################################
 357
 358 def action(directory, upload):
 359     changes = upload.changes
 360     processed = True
 361
 362     global Logger
 363
 364     cnf = Config()
 365
 366     okay = upload.check()
 367
 368     summary = changes.changes.get('Changes', '')
 369
 370     package_info = []
 371     if okay:
 372         if changes.source is not None:
 373             package_info.append("source:{0}".format(changes.source.dsc['Source']))
 374         for binary in changes.binaries:
 375             package_info.append("binary:{0}".format(binary.control['Package']))
 376
 377     (prompt, answer) = ("", "XXX")
 378     if Options["No-Action"] or Options["Automatic"]:
 379         answer = 'S'
 380
 381     print summary
 382     print
 383     print "\n".join(package_info)
 384     print
 385     if len(upload.warnings) > 0:
 386         print "\n".join(upload.warnings)
 387         print
 388
 389     if len(upload.reject_reasons) > 0:
 390         print "Reason:"
 391         print "\n".join(upload.reject_reasons)
 392         print
 393
 394         path = os.path.join(directory, changes.filename)
 395         created = os.stat(path).st_mtime
 396         now = time.time()
 397         too_new = (now - created < int(cnf['Dinstall::SkipTime']))
 398
 399         if too_new:
 400             print "SKIP (too new)"
 401             prompt = "[S]kip, Quit ?"
 402         else:
 403             prompt = "[R]eject, Skip, Quit ?"
 404             if Options["Automatic"]:
 405                 answer = 'R'
 406     elif upload.new:
 407         prompt = "[N]ew, Skip, Quit ?"
 408         if Options['Automatic']:
 409             answer = 'N'
 410     else:
 411         prompt = "[A]ccept, Skip, Quit ?"
 412         if Options['Automatic']:
 413             answer = 'A'
 414
 415     while prompt.find(answer) == -1:
 416         answer = utils.our_raw_input(prompt)
 417         m = re_default_answer.match(prompt)
 418         if answer == "":
 419             answer = m.group(1)
 420         answer = answer[:1].upper()
 421
 422     if answer == 'R':
 423         reject(directory, upload)
 424     elif answer == 'A':
 425         # upload.try_autobyhand must not be run with No-Action.
 426         if Options['No-Action']:
 427             accept(directory, upload)
 428         elif upload.try_autobyhand():
 429             accept(directory, upload)
 430         else:
 431             print "W: redirecting to BYHAND as automatic processing failed."
 432             accept_to_new(directory, upload)
 433     elif answer == 'N':
 434         accept_to_new(directory, upload)
 435     elif answer == 'Q':
 436         sys.exit(0)
 437     elif answer == 'S':
 438         processed = False
 439
 440     if not Options['No-Action']:
 441         upload.commit()
 442
 443     return processed
 444
 445 ###############################################################################
 446
 447 def unlink_if_exists(path):
 448     try:
 449         os.unlink(path)
 450     except OSError as e:
 451         if e.errno != errno.ENOENT:
 452             raise
 453
 454 def process_it(directory, changes, keyrings):
 455     global Logger
 456
 457     print "\n{0}\n".format(changes.filename)
 458     Logger.log(["Processing changes file", changes.filename])
 459
 460     with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
 461         processed = action(directory, upload)
 462         if processed and not Options['No-Action']:
 463             session = DBConn().session()
 464             history = SignatureHistory.from_signed_file(upload.changes)
 465             if history.query(session) is None:
 466                 session.add(history)
 467                 session.commit()
 468             session.close()
 469
 470             unlink_if_exists(os.path.join(directory, changes.filename))
 471             for fn in changes.files:
 472                 unlink_if_exists(os.path.join(directory, fn))
 473
 474 ###############################################################################
 475
 476 def process_changes(changes_filenames):
 477     session = DBConn().session()
 478     keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
 479     keyring_files = [ k.keyring_name for k in keyrings ]
 480     session.close()
 481
 482     changes = []
 483     for fn in changes_filenames:
 484         try:
 485             directory, filename = os.path.split(fn)
 486             c = daklib.upload.Changes(directory, filename, keyring_files)
 487             changes.append([directory, c])
 488         except Exception as e:
 489             Logger.log([filename, "Error while loading changes: {0}".format(e)])
 490
 491     changes.sort(key=lambda x: x[1])
 492
 493     for directory, c in changes:
 494         process_it(directory, c, keyring_files)
 495
 496 ###############################################################################
 497
 498 def main():
 499     global Options, Logger
 500
 501     cnf = Config()
 502     summarystats = SummaryStats()
 503
 504     Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
 505                  ('h',"help","Dinstall::Options::Help"),
 506                  ('n',"no-action","Dinstall::Options::No-Action"),
 507                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
 508                  ('s',"no-mail", "Dinstall::Options::No-Mail"),
 509                  ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
 510
 511     for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
 512               "version", "directory"]:
 513         if not cnf.has_key("Dinstall::Options::%s" % (i)):
 514             cnf["Dinstall::Options::%s" % (i)] = ""
 515
 516     changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
 517     Options = cnf.subtree("Dinstall::Options")
 518
 519     if Options["Help"]:
 520         usage()
 521
 522     # -n/--dry-run invalidates some other options which would involve things happening
 523     if Options["No-Action"]:
 524         Options["Automatic"] = ""
 525
 526     # Check that we aren't going to clash with the daily cron job
 527     if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
 528         utils.fubar("Archive maintenance in progress.  Try again later.")
 529
 530     # Obtain lock if not in no-action mode and initialize the log
 531     if not Options["No-Action"]:
 532         lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
 533         try:
 534             fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 535         except IOError as e:
 536             if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
 537                 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
 538             else:
 539                 raise
 540
 541         # Initialise UrgencyLog() - it will deal with the case where we don't
 542         # want to log urgencies
 543         urgencylog = UrgencyLog()
 544
 545     Logger = daklog.Logger("process-upload", Options["No-Action"])
 546
 547     # If we have a directory flag, use it to find our files
 548     if cnf["Dinstall::Options::Directory"] != "":
 549         # Note that we clobber the list of files we were given in this case
 550         # so warn if the user has done both
 551         if len(changes_files) > 0:
 552             utils.warn("Directory provided so ignoring files given on command line")
 553
 554         changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
 555         Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
 556     elif not len(changes_files) > 0:
 557         utils.fubar("No changes files given and no directory specified")
 558     else:
 559         Logger.log(["Using changes files from command-line", len(changes_files)])
 560
 561     process_changes(changes_files)
 562
 563     if summarystats.accept_count:
 564         sets = "set"
 565         if summarystats.accept_count > 1:
 566             sets = "sets"
 567         print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
 568                                                 utils.size_type(int(summarystats.accept_bytes)))
 569         Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
 570
 571     if summarystats.reject_count:
 572         sets = "set"
 573         if summarystats.reject_count > 1:
 574             sets = "sets"
 575         print "Rejected %d package %s." % (summarystats.reject_count, sets)
 576         Logger.log(["rejected", summarystats.reject_count])
 577
 578     if not Options["No-Action"]:
 579         urgencylog.close()
 580
 581     Logger.close()
 582
 583 ###############################################################################
 584
 585 if __name__ == '__main__':
 586     main()