dak/process_upload.py

   1 #!/usr/bin/env python
   2
   3 """
   4 Checks Debian packages from Incoming
   5 @contact: Debian FTP Master <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   7 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
   8 @copyright: 2009  Mark Hymers <mhy@debian.org>
   9 @copyright: 2009  Frank Lichtenheld <djpig@debian.org>
  10 @license: GNU General Public License version 2 or later
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 # based on process-unchecked and process-accepted
  28
  29 ## pu|pa: locking (daily.lock)
  30 ## pu|pa: parse arguments -> list of changes files
  31 ## pa: initialize urgency log
  32 ## pu|pa: sort changes list
  33
  34 ## foreach changes:
  35 ###  pa: load dak file
  36 ##   pu: copy CHG to tempdir
  37 ##   pu: check CHG signature
  38 ##   pu: parse changes file
  39 ##   pu: checks:
  40 ##     pu: check distribution (mappings, rejects)
  41 ##     pu: copy FILES to tempdir
  42 ##     pu: check whether CHG already exists in CopyChanges
  43 ##     pu: check whether FILES already exist in one of the policy queues
  44 ##     for deb in FILES:
  45 ##       pu: extract control information
  46 ##       pu: various checks on control information
  47 ##       pu|pa: search for source (in CHG, projectb, policy queues)
  48 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  49 ##       pu|pa: check whether deb already exists in the pool
  50 ##     for src in FILES:
  51 ##       pu: various checks on filenames and CHG consistency
  52 ##       pu: if isdsc: check signature
  53 ##     for file in FILES:
  54 ##       pu: various checks
  55 ##       pu: NEW?
  56 ##       //pu: check whether file already exists in the pool
  57 ##       pu: store what "Component" the package is currently in
  58 ##     pu: check whether we found everything we were looking for in CHG
  59 ##     pu: check the DSC:
  60 ##       pu: check whether we need and have ONE DSC
  61 ##       pu: parse the DSC
  62 ##       pu: various checks //maybe drop some of the in favor of lintian
  63 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
  64 ##       pu: check whether DSC_FILES is consistent with "Format"
  65 ##       for src in DSC_FILES:
  66 ##         pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
  67 ##     pu: create new tempdir
  68 ##     pu: create symlink mirror of source
  69 ##     pu: unpack source
  70 ##     pu: extract changelog information for BTS
  71 ##     //pu: create missing .orig symlink
  72 ##     pu: check with lintian
  73 ##     for file in FILES:
  74 ##       pu: check checksums and sizes
  75 ##     for file in DSC_FILES:
  76 ##       pu: check checksums and sizes
  77 ##     pu: CHG: check urgency
  78 ##     for deb in FILES:
  79 ##       pu: extract contents list and check for dubious timestamps
  80 ##     pu: check that the uploader is actually allowed to upload the package
  81 ###  pa: install:
  82 ###    if stable_install:
  83 ###      pa: remove from p-u
  84 ###      pa: add to stable
  85 ###      pa: move CHG to morgue
  86 ###      pa: append data to ChangeLog
  87 ###      pa: send mail
  88 ###      pa: remove .dak file
  89 ###    else:
  90 ###      pa: add dsc to db:
  91 ###        for file in DSC_FILES:
  92 ###          pa: add file to file
  93 ###          pa: add file to dsc_files
  94 ###        pa: create source entry
  95 ###        pa: update source associations
  96 ###        pa: update src_uploaders
  97 ###      for deb in FILES:
  98 ###        pa: add deb to db:
  99 ###          pa: add file to file
 100 ###          pa: find source entry
 101 ###          pa: create binaries entry
 102 ###          pa: update binary associations
 103 ###      pa: .orig component move
 104 ###      pa: move files to pool
 105 ###      pa: save CHG
 106 ###      pa: move CHG to done/
 107 ###      pa: change entry in queue_build
 108 ##   pu: use dispatch table to choose target queue:
 109 ##     if NEW:
 110 ##       pu: write .dak file
 111 ##       pu: move to NEW
 112 ##       pu: send mail
 113 ##     elsif AUTOBYHAND:
 114 ##       pu: run autobyhand script
 115 ##       pu: if stuff left, do byhand or accept
 116 ##     elsif targetqueue in (oldstable, stable, embargo, unembargo):
 117 ##       pu: write .dak file
 118 ##       pu: check overrides
 119 ##       pu: move to queue
 120 ##       pu: send mail
 121 ##     else:
 122 ##       pu: write .dak file
 123 ##       pu: move to ACCEPTED
 124 ##       pu: send mails
 125 ##       pu: create files for BTS
 126 ##       pu: create entry in queue_build
 127 ##       pu: check overrides
 128
 129 # Integrity checks
 130 ## GPG
 131 ## Parsing changes (check for duplicates)
 132 ## Parse dsc
 133 ## file list checks
 134
 135 # New check layout (TODO: Implement)
 136 ## Permission checks
 137 ### suite mappings
 138 ### ACLs
 139 ### version checks (suite)
 140 ### override checks
 141
 142 ## Source checks
 143 ### copy orig
 144 ### unpack
 145 ### BTS changelog
 146 ### src contents
 147 ### lintian
 148 ### urgency log
 149
 150 ## Binary checks
 151 ### timestamps
 152 ### control checks
 153 ### src relation check
 154 ### contents
 155
 156 ## Database insertion (? copy from stuff)
 157 ### BYHAND / NEW / Policy queues
 158 ### Pool
 159
 160 ## Queue builds
 161
 162 import datetime
 163 import errno
 164 from errno import EACCES, EAGAIN
 165 import fcntl
 166 import os
 167 import sys
 168 import traceback
 169 import apt_pkg
 170 import time
 171 from sqlalchemy.orm.exc import NoResultFound
 172
 173 from daklib import daklog
 174 from daklib.dbconn import *
 175 from daklib.urgencylog import UrgencyLog
 176 from daklib.summarystats import SummaryStats
 177 from daklib.config import Config
 178 import daklib.utils as utils
 179 from daklib.regexes import *
 180
 181 import daklib.announce
 182 import daklib.archive
 183 import daklib.checks
 184 import daklib.upload
 185
 186 ###############################################################################
 187
 188 Options = None
 189 Logger = None
 190
 191 ###############################################################################
 192
 193 def usage (exit_code=0):
 194     print """Usage: dak process-upload [OPTION]... [CHANGES]...
 195   -a, --automatic           automatic run
 196   -d, --directory <DIR>     process uploads in <DIR>
 197   -h, --help                show this help and exit.
 198   -n, --no-action           don't do anything
 199   -p, --no-lock             don't check lockfile !! for cron.daily only !!
 200   -s, --no-mail             don't send any mail
 201   -V, --version             display the version number and exit"""
 202     sys.exit(exit_code)
 203
 204 ###############################################################################
 205
 206 def try_or_reject(function):
 207     """Try to call function or reject the upload if that fails
 208     """
 209     def wrapper(directory, upload, *args, **kwargs):
 210         reason = 'No exception caught. This should not happen.'
 211
 212         try:
 213             return function(directory, upload, *args, **kwargs)
 214         except (daklib.archive.ArchiveException, daklib.checks.Reject) as e:
 215             reason = unicode(e)
 216         except Exception as e:
 217             reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
 218
 219         try:
 220             upload.rollback()
 221             return real_reject(directory, upload, reason=reason)
 222         except Exception as e:
 223             reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
 224             upload.rollback()
 225             return real_reject(directory, upload, reason=reason, notify=False)
 226
 227         raise Exception('Rejecting upload failed after multiple tries. Giving up. Last reason:\n{0}'.format(reason))
 228
 229     return wrapper
 230
 231 def get_processed_upload(upload):
 232     changes = upload.changes
 233     control = upload.changes.changes
 234
 235     pu = daklib.announce.ProcessedUpload()
 236
 237     pu.maintainer = control.get('Maintainer')
 238     pu.changed_by = control.get('Changed-By')
 239     pu.fingerprint = changes.primary_fingerprint
 240
 241     pu.suites = upload.final_suites or []
 242     pu.from_policy_suites = []
 243
 244     pu.changes = open(upload.changes.path, 'r').read()
 245     pu.changes_filename = upload.changes.filename
 246     pu.sourceful = upload.changes.sourceful
 247     pu.source = control.get('Source')
 248     pu.version = control.get('Version')
 249     pu.architecture = control.get('Architecture')
 250     pu.bugs = changes.closed_bugs
 251
 252     pu.program = "process-upload"
 253
 254     pu.warnings = upload.warnings
 255
 256     return pu
 257
 258 @try_or_reject
 259 def accept(directory, upload):
 260     cnf = Config()
 261
 262     Logger.log(['ACCEPT', upload.changes.filename])
 263     print "ACCEPT"
 264
 265     upload.install()
 266
 267     accepted_to_real_suite = False
 268     for suite in upload.final_suites:
 269         accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
 270
 271     sourceful_upload = 'source' in upload.changes.architectures
 272
 273     control = upload.changes.changes
 274     if sourceful_upload and not Options['No-Action']:
 275         urgency = control.get('Urgency')
 276         if urgency not in cnf.value_list('Urgency::Valid'):
 277             urgency = cnf['Urgency::Default']
 278         UrgencyLog().log(control['Source'], control['Version'], urgency)
 279
 280     pu = get_processed_upload(upload)
 281     daklib.announce.announce_accept(pu)
 282
 283     # Move .changes to done, but only for uploads that were accepted to a
 284     # real suite.  process-policy will handle this for uploads to queues.
 285     if accepted_to_real_suite:
 286         src = os.path.join(upload.directory, upload.changes.filename)
 287
 288         now = datetime.datetime.now()
 289         donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
 290         dst = os.path.join(donedir, upload.changes.filename)
 291         dst = utils.find_next_free(dst)
 292
 293         upload.transaction.fs.copy(src, dst, mode=0o644)
 294
 295     SummaryStats().accept_count += 1
 296     SummaryStats().accept_bytes += upload.changes.bytes
 297
 298 @try_or_reject
 299 def accept_to_new(directory, upload):
 300     cnf = Config()
 301
 302     Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
 303     print "ACCEPT-TO-NEW"
 304
 305     upload.install_to_new()
 306     # TODO: tag bugs pending
 307
 308     pu = get_processed_upload(upload)
 309     daklib.announce.announce_new(pu)
 310
 311     SummaryStats().accept_count += 1
 312     SummaryStats().accept_bytes += upload.changes.bytes
 313
 314 @try_or_reject
 315 def reject(directory, upload, reason=None, notify=True):
 316     real_reject(directory, upload, reason, notify)
 317
 318 def real_reject(directory, upload, reason=None, notify=True):
 319     # XXX: rejection itself should go to daklib.archive.ArchiveUpload
 320     cnf = Config()
 321
 322     Logger.log(['REJECT', upload.changes.filename])
 323     print "REJECT"
 324
 325     fs = upload.transaction.fs
 326     rejectdir = cnf['Dir::Reject']
 327
 328     files = [ f.filename for f in upload.changes.files.itervalues() ]
 329     files.append(upload.changes.filename)
 330
 331     for fn in files:
 332         src = os.path.join(upload.directory, fn)
 333         dst = utils.find_next_free(os.path.join(rejectdir, fn))
 334         if not os.path.exists(src):
 335             continue
 336         fs.copy(src, dst)
 337
 338     if upload.reject_reasons is not None:
 339         if reason is None:
 340             reason = ''
 341         reason = reason + '\n' + '\n'.join(upload.reject_reasons)
 342
 343     if reason is None:
 344         reason = '(Unknown reason. Please check logs.)'
 345
 346     dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
 347     fh = fs.create(dst)
 348     fh.write(reason)
 349     fh.close()
 350
 351     if notify:
 352         pu = get_processed_upload(upload)
 353         daklib.announce.announce_reject(pu, reason)
 354
 355     SummaryStats().reject_count += 1
 356
 357 ###############################################################################
 358
 359 def action(directory, upload):
 360     changes = upload.changes
 361     processed = True
 362
 363     global Logger
 364
 365     cnf = Config()
 366
 367     okay = upload.check()
 368
 369     summary = changes.changes.get('Changes', '')
 370
 371     package_info = []
 372     if okay:
 373         if changes.source is not None:
 374             package_info.append("source:{0}".format(changes.source.dsc['Source']))
 375         for binary in changes.binaries:
 376             package_info.append("binary:{0}".format(binary.control['Package']))
 377
 378     (prompt, answer) = ("", "XXX")
 379     if Options["No-Action"] or Options["Automatic"]:
 380         answer = 'S'
 381
 382     queuekey = ''
 383
 384     print summary
 385     print
 386     print "\n".join(package_info)
 387     print
 388     if len(upload.warnings) > 0:
 389         print "\n".join(upload.warnings)
 390         print
 391
 392     if len(upload.reject_reasons) > 0:
 393         print "Reason:"
 394         print "\n".join(upload.reject_reasons)
 395         print
 396
 397         path = os.path.join(directory, changes.filename)
 398         created = os.stat(path).st_mtime
 399         now = time.time()
 400         too_new = (now - created < int(cnf['Dinstall::SkipTime']))
 401
 402         if too_new:
 403             print "SKIP (too new)"
 404             prompt = "[S]kip, Quit ?"
 405         else:
 406             prompt = "[R]eject, Skip, Quit ?"
 407             if Options["Automatic"]:
 408                 answer = 'R'
 409     elif upload.new:
 410         prompt = "[N]ew, Skip, Quit ?"
 411         if Options['Automatic']:
 412             answer = 'N'
 413     else:
 414         prompt = "[A]ccept, Skip, Quit ?"
 415         if Options['Automatic']:
 416             answer = 'A'
 417
 418     while prompt.find(answer) == -1:
 419         answer = utils.our_raw_input(prompt)
 420         m = re_default_answer.match(prompt)
 421         if answer == "":
 422             answer = m.group(1)
 423         answer = answer[:1].upper()
 424
 425     if answer == 'R':
 426         reject(directory, upload)
 427     elif answer == 'A':
 428         # upload.try_autobyhand must not be run with No-Action.
 429         if Options['No-Action']:
 430             accept(directory, upload)
 431         elif upload.try_autobyhand():
 432             accept(directory, upload)
 433         else:
 434             print "W: redirecting to BYHAND as automatic processing failed."
 435             accept_to_new(directory, upload)
 436     elif answer == 'N':
 437         accept_to_new(directory, upload)
 438     elif answer == 'Q':
 439         sys.exit(0)
 440     elif answer == 'S':
 441         processed = False
 442
 443     if not Options['No-Action']:
 444         upload.commit()
 445
 446     return processed
 447
 448 ###############################################################################
 449
 450 def unlink_if_exists(path):
 451     try:
 452         os.unlink(path)
 453     except OSError as e:
 454         if e.errno != errno.ENOENT:
 455             raise
 456
 457 def process_it(directory, changes, keyrings, session):
 458     global Logger
 459
 460     print "\n{0}\n".format(changes.filename)
 461     Logger.log(["Processing changes file", changes.filename])
 462
 463     with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
 464         processed = action(directory, upload)
 465         if processed and not Options['No-Action']:
 466             unlink_if_exists(os.path.join(directory, changes.filename))
 467             for fn in changes.files:
 468                 unlink_if_exists(os.path.join(directory, fn))
 469
 470 ###############################################################################
 471
 472 def process_changes(changes_filenames):
 473     session = DBConn().session()
 474     keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
 475     keyring_files = [ k.keyring_name for k in keyrings ]
 476
 477     changes = []
 478     for fn in changes_filenames:
 479         try:
 480             directory, filename = os.path.split(fn)
 481             c = daklib.upload.Changes(directory, filename, keyring_files)
 482             changes.append([directory, c])
 483         except Exception as e:
 484             Logger.log([filename, "Error while loading changes: {0}".format(e)])
 485
 486     changes.sort(key=lambda x: x[1])
 487
 488     for directory, c in changes:
 489         process_it(directory, c, keyring_files, session)
 490
 491     session.rollback()
 492
 493 ###############################################################################
 494
 495 def main():
 496     global Options, Logger
 497
 498     cnf = Config()
 499     summarystats = SummaryStats()
 500
 501     Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
 502                  ('h',"help","Dinstall::Options::Help"),
 503                  ('n',"no-action","Dinstall::Options::No-Action"),
 504                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
 505                  ('s',"no-mail", "Dinstall::Options::No-Mail"),
 506                  ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
 507
 508     for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
 509               "version", "directory"]:
 510         if not cnf.has_key("Dinstall::Options::%s" % (i)):
 511             cnf["Dinstall::Options::%s" % (i)] = ""
 512
 513     changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
 514     Options = cnf.subtree("Dinstall::Options")
 515
 516     if Options["Help"]:
 517         usage()
 518
 519     # -n/--dry-run invalidates some other options which would involve things happening
 520     if Options["No-Action"]:
 521         Options["Automatic"] = ""
 522
 523     # Check that we aren't going to clash with the daily cron job
 524     if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
 525         utils.fubar("Archive maintenance in progress.  Try again later.")
 526
 527     # Obtain lock if not in no-action mode and initialize the log
 528     if not Options["No-Action"]:
 529         lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
 530         try:
 531             fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 532         except IOError as e:
 533             if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
 534                 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
 535             else:
 536                 raise
 537
 538         # Initialise UrgencyLog() - it will deal with the case where we don't
 539         # want to log urgencies
 540         urgencylog = UrgencyLog()
 541
 542     Logger = daklog.Logger("process-upload", Options["No-Action"])
 543
 544     # If we have a directory flag, use it to find our files
 545     if cnf["Dinstall::Options::Directory"] != "":
 546         # Note that we clobber the list of files we were given in this case
 547         # so warn if the user has done both
 548         if len(changes_files) > 0:
 549             utils.warn("Directory provided so ignoring files given on command line")
 550
 551         changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
 552         Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
 553     elif not len(changes_files) > 0:
 554         utils.fubar("No changes files given and no directory specified")
 555     else:
 556         Logger.log(["Using changes files from command-line", len(changes_files)])
 557
 558     process_changes(changes_files)
 559
 560     if summarystats.accept_count:
 561         sets = "set"
 562         if summarystats.accept_count > 1:
 563             sets = "sets"
 564         print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
 565                                                 utils.size_type(int(summarystats.accept_bytes)))
 566         Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
 567
 568     if summarystats.reject_count:
 569         sets = "set"
 570         if summarystats.reject_count > 1:
 571             sets = "sets"
 572         print "Rejected %d package %s." % (summarystats.reject_count, sets)
 573         Logger.log(["rejected", summarystats.reject_count])
 574
 575     if not Options["No-Action"]:
 576         urgencylog.close()
 577
 578     Logger.close()
 579
 580 ###############################################################################
 581
 582 if __name__ == '__main__':
 583     main()