]> git.decadent.org.uk Git - dak.git/blob - dak/process_upload.py
send announcement only for sourceful uploads
[dak.git] / dak / process_upload.py
1 #!/usr/bin/env python
2
3 """
4 Checks Debian packages from Incoming
5 @contact: Debian FTP Master <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
7 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
8 @copyright: 2009  Mark Hymers <mhy@debian.org>
9 @copyright: 2009  Frank Lichtenheld <djpig@debian.org>
10 @license: GNU General Public License version 2 or later
11 """
12
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
17
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21 # GNU General Public License for more details.
22
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26
27 # based on process-unchecked and process-accepted
28
29 ## pu|pa: locking (daily.lock)
30 ## pu|pa: parse arguments -> list of changes files
31 ## pa: initialize urgency log
32 ## pu|pa: sort changes list
33
34 ## foreach changes:
35 ###  pa: load dak file
36 ##   pu: copy CHG to tempdir
37 ##   pu: check CHG signature
38 ##   pu: parse changes file
39 ##   pu: checks:
40 ##     pu: check distribution (mappings, rejects)
41 ##     pu: copy FILES to tempdir
42 ##     pu: check whether CHG already exists in CopyChanges
43 ##     pu: check whether FILES already exist in one of the policy queues
44 ##     for deb in FILES:
45 ##       pu: extract control information
46 ##       pu: various checks on control information
47 ##       pu|pa: search for source (in CHG, projectb, policy queues)
48 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
49 ##       pu|pa: check whether deb already exists in the pool
50 ##     for src in FILES:
51 ##       pu: various checks on filenames and CHG consistency
52 ##       pu: if isdsc: check signature
53 ##     for file in FILES:
54 ##       pu: various checks
55 ##       pu: NEW?
56 ##       //pu: check whether file already exists in the pool
57 ##       pu: store what "Component" the package is currently in
58 ##     pu: check whether we found everything we were looking for in CHG
59 ##     pu: check the DSC:
60 ##       pu: check whether we need and have ONE DSC
61 ##       pu: parse the DSC
62 ##       pu: various checks //maybe drop some of the in favor of lintian
63 ##       pu|pa: check whether "Version" fulfills target suite requirements/suite propagation
64 ##       pu: check whether DSC_FILES is consistent with "Format"
65 ##       for src in DSC_FILES:
66 ##         pu|pa: check whether file already exists in the pool (with special handling for .orig.tar.gz)
67 ##     pu: create new tempdir
68 ##     pu: create symlink mirror of source
69 ##     pu: unpack source
70 ##     pu: extract changelog information for BTS
71 ##     //pu: create missing .orig symlink
72 ##     pu: check with lintian
73 ##     for file in FILES:
74 ##       pu: check checksums and sizes
75 ##     for file in DSC_FILES:
76 ##       pu: check checksums and sizes
77 ##     pu: CHG: check urgency
78 ##     for deb in FILES:
79 ##       pu: extract contents list and check for dubious timestamps
80 ##     pu: check that the uploader is actually allowed to upload the package
81 ###  pa: install:
82 ###    if stable_install:
83 ###      pa: remove from p-u
84 ###      pa: add to stable
85 ###      pa: move CHG to morgue
86 ###      pa: append data to ChangeLog
87 ###      pa: send mail
88 ###      pa: remove .dak file
89 ###    else:
90 ###      pa: add dsc to db:
91 ###        for file in DSC_FILES:
92 ###          pa: add file to file
93 ###          pa: add file to dsc_files
94 ###        pa: create source entry
95 ###        pa: update source associations
96 ###        pa: update src_uploaders
97 ###      for deb in FILES:
98 ###        pa: add deb to db:
99 ###          pa: add file to file
100 ###          pa: find source entry
101 ###          pa: create binaries entry
102 ###          pa: update binary associations
103 ###      pa: .orig component move
104 ###      pa: move files to pool
105 ###      pa: save CHG
106 ###      pa: move CHG to done/
107 ###      pa: change entry in queue_build
108 ##   pu: use dispatch table to choose target queue:
109 ##     if NEW:
110 ##       pu: write .dak file
111 ##       pu: move to NEW
112 ##       pu: send mail
113 ##     elsif AUTOBYHAND:
114 ##       pu: run autobyhand script
115 ##       pu: if stuff left, do byhand or accept
116 ##     elsif targetqueue in (oldstable, stable, embargo, unembargo):
117 ##       pu: write .dak file
118 ##       pu: check overrides
119 ##       pu: move to queue
120 ##       pu: send mail
121 ##     else:
122 ##       pu: write .dak file
123 ##       pu: move to ACCEPTED
124 ##       pu: send mails
125 ##       pu: create files for BTS
126 ##       pu: create entry in queue_build
127 ##       pu: check overrides
128
129 # Integrity checks
130 ## GPG
131 ## Parsing changes (check for duplicates)
132 ## Parse dsc
133 ## file list checks
134
135 # New check layout (TODO: Implement)
136 ## Permission checks
137 ### suite mappings
138 ### ACLs
139 ### version checks (suite)
140 ### override checks
141
142 ## Source checks
143 ### copy orig
144 ### unpack
145 ### BTS changelog
146 ### src contents
147 ### lintian
148 ### urgency log
149
150 ## Binary checks
151 ### timestamps
152 ### control checks
153 ### src relation check
154 ### contents
155
156 ## Database insertion (? copy from stuff)
157 ### BYHAND / NEW / Policy queues
158 ### Pool
159
160 ## Queue builds
161
162 import datetime
163 import errno
164 from errno import EACCES, EAGAIN
165 import fcntl
166 import os
167 import sys
168 import traceback
169 import apt_pkg
170 import time
171 from sqlalchemy.orm.exc import NoResultFound
172
173 from daklib import daklog
174 from daklib.dbconn import *
175 from daklib.urgencylog import UrgencyLog
176 from daklib.summarystats import SummaryStats
177 from daklib.config import Config
178 import daklib.utils as utils
179 from daklib.textutils import fix_maintainer
180 from daklib.regexes import *
181
182 import daklib.archive
183 import daklib.upload
184
185 ###############################################################################
186
187 Options = None
188 Logger = None
189
190 ###############################################################################
191
192 def usage (exit_code=0):
193     print """Usage: dak process-upload [OPTION]... [CHANGES]...
194   -a, --automatic           automatic run
195   -d, --directory <DIR>     process uploads in <DIR>
196   -h, --help                show this help and exit.
197   -n, --no-action           don't do anything
198   -p, --no-lock             don't check lockfile !! for cron.daily only !!
199   -s, --no-mail             don't send any mail
200   -V, --version             display the version number and exit"""
201     sys.exit(exit_code)
202
203 ###############################################################################
204
205 def try_or_reject(function):
206     """Try to call function or reject the upload if that fails
207     """
208     def wrapper(directory, upload, *args, **kwargs):
209         try:
210             return function(directory, upload, *args, **kwargs)
211         except Exception as e:
212             try:
213                 reason = "There was an uncaught exception when processing your upload:\n{0}\nAny original reject reason follows below.".format(traceback.format_exc())
214                 upload.rollback()
215                 return real_reject(directory, upload, reason=reason)
216             except Exception as e:
217                 reason = "In addition there was an exception when rejecting the package:\n{0}\nPrevious reasons:\n{1}".format(traceback.format_exc(), reason)
218                 upload.rollback()
219                 return real_reject(directory, upload, reason=reason, notify=False)
220
221     return wrapper
222
223 def subst_for_upload(upload):
224     cnf = Config()
225
226     changes = upload.changes
227     control = upload.changes.changes
228
229     if upload.final_suites is None or len(upload.final_suites) == 0:
230         suite_name = '(unknown)'
231     else:
232         suite_names = []
233         for suite in upload.final_suites:
234             if suite.policy_queue:
235                 suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name))
236             else:
237                 suite_names.append(suite.suite_name)
238         suite_name = ','.join(suite_names)
239
240     maintainer_field = control.get('Maintainer', cnf['Dinstall::MyEmailAddress'])
241     changed_by_field = control.get('Changed-By', maintainer_field)
242     maintainer = fix_maintainer(changed_by_field)
243     if upload.changes.source is not None:
244         addresses = utils.mail_addresses_for_upload(maintainer_field, changed_by_field, changes.primary_fingerprint)
245     else:
246         addresses = utils.mail_addresses_for_upload(maintainer_field, maintainer_field, changes.primary_fingerprint)
247
248     bcc = 'X-DAK: dak process-upload'
249     if 'Dinstall::Bcc' in cnf:
250         bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
251
252     subst = {
253         '__DISTRO__': cnf['Dinstall::MyDistribution'],
254         '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'],
255
256         '__CHANGES_FILENAME__': upload.changes.filename,
257
258         '__SOURCE__': control.get('Source', '(unknown)'),
259         '__ARCHITECTURE__': control.get('Architecture', '(unknown)'),
260         '__VERSION__': control.get('Version', '(unknown)'),
261
262         '__SUITE__': suite_name,
263
264         '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
265         '__MAINTAINER_FROM__': maintainer[1],
266         '__MAINTAINER_TO__': ", ".join(addresses),
267         '__MAINTAINER__': changed_by_field,
268         '__BCC__': bcc,
269
270         '__BUG_SERVER__': cnf.get('Dinstall::BugServer'),
271
272         '__FILE_CONTENTS__': open(upload.changes.path, 'r').read(),
273         }
274
275     override_maintainer = cnf.get('Dinstall::OverrideMaintainer')
276     if override_maintainer:
277         subst['__MAINTAINER_TO__'] = subst['__MAINTAINER_FROM__'] = override_maintainer
278
279     return subst
280
281 @try_or_reject
282 def accept(directory, upload):
283     cnf = Config()
284
285     Logger.log(['ACCEPT', upload.changes.filename])
286
287     upload.install()
288
289     accepted_to_real_suite = False
290     for suite in upload.final_suites:
291         accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
292
293     sourceful_upload = 'source' in upload.changes.architectures
294
295     control = upload.changes.changes
296     if sourceful_upload and not Options['No-Action']:
297         urgency = control.get('Urgency')
298         if urgency not in cnf.value_list('Urgency::Valid'):
299             urgency = cnf['Urgency::Default']
300         UrgencyLog().log(control['Source'], control['Version'], urgency)
301
302     # send mail to maintainer
303     subst = subst_for_upload(upload)
304     message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted'))
305     utils.send_mail(message)
306
307     # send mail to announce lists and tracking server
308     if accepted_to_real_suite and sourceful_upload:
309         subst = subst_for_upload(upload)
310         announce = set()
311         for suite in upload.final_suites:
312             if suite.policy_queue is not None:
313                 continue
314             announce.update(suite.announce or [])
315         announce_address = ", ".join(announce)
316
317         tracking = cnf.get('Dinstall::TrackingServer')
318         if tracking and 'source' in upload.changes.architectures:
319             announce_address = '{0}\nBcc: {1}@{2}'.format(announce_address, control['Source'], tracking)
320
321         subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_address
322
323         message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce'))
324         utils.send_mail(message)
325
326     # Only close bugs for uploads that were not redirected to a policy queue.
327     # process-policy will close bugs for those once they are accepted.
328     subst = subst_for_upload(upload)
329     if accepted_to_real_suite and cnf.find_b('Dinstall::CloseBugs') and sourceful_upload:
330         for bugnum in upload.changes.closed_bugs:
331             subst['__BUG_NUMBER__'] = str(bugnum)
332
333             message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close'))
334             utils.send_mail(message)
335
336             del subst['__BUG_NUMBER__']
337
338     # Move .changes to done, but only for uploads that were accepted to a
339     # real suite.  process-policy will handle this for uploads to queues.
340     if accepted_to_real_suite:
341         src = os.path.join(upload.directory, upload.changes.filename)
342
343         now = datetime.datetime.now()
344         donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
345         dst = os.path.join(donedir, upload.changes.filename)
346         dst = utils.find_next_free(dst)
347
348         upload.transaction.fs.copy(src, dst, mode=0o644)
349
350     SummaryStats().accept_count += 1
351     SummaryStats().accept_bytes += upload.changes.bytes
352
353 @try_or_reject
354 def accept_to_new(directory, upload):
355     cnf = Config()
356
357     Logger.log(['ACCEPT-TO-NEW', upload.changes.filename])
358
359     upload.install_to_new()
360     # TODO: tag bugs pending, send announcement
361
362     subst = subst_for_upload(upload)
363     message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
364     utils.send_mail(message)
365
366     SummaryStats().accept_count += 1
367     SummaryStats().accept_bytes += upload.changes.bytes
368
369 @try_or_reject
370 def reject(directory, upload, reason=None, notify=True):
371     real_reject(directory, upload, reason, notify)
372
373 def real_reject(directory, upload, reason=None, notify=True):
374     # XXX: rejection itself should go to daklib.archive.ArchiveUpload
375     cnf = Config()
376
377     Logger.log(['REJECT', upload.changes.filename])
378
379     fs = upload.transaction.fs
380     rejectdir = cnf['Dir::Reject']
381
382     files = [ f.filename for f in upload.changes.files.itervalues() ]
383     files.append(upload.changes.filename)
384
385     for fn in files:
386         src = os.path.join(upload.directory, fn)
387         dst = utils.find_next_free(os.path.join(rejectdir, fn))
388         if not os.path.exists(src):
389             continue
390         fs.copy(src, dst)
391
392     if upload.reject_reasons is not None:
393         if reason is None:
394             reason = ''
395         reason = reason + '\n' + '\n'.join(upload.reject_reasons)
396
397     if reason is None:
398         reason = '(Unknown reason. Please check logs.)'
399
400     dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(upload.changes.filename)))
401     fh = fs.create(dst)
402     fh.write(reason)
403     fh.close()
404
405     # TODO: fix
406     if notify:
407         subst = subst_for_upload(upload)
408         subst['__REJECTOR_ADDRESS__'] = cnf['Dinstall::MyEmailAddress']
409         subst['__MANUAL_REJECT_MESSAGE__'] = ''
410         subst['__REJECT_MESSAGE__'] = reason
411         subst['__CC__'] = 'X-DAK-Rejection: automatic (moo)'
412
413         message = utils.TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'queue.rejected'))
414         utils.send_mail(message)
415
416     SummaryStats().reject_count += 1
417
418 ###############################################################################
419
420 def action(directory, upload):
421     changes = upload.changes
422     processed = True
423
424     global Logger
425
426     cnf = Config()
427
428     okay = upload.check()
429
430     summary = changes.changes.get('Changes', '')
431
432     package_info = []
433     if okay:
434         if changes.source is not None:
435             package_info.append("source:{0}".format(changes.source.dsc['Source']))
436         for binary in changes.binaries:
437             package_info.append("binary:{0}".format(binary.control['Package']))
438
439     (prompt, answer) = ("", "XXX")
440     if Options["No-Action"] or Options["Automatic"]:
441         answer = 'S'
442
443     queuekey = ''
444
445     print summary
446     print
447     print "\n".join(package_info)
448     print
449
450     if len(upload.reject_reasons) > 0:
451         print "Reason:"
452         print "\n".join(upload.reject_reasons)
453         print
454
455         path = os.path.join(directory, changes.filename)
456         created = os.stat(path).st_mtime
457         now = time.time()
458         too_new = (now - created < int(cnf['Dinstall::SkipTime']))
459
460         if too_new:
461             print "SKIP (too new)"
462             prompt = "[S]kip, Quit ?"
463         else:
464             prompt = "[R]eject, Skip, Quit ?"
465             if Options["Automatic"]:
466                 answer = 'R'
467     elif upload.new:
468         prompt = "[N]ew, Skip, Quit ?"
469         if Options['Automatic']:
470             answer = 'N'
471     else:
472         prompt = "[A]ccept, Skip, Quit ?"
473         if Options['Automatic']:
474             answer = 'A'
475
476     while prompt.find(answer) == -1:
477         answer = utils.our_raw_input(prompt)
478         m = re_default_answer.match(prompt)
479         if answer == "":
480             answer = m.group(1)
481         answer = answer[:1].upper()
482
483     if answer == 'R':
484         reject(directory, upload)
485     elif answer == 'A':
486         # upload.try_autobyhand must not be run with No-Action.
487         if Options['No-Action']:
488             accept(directory, upload)
489         elif upload.try_autobyhand():
490             accept(directory, upload)
491         else:
492             print "W: redirecting to BYHAND as automatic processing failed."
493             accept_to_new(directory, upload)
494     elif answer == 'N':
495         accept_to_new(directory, upload)
496     elif answer == 'Q':
497         sys.exit(0)
498     elif answer == 'S':
499         processed = False
500
501     #raise Exception("FAIL")
502     if not Options['No-Action']:
503         upload.commit()
504
505     return processed
506
507 ###############################################################################
508
509 def unlink_if_exists(path):
510     try:
511         os.unlink(path)
512     except OSError as e:
513         if e.errno != errno.ENOENT:
514             raise
515
516 def process_it(directory, changes, keyrings, session):
517     global Logger
518
519     print "\n{0}\n".format(changes.filename)
520     Logger.log(["Processing changes file", changes.filename])
521
522     cnf = Config()
523
524     # Some defaults in case we can't fully process the .changes file
525     #u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
526     #u.pkg.changes["changedby2047"] = cnf["Dinstall::MyEmailAddress"]
527
528     # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header
529     bcc = "X-DAK: dak process-upload"
530     #if cnf.has_key("Dinstall::Bcc"):
531     #    u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"])
532     #else:
533     #    u.Subst["__BCC__"] = bcc
534
535     with daklib.archive.ArchiveUpload(directory, changes, keyrings) as upload:
536         processed = action(directory, upload)
537         if processed and not Options['No-Action']:
538             unlink_if_exists(os.path.join(directory, changes.filename))
539             for fn in changes.files:
540                 unlink_if_exists(os.path.join(directory, fn))
541
542 ###############################################################################
543
544 def process_changes(changes_filenames):
545     session = DBConn().session()
546     keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
547     keyring_files = [ k.keyring_name for k in keyrings ]
548
549     changes = []
550     for fn in changes_filenames:
551         try:
552             directory, filename = os.path.split(fn)
553             c = daklib.upload.Changes(directory, filename, keyring_files)
554             changes.append([directory, c])
555         except Exception as e:
556             Logger.log([filename, "Error while loading changes: {0}".format(e)])
557
558     changes.sort(key=lambda x: x[1])
559
560     for directory, c in changes:
561         process_it(directory, c, keyring_files, session)
562
563     session.rollback()
564
565 ###############################################################################
566
567 def main():
568     global Options, Logger
569
570     cnf = Config()
571     summarystats = SummaryStats()
572
573     Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
574                  ('h',"help","Dinstall::Options::Help"),
575                  ('n',"no-action","Dinstall::Options::No-Action"),
576                  ('p',"no-lock", "Dinstall::Options::No-Lock"),
577                  ('s',"no-mail", "Dinstall::Options::No-Mail"),
578                  ('d',"directory", "Dinstall::Options::Directory", "HasArg")]
579
580     for i in ["automatic", "help", "no-action", "no-lock", "no-mail",
581               "version", "directory"]:
582         if not cnf.has_key("Dinstall::Options::%s" % (i)):
583             cnf["Dinstall::Options::%s" % (i)] = ""
584
585     changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
586     Options = cnf.subtree("Dinstall::Options")
587
588     if Options["Help"]:
589         usage()
590
591     # -n/--dry-run invalidates some other options which would involve things happening
592     if Options["No-Action"]:
593         Options["Automatic"] = ""
594
595     # Check that we aren't going to clash with the daily cron job
596     if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]:
597         utils.fubar("Archive maintenance in progress.  Try again later.")
598
599     # Obtain lock if not in no-action mode and initialize the log
600     if not Options["No-Action"]:
601         lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
602         try:
603             fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
604         except IOError as e:
605             if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
606                 utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
607             else:
608                 raise
609
610         # Initialise UrgencyLog() - it will deal with the case where we don't
611         # want to log urgencies
612         urgencylog = UrgencyLog()
613
614     Logger = daklog.Logger("process-upload", Options["No-Action"])
615
616     # If we have a directory flag, use it to find our files
617     if cnf["Dinstall::Options::Directory"] != "":
618         # Note that we clobber the list of files we were given in this case
619         # so warn if the user has done both
620         if len(changes_files) > 0:
621             utils.warn("Directory provided so ignoring files given on command line")
622
623         changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"])
624         Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)])
625     elif not len(changes_files) > 0:
626         utils.fubar("No changes files given and no directory specified")
627     else:
628         Logger.log(["Using changes files from command-line", len(changes_files)])
629
630     process_changes(changes_files)
631
632     if summarystats.accept_count:
633         sets = "set"
634         if summarystats.accept_count > 1:
635             sets = "sets"
636         print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
637                                                 utils.size_type(int(summarystats.accept_bytes)))
638         Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
639
640     if summarystats.reject_count:
641         sets = "set"
642         if summarystats.reject_count > 1:
643             sets = "sets"
644         print "Rejected %d package %s." % (summarystats.reject_count, sets)
645         Logger.log(["rejected", summarystats.reject_count])
646
647     if not Options["No-Action"]:
648         urgencylog.close()
649
650     Logger.close()
651
652 ###############################################################################
653
654 if __name__ == '__main__':
655     main()