]> git.decadent.org.uk Git - dak.git/blob - dak/new_security_install.py
Merge commit 'sec-merge' into security
[dak.git] / dak / new_security_install.py
1 #!/usr/bin/env python
2
3 # Wrapper for Debian Security team
4 # Copyright (C) 2006  Anthony Towns <ajt@debian.org>
5
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 # General Public License for more details.
15
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19 # USA
20
21 ################################################################################
22
23 import daklib.queue as queue
24 import daklib.logging as logging
25 import daklib.utils as utils
26 import daklib.database as database
27 import apt_pkg, os, sys, pwd, time, re, commands
28
29 re_taint_free = re.compile(r"^['/;\-\+\.~\s\w]+$");
30
31 Cnf = None
32 Options = None
33 Upload = None
34 Logger = None
35
36 advisory = None
37 changes = []
38 srcverarches = {}
39
40 def init():
41     global Cnf, Upload, Options, Logger
42
43     Cnf = utils.get_conf()
44     Cnf["Dinstall::Options::No-Mail"] = "y"
45     Arguments = [('h', "help", "Security-Install::Options::Help"),
46                  ('a', "automatic", "Security-Install::Options::Automatic"),
47                  ('n', "no-action", "Security-Install::Options::No-Action"),
48                  ('s', "sudo", "Security-Install::Options::Sudo"),
49                  (' ', "no-upload", "Security-Install::Options::No-Upload"),
50                  ('u', "fg-upload", "Security-Install::Options::Foreground-Upload"),
51                  (' ', "drop-advisory", "Security-Install::Options::Drop-Advisory"),
52                  ('A', "approve", "Security-Install::Options::Approve"),
53                  ('R', "reject", "Security-Install::Options::Reject"),
54                  ('D', "disembargo", "Security-Install::Options::Disembargo") ]
55
56     for i in Arguments:
57         Cnf[i[2]] = ""
58
59     arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
60
61     Options = Cnf.SubTree("Security-Install::Options")
62
63     whoami = os.getuid()
64     whoamifull = pwd.getpwuid(whoami)
65     username = whoamifull[0]
66     if username != "dak":
67         print "Non-dak user: %s" % username
68         Options["Sudo"] = "y"
69
70     if Options["Help"]:
71         print "help yourself"
72         sys.exit(0)
73
74     if len(arguments) == 0:
75         utils.fubar("Process what?")
76
77     Upload = queue.Upload(Cnf)
78     if Options["No-Action"]:
79         Options["Sudo"] = ""
80     if not Options["Sudo"] and not Options["No-Action"]:
81         Logger = Upload.Logger = logging.Logger(Cnf, "new-security-install")
82
83     return arguments
84
85 def quit():
86     if Logger:
87         Logger.close()
88     sys.exit(0)
89
90 def load_args(arguments):
91     global advisory, changes
92
93     adv_ids = {}
94     if not arguments[0].endswith(".changes"):
95         adv_ids [arguments[0]] = 1
96         arguments = arguments[1:]
97
98     null_adv_changes = []
99
100     changesfiles = {}
101     for a in arguments:
102         if "/" in a:
103             utils.fubar("can only deal with files in the current directory")
104         if not a.endswith(".changes"):
105             utils.fubar("not a .changes file: %s" % (a))
106         Upload.init_vars()
107         Upload.pkg.changes_file = a
108         Upload.update_vars()
109         if "adv id" in Upload.pkg.changes:
110             changesfiles[a] = 1
111             adv_ids[Upload.pkg.changes["adv id"]] = 1
112         else:
113             null_adv_changes.append(a)
114
115     adv_ids = adv_ids.keys()
116     if len(adv_ids) > 1:
117         utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids)))
118     if adv_ids == []:
119         advisory = None
120     else:
121         advisory = adv_ids[0]
122
123     changes = changesfiles.keys()
124     return null_adv_changes
125
126 def load_adv_changes():
127     global srcverarches, changes
128
129     for c in os.listdir("."):
130         if not c.endswith(".changes"): continue
131         Upload.init_vars()
132         Upload.pkg.changes_file = c
133         Upload.update_vars()
134         if "adv id" not in Upload.pkg.changes:
135             continue
136         if Upload.pkg.changes["adv id"] != advisory:
137             continue
138
139         if c not in changes: changes.append(c)
140         srcver = "%s %s" % (Upload.pkg.changes["source"],
141                             Upload.pkg.changes["version"])
142         srcverarches.setdefault(srcver, {})
143         for arch in Upload.pkg.changes["architecture"].keys():
144             srcverarches[srcver][arch] = 1
145
146 def advisory_info():
147     if advisory != None:
148         print "Advisory: %s" % (advisory)
149     print "Changes:"
150     for c in changes:
151         print " %s" % (c)
152
153     print "Packages:"
154     svs = srcverarches.keys()
155     svs.sort()
156     for sv in svs:
157         as = srcverarches[sv].keys()
158         as.sort()
159         print " %s (%s)" % (sv, ", ".join(as))
160
161 def prompt(opts, default):
162     p = ""
163     v = {}
164     for o in opts:
165         v[o[0].upper()] = o
166         if o[0] == default:
167             p += ", [%s]%s" % (o[0], o[1:])
168         else:
169             p += ", " + o
170     p = p[2:] + "? "
171     a = None
172
173     if Options["Automatic"]:
174         a = default
175
176     while a not in v:
177         a = utils.our_raw_input(p) + default
178         a = a[:1].upper()
179
180     return v[a]
181
182 def add_changes(extras):
183     for c in extras:
184         changes.append(c)
185         Upload.init_vars()
186         Upload.pkg.changes_file = c
187         Upload.update_vars()
188         srcver = "%s %s" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])
189         srcverarches.setdefault(srcver, {})
190         for arch in Upload.pkg.changes["architecture"].keys():
191             srcverarches[srcver][arch] = 1
192         Upload.pkg.changes["adv id"] = advisory
193         Upload.dump_vars(os.getcwd())
194
195 def yes_no(prompt):
196     if Options["Automatic"]: return True
197     while 1:
198         answer = utils.our_raw_input(prompt + " ").lower()
199         if answer in "yn":
200             return answer == "y"
201         print "Invalid answer; please try again."
202
203 def do_upload():
204     if Options["No-Upload"]:
205         print "Not uploading as requested"
206     elif Options["Foreground-Upload"]:
207         actually_upload(changes)
208     else:
209         child = os.fork()
210         if child == 0:
211             actually_upload(changes)
212             os._exit(0)
213         print "Uploading in the background"
214
215 def actually_upload(changes_files):
216     file_list = ""
217     suites = {}
218     component_mapping = {}
219     for component in Cnf.SubTree("Security-Install::ComponentMappings").List():
220         component_mapping[component] = Cnf["Security-Install::ComponentMappings::%s" % (component)]
221     uploads = {}; # uploads[uri] = file_list
222     changesfiles = {}; # changesfiles[uri] = file_list
223     package_list = {} # package_list[source_name][version]
224     changes_files.sort(utils.changes_compare)
225     for changes_file in changes_files:
226         changes_file = utils.validate_changes_file_arg(changes_file)
227         # Reset variables
228         components = {}
229         upload_uris = {}
230         file_list = []
231         Upload.init_vars()
232         # Parse the .dak file for the .changes file
233         Upload.pkg.changes_file = changes_file
234         Upload.update_vars()
235         files = Upload.pkg.files
236         changes = Upload.pkg.changes
237         dsc = Upload.pkg.dsc
238         # We have the changes, now return if its amd64, to not upload them to ftp-master
239         if changes["distribution"].has_key("oldstable-security") and changes["architecture"].has_key("amd64"):
240             print "Not uploading amd64 oldstable-security changes to ftp-master\n"
241             continue
242         # Build the file list for this .changes file
243         for file in files.keys():
244             poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"],
245                                     utils.poolify(changes["source"], files[file]["component"]),
246                                     file)
247             file_list.append(poolname)
248             orig_component = files[file].get("original component", files[file]["component"])
249             components[orig_component] = ""
250         # Determine the upload uri for this .changes file
251         for component in components.keys():
252             upload_uri = component_mapping.get(component)
253             if upload_uri:
254                 upload_uris[upload_uri] = ""
255         num_upload_uris = len(upload_uris.keys())
256         if num_upload_uris == 0:
257             utils.fubar("%s: No valid upload URI found from components (%s)."
258                         % (changes_file, ", ".join(components.keys())))
259         elif num_upload_uris > 1:
260             utils.fubar("%s: more than one upload URI (%s) from components (%s)."
261                         % (changes_file, ", ".join(upload_uris.keys()),
262                            ", ".join(components.keys())))
263         upload_uri = upload_uris.keys()[0]
264         # Update the file list for the upload uri
265         if not uploads.has_key(upload_uri):
266             uploads[upload_uri] = []
267         uploads[upload_uri].extend(file_list)
268         # Update the changes list for the upload uri
269         if not changesfiles.has_key(upload_uri):
270             changesfiles[upload_uri] = []
271         changesfiles[upload_uri].append(changes_file)
272         # Remember the suites and source name/version
273         for suite in changes["distribution"].keys():
274             suites[suite] = ""
275         # Remember the source name and version
276         if changes["architecture"].has_key("source") and \
277            changes["distribution"].has_key("testing"):
278             if not package_list.has_key(dsc["source"]):
279                 package_list[dsc["source"]] = {}
280             package_list[dsc["source"]][dsc["version"]] = ""
281
282     for uri in uploads.keys():
283         uploads[uri].extend(changesfiles[uri])
284         (host, path) = uri.split(":")
285         file_list = " ".join(uploads[uri])
286         print "Uploading files to %s..." % (host)
287         spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list))
288
289     if not Options["No-Action"]:
290         filename = "%s/testing-processed" % (Cnf["Dir::Log"])
291         file = utils.open_file(filename, 'a')
292         for source in package_list.keys():
293             for version in package_list[source].keys():
294                 file.write(" ".join([source, version])+'\n')
295         file.close()
296
297 def generate_advisory(template):
298     global changes, advisory
299
300     adv_packages = []
301     updated_pkgs = {};  # updated_pkgs[distro][arch][file] = {path,md5,size}
302
303     for arg in changes:
304         arg = utils.validate_changes_file_arg(arg)
305         Upload.pkg.changes_file = arg
306         Upload.init_vars()
307         Upload.update_vars()
308
309         src = Upload.pkg.changes["source"]
310         src_ver = "%s (%s)" % (src, Upload.pkg.changes["version"])
311         if src_ver not in adv_packages:
312             adv_packages.append(src_ver)
313
314         suites = Upload.pkg.changes["distribution"].keys()
315         for suite in suites:
316             if not updated_pkgs.has_key(suite):
317                 updated_pkgs[suite] = {}
318
319         files = Upload.pkg.files
320         for file in files.keys():
321             arch = files[file]["architecture"]
322             md5 = files[file]["md5sum"]
323             size = files[file]["size"]
324             poolname = Cnf["Dir::PoolRoot"] + \
325                 utils.poolify(src, files[file]["component"])
326             if arch == "source" and file.endswith(".dsc"):
327                 dscpoolname = poolname
328             for suite in suites:
329                 if not updated_pkgs[suite].has_key(arch):
330                     updated_pkgs[suite][arch] = {}
331                 updated_pkgs[suite][arch][file] = {
332                     "md5": md5, "size": size, "poolname": poolname }
333
334         dsc_files = Upload.pkg.dsc_files
335         for file in dsc_files.keys():
336             arch = "source"
337             if not dsc_files[file].has_key("files id"):
338                 continue
339
340             # otherwise, it's already in the pool and needs to be
341             # listed specially
342             md5 = dsc_files[file]["md5sum"]
343             size = dsc_files[file]["size"]
344             for suite in suites:
345                 if not updated_pkgs[suite].has_key(arch):
346                     updated_pkgs[suite][arch] = {}
347                 updated_pkgs[suite][arch][file] = {
348                     "md5": md5, "size": size, "poolname": dscpoolname }
349
350     if os.environ.has_key("SUDO_UID"):
351         whoami = long(os.environ["SUDO_UID"])
352     else:
353         whoami = os.getuid()
354     whoamifull = pwd.getpwuid(whoami)
355     username = whoamifull[4].split(",")[0]
356
357     Subst = {
358         "__ADVISORY__": advisory,
359         "__WHOAMI__": username,
360         "__DATE__": time.strftime("%B %d, %Y", time.gmtime(time.time())),
361         "__PACKAGE__": ", ".join(adv_packages),
362         "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"]
363         }
364
365     if Cnf.has_key("Dinstall::Bcc"):
366         Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"])
367
368     adv = ""
369     archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())]
370     for suite in updated_pkgs.keys():
371         ver = Cnf["Suite::%s::Version" % suite]
372         if ver != "": ver += " "
373         suite_header = "%s %s(%s)" % (Cnf["Dinstall::MyDistribution"],
374                                        ver, suite)
375         adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header))
376
377         arches = Cnf.ValueList("Suite::%s::Architectures" % suite)
378         if "source" in arches:
379             arches.remove("source")
380         if "all" in arches:
381             arches.remove("all")
382         arches.sort()
383
384         adv += "%s updates are available for %s.\n\n" % (
385                 suite.capitalize(), utils.join_with_commas_and(arches))
386
387         for a in ["source", "all"] + arches:
388             if not updated_pkgs[suite].has_key(a):
389                 continue
390
391             if a == "source":
392                 adv += "Source archives:\n\n"
393             elif a == "all":
394                 adv += "Architecture independent packages:\n\n"
395             else:
396                 adv += "%s architecture (%s)\n\n" % (a,
397                         Cnf["Architectures::%s" % a])
398
399             for file in updated_pkgs[suite][a].keys():
400                 adv += "  http://%s/%s%s\n" % (
401                                 archive, updated_pkgs[suite][a][file]["poolname"], file)
402                 adv += "    Size/MD5 checksum: %8s %s\n" % (
403                         updated_pkgs[suite][a][file]["size"],
404                         updated_pkgs[suite][a][file]["md5"])
405             adv += "\n"
406     adv = adv.rstrip()
407
408     Subst["__ADVISORY_TEXT__"] = adv
409
410     adv = utils.TemplateSubst(Subst, template)
411     return adv
412
413 def spawn(command):
414     if not re_taint_free.match(command):
415         utils.fubar("Invalid character in \"%s\"." % (command))
416
417     if Options["No-Action"]:
418         print "[%s]" % (command)
419     else:
420         (result, output) = commands.getstatusoutput(command)
421         if (result != 0):
422             utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result)
423
424
425 ##################### ! ! ! N O T E ! ! !  #####################
426 #
427 # These functions will be reinvoked by semi-priveleged users, be careful not
428 # to invoke external programs that will escalate privileges, etc.
429 #
430 ##################### ! ! ! N O T E ! ! !  #####################
431
432 def sudo(arg, fn, exit):
433     if Options["Sudo"]:
434         if advisory == None:
435             utils.fubar("Must set advisory name")
436         os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H",
437                   "/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory)
438     else:
439         fn()
440     if exit:
441         quit()
442
443 def do_Approve(): sudo("A", _do_Approve, True)
444 def _do_Approve():
445     # 1. dump advisory in drafts
446     draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory)
447     print "Advisory in %s" % (draft)
448     if not Options["No-Action"]:
449         adv_file = "./advisory.%s" % (advisory)
450         if not os.path.exists(adv_file):
451             adv_file = Cnf["Dir::Templates"]+"/security-install.advisory"
452         adv_fd = os.open(draft, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0664)
453         os.write(adv_fd, generate_advisory(adv_file))
454         os.close(adv_fd)
455         adv_fd = None
456
457     # 2. run dak process-accepted on changes
458     print "Accepting packages..."
459     spawn("dak process-accepted -pa %s" % (" ".join(changes)))
460
461     # 3. run dak make-suite-file-list / apt-ftparchve / dak generate-releases
462     print "Updating file lists for apt-ftparchive..."
463     spawn("dak make-suite-file-list")
464     print "Updating Packages and Sources files..."
465     spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file()))
466     print "Updating Release files..."
467     spawn("dak generate-releases")
468     print "Triggering security mirrors..."
469     spawn("sudo -u archvsync /home/archvsync/signal_security")
470
471     # 4. chdir to done - do upload
472     if not Options["No-Action"]:
473         os.chdir(Cnf["Dir::Queue::Done"])
474     do_upload()
475
476 def do_Disembargo(): sudo("D", _do_Disembargo, True)
477 def _do_Disembargo():
478     if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"):
479         utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"])
480
481     dest = Cnf["Dir::Queue::Unembargoed"]
482     emb_q = database.get_or_set_queue_id("embargoed")
483     une_q = database.get_or_set_queue_id("unembargoed")
484
485     for c in changes:
486         print "Disembargoing %s" % (c)
487
488         Upload.init_vars()
489         Upload.pkg.changes_file = c
490         Upload.update_vars()
491
492         if "source" in Upload.pkg.changes["architecture"].keys():
493             print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])
494             Upload.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]))
495
496         files = {}
497         for suite in Upload.pkg.changes["distribution"].keys():
498             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
499                 continue
500             dest_dir = Cnf["Dir::QueueBuild"]
501             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
502                 dest_dir = os.path.join(dest_dir, suite)
503             for file in Upload.pkg.files.keys():
504                 files[os.path.join(dest_dir, file)] = 1
505
506         files = files.keys()
507         Upload.projectB.query("BEGIN WORK")
508         for f in files:
509             Upload.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q))
510         Upload.projectB.query("COMMIT WORK")
511
512         for file in Upload.pkg.files.keys():
513             utils.copy(file, os.path.join(dest, file))
514             os.unlink(file)
515
516     for c in changes:
517         utils.copy(c, os.path.join(dest, c))
518         os.unlink(c)
519         k = c[:-8] + ".dak"
520         utils.copy(k, os.path.join(dest, k))
521         os.unlink(k)
522
523 def do_Reject(): sudo("R", _do_Reject, True)
524 def _do_Reject():
525     global changes
526     for c in changes:
527         print "Rejecting %s..." % (c)
528         Upload.init_vars()
529         Upload.pkg.changes_file = c
530         Upload.update_vars()
531         files = {}
532         for suite in Upload.pkg.changes["distribution"].keys():
533             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
534                 continue
535             dest_dir = Cnf["Dir::QueueBuild"]
536             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
537                 dest_dir = os.path.join(dest_dir, suite)
538             for file in Upload.pkg.files.keys():
539                 files[os.path.join(dest_dir, file)] = 1
540
541         files = files.keys()
542
543         aborted = Upload.do_reject()
544         if not aborted:
545             os.unlink(c[:-8]+".dak")
546             for f in files:
547                 Upload.projectB.query(
548                     "DELETE FROM queue_build WHERE filename = '%s'" % (f))
549                 os.unlink(f)
550
551     print "Updating buildd information..."
552     spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd")
553
554     adv_file = "./advisory.%s" % (advisory)
555     if os.path.exists(adv_file):
556         os.unlink(adv_file)
557
558 def do_DropAdvisory():
559     for c in changes:
560         Upload.init_vars()
561         Upload.pkg.changes_file = c
562         Upload.update_vars()
563         del Upload.pkg.changes["adv id"]
564         Upload.dump_vars(os.getcwd())
565     quit()
566
567 def do_Edit():
568     adv_file = "./advisory.%s" % (advisory)
569     if not os.path.exists(adv_file):
570         utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file)
571     editor = os.environ.get("EDITOR", "vi")
572     result = os.system("%s %s" % (editor, adv_file))
573     if result != 0:
574         utils.fubar("%s invocation failed for %s." % (editor, adv_file))
575
576 def do_Show():
577     adv_file = "./advisory.%s" % (advisory)
578     if not os.path.exists(adv_file):
579         adv_file = Cnf["Dir::Templates"]+"/security-install.advisory"
580     print "====\n%s\n====" % (generate_advisory(adv_file))
581
582 def do_Quit():
583     quit()
584
585 def main():
586     global changes
587
588     args = init()
589     extras = load_args(args)
590     if advisory:
591         load_adv_changes()
592     if extras:
593         if not advisory:
594             changes = extras
595         else:
596             if srcverarches == {}:
597                 if not yes_no("Create new advisory %s?" % (advisory)):
598                     print "Not doing anything, then"
599                     quit()
600             else:
601                 advisory_info()
602                 doextras = []
603                 for c in extras:
604                     if yes_no("Add %s to %s?" % (c, advisory)):
605                         doextras.append(c)
606                 extras = doextras
607             add_changes(extras)
608
609     if not advisory:
610         utils.fubar("Must specify an advisory id")
611
612     if not changes:
613         utils.fubar("No changes specified")
614
615     if Options["Approve"]:
616         advisory_info()
617         do_Approve()
618     elif Options["Reject"]:
619         advisory_info()
620         do_Reject()
621     elif Options["Disembargo"]:
622         advisory_info()
623         do_Disembargo()
624     elif Options["Drop-Advisory"]:
625         advisory_info()
626         do_DropAdvisory()
627     else:
628         while 1:
629             default = "Q"
630             opts = ["Approve", "Edit advisory"]
631             if os.path.exists("./advisory.%s" % advisory):
632                 default = "A"
633             else:
634                 default = "E"
635             if os.getcwd() == Cnf["Dir::Queue::Embargoed"].rstrip("/"):
636                 opts.append("Disembargo")
637             opts += ["Show advisory", "Reject", "Quit"]
638
639             advisory_info()
640             what = prompt(opts, default)
641
642             if what == "Quit":
643                 do_Quit()
644             elif what == "Approve":
645                 do_Approve()
646             elif what == "Edit advisory":
647                 do_Edit()
648             elif what == "Show advisory":
649                 do_Show()
650             elif what == "Disembargo":
651                 do_Disembargo()
652             elif what == "Reject":
653                 do_Reject()
654             else:
655                 utils.fubar("Impossible answer '%s', wtf?" % (what))
656
657 ################################################################################
658
659 if __name__ == '__main__':
660     main()
661
662 ################################################################################