]> git.decadent.org.uk Git - dak.git/blob - dak/new_security_install.py
new-security-install
[dak.git] / dak / new_security_install.py
1 #!/usr/bin/env python
2
3 # Wrapper for Debian Security team
4 # Copyright (C) 2006  Anthony Towns <ajt@debian.org>
5
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 # General Public License for more details.
15
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19 # USA
20
21 ################################################################################
22
23 from daklib import queue
24 from daklib import logging
25 from daklib import utils
26 from daklib import database
27 import apt_pkg, os, sys, pwd, time, re, commands
28
29 re_taint_free = re.compile(r"^['/;\-\+\.~\s\w]+$");
30
31 Cnf = None
32 Options = None
33 Upload = None
34 Logger = None
35
36 advisory = None
37 changes = []
38 srcverarches = {}
39
40 def init():
41     global Cnf, Upload, Options, Logger
42
43     Cnf = utils.get_conf()
44     Cnf["Dinstall::Options::No-Mail"] = "y"
45     Arguments = [('h', "help", "Security-Install::Options::Help"),
46                  ('a', "automatic", "Security-Install::Options::Automatic"),
47                  ('n', "no-action", "Security-Install::Options::No-Action"),
48                  ('s', "sudo", "Security-Install::Options::Sudo"),
49                  (' ', "no-upload", "Security-Install::Options::No-Upload"),
50                  ('u', "fg-upload", "Security-Install::Options::Foreground-Upload"),
51                  (' ', "drop-advisory", "Security-Install::Options::Drop-Advisory"),
52                  ('A', "approve", "Security-Install::Options::Approve"),
53                  ('R', "reject", "Security-Install::Options::Reject"),
54                  ('D', "disembargo", "Security-Install::Options::Disembargo") ]
55
56     for i in Arguments:
57         Cnf[i[2]] = ""
58
59     arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
60
61     Options = Cnf.SubTree("Security-Install::Options")
62
63     whoami = os.getuid()
64     whoamifull = pwd.getpwuid(whoami)
65     username = whoamifull[0]
66     if username != "dak":
67         print "Non-dak user: %s" % username
68         Options["Sudo"] = "y"
69
70     if Options["Help"]:
71         print "help yourself"
72         sys.exit(0)
73
74     if len(arguments) == 0:
75         utils.fubar("Process what?")
76
77     Upload = queue.Upload(Cnf)
78     if Options["No-Action"]:
79         Options["Sudo"] = ""
80     if not Options["Sudo"] and not Options["No-Action"]:
81         Logger = Upload.Logger = logging.Logger(Cnf, "new-security-install")
82
83     return arguments
84
85 def quit():
86     if Logger:
87         Logger.close()
88     sys.exit(0)
89
90 def load_args(arguments):
91     global advisory, changes
92
93     adv_ids = {}
94     if not arguments[0].endswith(".changes"):
95         adv_ids [arguments[0]] = 1
96         arguments = arguments[1:]
97
98     null_adv_changes = []
99
100     changesfiles = {}
101     for a in arguments:
102         if "/" in a:
103             utils.fubar("can only deal with files in the current directory")
104         if not a.endswith(".changes"):
105             utils.fubar("not a .changes file: %s" % (a))
106         Upload.init_vars()
107         Upload.pkg.changes_file = a
108         Upload.update_vars()
109         if "adv id" in Upload.pkg.changes:
110             changesfiles[a] = 1
111             adv_ids[Upload.pkg.changes["adv id"]] = 1
112         else:
113             null_adv_changes.append(a)
114
115     adv_ids = adv_ids.keys()
116     if len(adv_ids) > 1:
117         utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids)))
118     if adv_ids == []:
119         advisory = None
120     else:
121         advisory = adv_ids[0]
122
123     changes = changesfiles.keys()
124     return null_adv_changes
125
126 def load_adv_changes():
127     global srcverarches, changes
128
129     for c in os.listdir("."):
130         if not c.endswith(".changes"): continue
131         Upload.init_vars()
132         Upload.pkg.changes_file = c
133         Upload.update_vars()
134         if "adv id" not in Upload.pkg.changes:
135             continue
136         if Upload.pkg.changes["adv id"] != advisory:
137             continue
138
139         if c not in changes: changes.append(c)
140         srcver = "%s %s" % (Upload.pkg.changes["source"],
141                             Upload.pkg.changes["version"])
142         srcverarches.setdefault(srcver, {})
143         for arch in Upload.pkg.changes["architecture"].keys():
144             srcverarches[srcver][arch] = 1
145
146 def advisory_info():
147     if advisory != None:
148         print "Advisory: %s" % (advisory)
149     print "Changes:"
150     for c in changes:
151         print " %s" % (c)
152
153     print "Packages:"
154     svs = srcverarches.keys()
155     svs.sort()
156     for sv in svs:
157         as = srcverarches[sv].keys()
158         as.sort()
159         print " %s (%s)" % (sv, ", ".join(as))
160
161 def prompt(opts, default):
162     p = ""
163     v = {}
164     for o in opts:
165         v[o[0].upper()] = o
166         if o[0] == default:
167             p += ", [%s]%s" % (o[0], o[1:])
168         else:
169             p += ", " + o
170     p = p[2:] + "? "
171     a = None
172
173     if Options["Automatic"]:
174         a = default
175
176     while a not in v:
177         a = utils.our_raw_input(p) + default
178         a = a[:1].upper()
179
180     return v[a]
181
182 def add_changes(extras):
183     for c in extras:
184         changes.append(c)
185         Upload.init_vars()
186         Upload.pkg.changes_file = c
187         Upload.update_vars()
188         srcver = "%s %s" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])
189         srcverarches.setdefault(srcver, {})
190         for arch in Upload.pkg.changes["architecture"].keys():
191             srcverarches[srcver][arch] = 1
192         Upload.pkg.changes["adv id"] = advisory
193         Upload.dump_vars(os.getcwd())
194
195 def yes_no(prompt):
196     if Options["Automatic"]: return True
197     while 1:
198         answer = utils.our_raw_input(prompt + " ").lower()
199         if answer in "yn":
200             return answer == "y"
201         print "Invalid answer; please try again."
202
203 def do_upload():
204     if Options["No-Upload"]:
205         print "Not uploading as requested"
206     elif Options["Foreground-Upload"]:
207         actually_upload(changes)
208     else:
209         child = os.fork()
210         if child == 0:
211             actually_upload(changes)
212             os._exit(0)
213         print "Uploading in the background"
214
215 def actually_upload(changes_files):
216     file_list = ""
217     suites = {}
218     component_mapping = {}
219     for component in Cnf.SubTree("Security-Install::ComponentMappings").List():
220         component_mapping[component] = Cnf["Security-Install::ComponentMappings::%s" % (component)]
221     uploads = {}; # uploads[uri] = file_list
222     changesfiles = {}; # changesfiles[uri] = file_list
223     package_list = {} # package_list[source_name][version]
224     changes_files.sort(utils.changes_compare)
225     for changes_file in changes_files:
226         changes_file = utils.validate_changes_file_arg(changes_file)
227         # Reset variables
228         components = {}
229         upload_uris = {}
230         file_list = []
231         Upload.init_vars()
232         # Parse the .dak file for the .changes file
233         Upload.pkg.changes_file = changes_file
234         Upload.update_vars()
235         files = Upload.pkg.files
236         changes = Upload.pkg.changes
237         dsc = Upload.pkg.dsc
238         # Build the file list for this .changes file
239         for file in files.keys():
240             poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"],
241                                     utils.poolify(changes["source"], files[file]["component"]),
242                                     file)
243             file_list.append(poolname)
244             orig_component = files[file].get("original component", files[file]["component"])
245             components[orig_component] = ""
246         # Determine the upload uri for this .changes file
247         for component in components.keys():
248             upload_uri = component_mapping.get(component)
249             if upload_uri:
250                 upload_uris[upload_uri] = ""
251         num_upload_uris = len(upload_uris.keys())
252         if num_upload_uris == 0:
253             utils.fubar("%s: No valid upload URI found from components (%s)."
254                         % (changes_file, ", ".join(components.keys())))
255         elif num_upload_uris > 1:
256             utils.fubar("%s: more than one upload URI (%s) from components (%s)."
257                         % (changes_file, ", ".join(upload_uris.keys()),
258                            ", ".join(components.keys())))
259         upload_uri = upload_uris.keys()[0]
260         # Update the file list for the upload uri
261         if not uploads.has_key(upload_uri):
262             uploads[upload_uri] = []
263         uploads[upload_uri].extend(file_list)
264         # Update the changes list for the upload uri
265         if not changesfiles.has_key(upload_uri):
266             changesfiles[upload_uri] = []
267         changesfiles[upload_uri].append(changes_file)
268         # Remember the suites and source name/version
269         for suite in changes["distribution"].keys():
270             suites[suite] = ""
271         # Remember the source name and version
272         if changes["architecture"].has_key("source") and \
273            changes["distribution"].has_key("testing"):
274             if not package_list.has_key(dsc["source"]):
275                 package_list[dsc["source"]] = {}
276             package_list[dsc["source"]][dsc["version"]] = ""
277
278     for uri in uploads.keys():
279         uploads[uri].extend(changesfiles[uri])
280         (host, path) = uri.split(":")
281         file_list = " ".join(uploads[uri])
282         print "Uploading files to %s..." % (host)
283         spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list))
284
285     if not Options["No-Action"]:
286         filename = "%s/testing-processed" % (Cnf["Dir::Log"])
287         file = utils.open_file(filename, 'a')
288         for source in package_list.keys():
289             for version in package_list[source].keys():
290                 file.write(" ".join([source, version])+'\n')
291         file.close()
292
293 def generate_advisory(template):
294     global changes, advisory
295
296     adv_packages = []
297     updated_pkgs = {};  # updated_pkgs[distro][arch][file] = {path,md5,size}
298
299     for arg in changes:
300         arg = utils.validate_changes_file_arg(arg)
301         Upload.pkg.changes_file = arg
302         Upload.init_vars()
303         Upload.update_vars()
304
305         src = Upload.pkg.changes["source"]
306         src_ver = "%s (%s)" % (src, Upload.pkg.changes["version"])
307         if src_ver not in adv_packages:
308             adv_packages.append(src_ver)
309
310         suites = Upload.pkg.changes["distribution"].keys()
311         for suite in suites:
312             if not updated_pkgs.has_key(suite):
313                 updated_pkgs[suite] = {}
314
315         files = Upload.pkg.files
316         for file in files.keys():
317             arch = files[file]["architecture"]
318             md5 = files[file]["md5sum"]
319             size = files[file]["size"]
320             poolname = Cnf["Dir::PoolRoot"] + \
321                 utils.poolify(src, files[file]["component"])
322             if arch == "source" and file.endswith(".dsc"):
323                 dscpoolname = poolname
324             for suite in suites:
325                 if not updated_pkgs[suite].has_key(arch):
326                     updated_pkgs[suite][arch] = {}
327                 updated_pkgs[suite][arch][file] = {
328                     "md5": md5, "size": size, "poolname": poolname }
329
330         dsc_files = Upload.pkg.dsc_files
331         for file in dsc_files.keys():
332             arch = "source"
333             if not dsc_files[file].has_key("files id"):
334                 continue
335
336             # otherwise, it's already in the pool and needs to be
337             # listed specially
338             md5 = dsc_files[file]["md5sum"]
339             size = dsc_files[file]["size"]
340             for suite in suites:
341                 if not updated_pkgs[suite].has_key(arch):
342                     updated_pkgs[suite][arch] = {}
343                 updated_pkgs[suite][arch][file] = {
344                     "md5": md5, "size": size, "poolname": dscpoolname }
345
346     if os.environ.has_key("SUDO_UID"):
347         whoami = long(os.environ["SUDO_UID"])
348     else:
349         whoami = os.getuid()
350     whoamifull = pwd.getpwuid(whoami)
351     username = whoamifull[4].split(",")[0]
352
353     Subst = {
354         "__ADVISORY__": advisory,
355         "__WHOAMI__": username,
356         "__DATE__": time.strftime("%B %d, %Y", time.gmtime(time.time())),
357         "__PACKAGE__": ", ".join(adv_packages),
358         "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"]
359         }
360
361     if Cnf.has_key("Dinstall::Bcc"):
362         Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"])
363
364     adv = ""
365     archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())]
366     for suite in updated_pkgs.keys():
367         ver = Cnf["Suite::%s::Version" % suite]
368         if ver != "": ver += " "
369         suite_header = "%s %s(%s)" % (Cnf["Dinstall::MyDistribution"],
370                                        ver, suite)
371         adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header))
372
373         arches = Cnf.ValueList("Suite::%s::Architectures" % suite)
374         if "source" in arches:
375             arches.remove("source")
376         if "all" in arches:
377             arches.remove("all")
378         arches.sort()
379
380         adv += "%s updates are available for %s.\n\n" % (
381                 suite.capitalize(), utils.join_with_commas_and(arches))
382
383         for a in ["source", "all"] + arches:
384             if not updated_pkgs[suite].has_key(a):
385                 continue
386
387             if a == "source":
388                 adv += "Source archives:\n\n"
389             elif a == "all":
390                 adv += "Architecture independent packages:\n\n"
391             else:
392                 adv += "%s architecture (%s)\n\n" % (a,
393                         Cnf["Architectures::%s" % a])
394
395             for file in updated_pkgs[suite][a].keys():
396                 adv += "  http://%s/%s%s\n" % (
397                                 archive, updated_pkgs[suite][a][file]["poolname"], file)
398                 adv += "    Size/MD5 checksum: %8s %s\n" % (
399                         updated_pkgs[suite][a][file]["size"],
400                         updated_pkgs[suite][a][file]["md5"])
401             adv += "\n"
402     adv = adv.rstrip()
403
404     Subst["__ADVISORY_TEXT__"] = adv
405
406     adv = utils.TemplateSubst(Subst, template)
407     return adv
408
409 def spawn(command):
410     if not re_taint_free.match(command):
411         utils.fubar("Invalid character in \"%s\"." % (command))
412
413     if Options["No-Action"]:
414         print "[%s]" % (command)
415     else:
416         (result, output) = commands.getstatusoutput(command)
417         if (result != 0):
418             utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result)
419
420
421 ##################### ! ! ! N O T E ! ! !  #####################
422 #
423 # These functions will be reinvoked by semi-priveleged users, be careful not
424 # to invoke external programs that will escalate privileges, etc.
425 #
426 ##################### ! ! ! N O T E ! ! !  #####################
427
428 def sudo(arg, fn, exit):
429     if Options["Sudo"]:
430         if advisory == None:
431             utils.fubar("Must set advisory name")
432         os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H",
433                   "/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory)
434     else:
435         fn()
436     if exit:
437         quit()
438
439 def do_Approve(): sudo("A", _do_Approve, True)
440 def _do_Approve():
441     # 1. dump advisory in drafts
442     draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory)
443     print "Advisory in %s" % (draft)
444     if not Options["No-Action"]:
445         adv_file = "./advisory.%s" % (advisory)
446         if not os.path.exists(adv_file):
447             adv_file = Cnf["Dir::Templates"]+"/security-install.advisory"
448         adv_fd = os.open(draft, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0664)
449         os.write(adv_fd, generate_advisory(adv_file))
450         os.close(adv_fd)
451         adv_fd = None
452
453     # 2. run dak process-accepted on changes
454     print "Accepting packages..."
455     spawn("dak process-accepted -pa %s" % (" ".join(changes)))
456
457     # 3. run dak make-suite-file-list / apt-ftparchve / dak generate-releases
458     print "Updating file lists for apt-ftparchive..."
459     spawn("dak make-suite-file-list")
460     print "Updating Packages and Sources files..."
461     spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file()))
462     print "Updating Release files..."
463     spawn("dak generate-releases")
464     print "Triggering security mirrors..."
465     spawn("sudo -u archvsync -H /home/archvsync/signal_security")
466
467     # 4. chdir to done - do upload
468     if not Options["No-Action"]:
469         os.chdir(Cnf["Dir::Queue::Done"])
470     do_upload()
471
472 def do_Disembargo(): sudo("D", _do_Disembargo, True)
473 def _do_Disembargo():
474     if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"):
475         utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"])
476
477     dest = Cnf["Dir::Queue::Unembargoed"]
478     emb_q = database.get_or_set_queue_id("embargoed")
479     une_q = database.get_or_set_queue_id("unembargoed")
480
481     for c in changes:
482         print "Disembargoing %s" % (c)
483
484         Upload.init_vars()
485         Upload.pkg.changes_file = c
486         Upload.update_vars()
487
488         if "source" in Upload.pkg.changes["architecture"].keys():
489             print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"])
490             Upload.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]))
491
492         files = {}
493         for suite in Upload.pkg.changes["distribution"].keys():
494             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
495                 continue
496             dest_dir = Cnf["Dir::QueueBuild"]
497             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
498                 dest_dir = os.path.join(dest_dir, suite)
499             for file in Upload.pkg.files.keys():
500                 files[os.path.join(dest_dir, file)] = 1
501
502         files = files.keys()
503         Upload.projectB.query("BEGIN WORK")
504         for f in files:
505             Upload.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q))
506         Upload.projectB.query("COMMIT WORK")
507
508         for file in Upload.pkg.files.keys():
509             utils.copy(file, os.path.join(dest, file))
510             os.unlink(file)
511
512     for c in changes:
513         utils.copy(c, os.path.join(dest, c))
514         os.unlink(c)
515         k = c[:-8] + ".dak"
516         utils.copy(k, os.path.join(dest, k))
517         os.unlink(k)
518
519 def do_Reject(): sudo("R", _do_Reject, True)
520 def _do_Reject():
521     global changes
522     for c in changes:
523         print "Rejecting %s..." % (c)
524         Upload.init_vars()
525         Upload.pkg.changes_file = c
526         Upload.update_vars()
527         files = {}
528         for suite in Upload.pkg.changes["distribution"].keys():
529             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
530                 continue
531             dest_dir = Cnf["Dir::QueueBuild"]
532             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
533                 dest_dir = os.path.join(dest_dir, suite)
534             for file in Upload.pkg.files.keys():
535                 files[os.path.join(dest_dir, file)] = 1
536
537         files = files.keys()
538
539         aborted = Upload.do_reject()
540         if not aborted:
541             os.unlink(c[:-8]+".dak")
542             for f in files:
543                 Upload.projectB.query(
544                     "DELETE FROM queue_build WHERE filename = '%s'" % (f))
545                 os.unlink(f)
546
547     print "Updating buildd information..."
548     spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd")
549
550     adv_file = "./advisory.%s" % (advisory)
551     if os.path.exists(adv_file):
552         os.unlink(adv_file)
553
554 def do_DropAdvisory():
555     for c in changes:
556         Upload.init_vars()
557         Upload.pkg.changes_file = c
558         Upload.update_vars()
559         del Upload.pkg.changes["adv id"]
560         Upload.dump_vars(os.getcwd())
561     quit()
562
563 def do_Edit():
564     adv_file = "./advisory.%s" % (advisory)
565     if not os.path.exists(adv_file):
566         utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file)
567     editor = os.environ.get("EDITOR", "vi")
568     result = os.system("%s %s" % (editor, adv_file))
569     if result != 0:
570         utils.fubar("%s invocation failed for %s." % (editor, adv_file))
571
572 def do_Show():
573     adv_file = "./advisory.%s" % (advisory)
574     if not os.path.exists(adv_file):
575         adv_file = Cnf["Dir::Templates"]+"/security-install.advisory"
576     print "====\n%s\n====" % (generate_advisory(adv_file))
577
578 def do_Quit():
579     quit()
580
581 def main():
582     global changes
583
584     args = init()
585     extras = load_args(args)
586     if advisory:
587         load_adv_changes()
588     if extras:
589         if not advisory:
590             changes = extras
591         else:
592             if srcverarches == {}:
593                 if not yes_no("Create new advisory %s?" % (advisory)):
594                     print "Not doing anything, then"
595                     quit()
596             else:
597                 advisory_info()
598                 doextras = []
599                 for c in extras:
600                     if yes_no("Add %s to %s?" % (c, advisory)):
601                         doextras.append(c)
602                 extras = doextras
603             add_changes(extras)
604
605     if not advisory:
606         utils.fubar("Must specify an advisory id")
607
608     if not changes:
609         utils.fubar("No changes specified")
610
611     if Options["Approve"]:
612         advisory_info()
613         do_Approve()
614     elif Options["Reject"]:
615         advisory_info()
616         do_Reject()
617     elif Options["Disembargo"]:
618         advisory_info()
619         do_Disembargo()
620     elif Options["Drop-Advisory"]:
621         advisory_info()
622         do_DropAdvisory()
623     else:
624         while 1:
625             default = "Q"
626             opts = ["Approve", "Edit advisory"]
627             if os.path.exists("./advisory.%s" % advisory):
628                 default = "A"
629             else:
630                 default = "E"
631             if os.getcwd() == Cnf["Dir::Queue::Embargoed"].rstrip("/"):
632                 opts.append("Disembargo")
633             opts += ["Show advisory", "Reject", "Quit"]
634
635             advisory_info()
636             what = prompt(opts, default)
637
638             if what == "Quit":
639                 do_Quit()
640             elif what == "Approve":
641                 do_Approve()
642             elif what == "Edit advisory":
643                 do_Edit()
644             elif what == "Show advisory":
645                 do_Show()
646             elif what == "Disembargo":
647                 do_Disembargo()
648             elif what == "Reject":
649                 do_Reject()
650             else:
651                 utils.fubar("Impossible answer '%s', wtf?" % (what))
652
653 ################################################################################
654
655 if __name__ == '__main__':
656     main()
657
658 ################################################################################