dak/new_security_install.py

   1 #!/usr/bin/env python
   2
   3 import katie, logging, utils, db_access
   4 import apt_pkg, os, sys, pwd, time, re, commands
   5
   6 re_taint_free = re.compile(r"^['/;\-\+\.\s\w]+$");
   7
   8 Cnf = None
   9 Options = None
  10 Katie = None
  11 Logger = None
  12
  13 advisory = None
  14 changes = []
  15 srcverarches = {}
  16
  17 def init():
  18     global Cnf, Katie, Options, Logger
  19
  20     Cnf = utils.get_conf()
  21     Cnf["Dinstall::Options::No-Mail"] = "y"
  22     Arguments = [('h', "help", "Amber::Options::Help"),
  23                  ('a', "automatic", "Amber::Options::Automatic"),
  24                  ('n', "no-action", "Amber::Options::No-Action"),
  25                  ('s', "sudo", "Amber::Options::Sudo"),
  26                  (' ', "no-upload", "Amber::Options::No-Upload"),
  27                  (' ', "drop-advisory", "Amber::Options::Drop-Advisory"),
  28                  ('A', "approve", "Amber::Options::Approve"),
  29                  ('R', "reject", "Amber::Options::Reject"),
  30                  ('D', "disembargo", "Amber::Options::Disembargo") ]
  31
  32     for i in Arguments:
  33          Cnf[i[2]] = ""
  34
  35     arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
  36
  37     Options = Cnf.SubTree("Amber::Options")
  38
  39     whoami = os.getuid()
  40     whoamifull = pwd.getpwuid(whoami)
  41     username = whoamifull[0]
  42     if username != "katie":
  43         print "Non-katie user: %s" % username
  44         Options["Sudo"] = "y"
  45
  46     if Options["Help"]:
  47         print "help yourself"
  48         sys.exit(0)
  49
  50     if len(arguments) == 0:
  51        utils.fubar("Process what?")
  52
  53     Katie = katie.Katie(Cnf)
  54     if not Options["Sudo"] and not Options["No-Action"]:
  55         Logger = Katie.Logger = logging.Logger(Cnf, "newamber")
  56
  57     return arguments
  58
  59 def quit():
  60     if Logger:
  61         Logger.close()
  62     sys.exit(0)
  63
  64 def load_args(arguments):
  65     global advisory, changes
  66
  67     adv_ids = {}
  68     if not arguments[0].endswith(".changes"):
  69        adv_ids [arguments[0]] = 1
  70        arguments = arguments[1:]
  71
  72     null_adv_changes = []
  73
  74     changesfiles = {}
  75     for a in arguments:
  76         if "/" in a:
  77             utils.fubar("can only deal with files in the current directory")
  78         if not a.endswith(".changes"):
  79             utils.fubar("not a .changes file: %s" % (a))
  80         Katie.init_vars()
  81         Katie.pkg.changes_file = a
  82         Katie.update_vars()
  83         if "adv id" in Katie.pkg.changes:
  84             changesfiles[a] = 1
  85             adv_ids[Katie.pkg.changes["adv id"]] = 1
  86         else:
  87             null_adv_changes.append(a)
  88
  89     adv_ids = adv_ids.keys()
  90     if len(adv_ids) > 1:
  91         utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids)))
  92     if adv_ids == []:
  93         advisory = None
  94     else:
  95         advisory = adv_ids[0]
  96
  97     changes = changesfiles.keys()
  98     return null_adv_changes
  99
 100 def load_adv_changes():
 101     global srcverarches, changes
 102
 103     for c in os.listdir("."):
 104         if not c.endswith(".changes"): continue
 105         Katie.init_vars()
 106         Katie.pkg.changes_file = c
 107         Katie.update_vars()
 108         if "adv id" not in Katie.pkg.changes:
 109             continue
 110         if Katie.pkg.changes["adv id"] != advisory:
 111             continue
 112
 113         if c not in changes: changes.append(c)
 114         srcver = "%s %s" % (Katie.pkg.changes["source"],
 115                             Katie.pkg.changes["version"])
 116         srcverarches.setdefault(srcver, {})
 117         for arch in Katie.pkg.changes["architecture"].keys():
 118             srcverarches[srcver][arch] = 1
 119
 120 def advisory_info():
 121     if advisory != None:
 122         print "Advisory: %s" % (advisory)
 123     print "Changes:"
 124     for c in changes:
 125         print " %s" % (c)
 126
 127     print "Packages:"
 128     svs = srcverarches.keys()
 129     svs.sort()
 130     for sv in svs:
 131         as = srcverarches[sv].keys()
 132         as.sort()
 133         print " %s (%s)" % (sv, ", ".join(as))
 134
 135 def prompt(opts, default):
 136     p = ""
 137     v = {}
 138     for o in opts:
 139         v[o[0].upper()] = o
 140         if o[0] == default:
 141             p += ", [%s]%s" % (o[0], o[1:])
 142         else:
 143             p += ", " + o
 144     p = p[2:] + "? "
 145     a = None
 146
 147     if Options["Automatic"]:
 148         a = default
 149
 150     while a not in v:
 151         a = utils.our_raw_input(p) + default
 152         a = a[:1].upper()
 153
 154     return v[a]
 155
 156 def add_changes(extras):
 157     for c in extras:
 158         changes.append(c)
 159         Katie.init_vars()
 160         Katie.pkg.changes_file = c
 161         Katie.update_vars()
 162         srcver = "%s %s" % (Katie.pkg.changes["source"], Katie.pkg.changes["version"])
 163         srcverarches.setdefault(srcver, {})
 164         for arch in Katie.pkg.changes["architecture"].keys():
 165             srcverarches[srcver][arch] = 1
 166         Katie.pkg.changes["adv id"] = advisory
 167         Katie.dump_vars(os.getcwd())
 168
 169 def yes_no(prompt):
 170     if Options["Automatic"]: return True
 171     while 1:
 172         answer = utils.our_raw_input(prompt + " ").lower()
 173         if answer in "yn":
 174             return answer == "y"
 175         print "Invalid answer; please try again."
 176
 177 def do_upload():
 178     if Options["No-Upload"]:
 179         print "Not uploading as requested"
 180         return
 181
 182     print "Would upload to ftp-master" # XXX
 183
 184 def generate_advisory(template):
 185     global changes, advisory
 186
 187     adv_packages = []
 188     updated_pkgs = {};  # updated_pkgs[distro][arch][file] = {path,md5,size}
 189
 190     for arg in changes:
 191         arg = utils.validate_changes_file_arg(arg)
 192         Katie.pkg.changes_file = arg
 193         Katie.init_vars()
 194         Katie.update_vars()
 195
 196         src = Katie.pkg.changes["source"]
 197         src_ver = "%s (%s)" % (src, Katie.pkg.changes["version"])
 198         if src_ver not in adv_packages:
 199             adv_packages.append(src_ver)
 200
 201         suites = Katie.pkg.changes["distribution"].keys()
 202         for suite in suites:
 203             if not updated_pkgs.has_key(suite):
 204                 updated_pkgs[suite] = {}
 205
 206         files = Katie.pkg.files
 207         for file in files.keys():
 208             arch = files[file]["architecture"]
 209             md5 = files[file]["md5sum"]
 210             size = files[file]["size"]
 211             poolname = Cnf["Dir::PoolRoot"] + \
 212                 utils.poolify(src, files[file]["component"])
 213             if arch == "source" and file.endswith(".dsc"):
 214                 dscpoolname = poolname
 215             for suite in suites:
 216                 if not updated_pkgs[suite].has_key(arch):
 217                     updated_pkgs[suite][arch] = {}
 218                 updated_pkgs[suite][arch][file] = {
 219                     "md5": md5, "size": size, "poolname": poolname }
 220
 221         dsc_files = Katie.pkg.dsc_files
 222         for file in dsc_files.keys():
 223             arch = "source"
 224             if not dsc_files[file].has_key("files id"):
 225                 continue
 226
 227             # otherwise, it's already in the pool and needs to be
 228             # listed specially
 229             md5 = dsc_files[file]["md5sum"]
 230             size = dsc_files[file]["size"]
 231             for suite in suites:
 232                 if not updated_pkgs[suite].has_key(arch):
 233                     updated_pkgs[suite][arch] = {}
 234                 updated_pkgs[suite][arch][file] = {
 235                     "md5": md5, "size": size, "poolname": dscpoolname }
 236
 237     if os.environ.has_key("SUDO_UID"):
 238         whoami = long(os.environ["SUDO_UID"])
 239     else:
 240         whoami = os.getuid()
 241     whoamifull = pwd.getpwuid(whoami)
 242     username = whoamifull[4].split(",")[0]
 243
 244     Subst = {
 245         "__ADVISORY__": advisory,
 246         "__WHOAMI__": username,
 247         "__DATE__": time.strftime("%B %d, %Y", time.gmtime(time.time())),
 248         "__PACKAGE__": ", ".join(adv_packages),
 249         "__KATIE_ADDRESS__": Cnf["Dinstall::MyEmailAddress"]
 250         }
 251
 252     if Cnf.has_key("Dinstall::Bcc"):
 253         Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"])
 254
 255     adv = ""
 256     archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())]
 257     for suite in updated_pkgs.keys():
 258         ver = Cnf["Suite::%s::Version" % suite]
 259         if ver != "": ver += " "
 260         suite_header = "%s %s(%s)" % (Cnf["Dinstall::MyDistribution"],
 261                                        ver, suite)
 262         adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header))
 263
 264         arches = Cnf.ValueList("Suite::%s::Architectures" % suite)
 265         if "source" in arches:
 266             arches.remove("source")
 267         if "all" in arches:
 268             arches.remove("all")
 269         arches.sort()
 270
 271         adv += "%s updates are available for %s.\n\n" % (
 272                 suite.capitalize(), utils.join_with_commas_and(arches))
 273
 274         for a in ["source", "all"] + arches:
 275             if not updated_pkgs[suite].has_key(a):
 276                 continue
 277
 278             if a == "source":
 279                 adv += "Source archives:\n\n"
 280             elif a == "all":
 281                 adv += "Architecture independent packages:\n\n"
 282             else:
 283                 adv += "%s architecture (%s)\n\n" % (a,
 284                         Cnf["Architectures::%s" % a])
 285
 286             for file in updated_pkgs[suite][a].keys():
 287                 adv += "  http://%s/%s%s\n" % (
 288                                 archive, updated_pkgs[suite][a][file]["poolname"], file)
 289                 adv += "    Size/MD5 checksum: %8s %s\n" % (
 290                         updated_pkgs[suite][a][file]["size"],
 291                         updated_pkgs[suite][a][file]["md5"])
 292             adv += "\n"
 293     adv = adv.rstrip()
 294
 295     Subst["__ADVISORY_TEXT__"] = adv
 296
 297     adv = utils.TemplateSubst(Subst, template)
 298     return adv
 299
 300
 301 def spawn(command):
 302     if not re_taint_free.match(command):
 303         utils.fubar("Invalid character in \"%s\"." % (command))
 304
 305     if Options["No-Action"]:
 306         print "[%s]" % (command)
 307     else:
 308         (result, output) = commands.getstatusoutput(command)
 309         if (result != 0):
 310             utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result)
 311
 312
 313 ##################### ! ! ! N O T E ! ! !  #####################
 314 #
 315 # These functions will be reinvoked by semi-priveleged users, be careful not
 316 # to invoke external programs that will escalate privileges, etc.
 317 #
 318 ##################### ! ! ! N O T E ! ! !  #####################
 319
 320 def sudo(arg, fn, exit):
 321     if Options["Sudo"]:
 322         if advisory == None:
 323             utils.fubar("Must set advisory name")
 324         os.spawnl(os.P_WAIT, "/usr/bin/sudo","/usr/bin/sudo", "-u", "katie", "-H",
 325            "/org/security.debian.org/katie/newamber", "-"+arg, "--", advisory)
 326     else:
 327         fn()
 328     if exit:
 329         quit()
 330
 331 def do_Approve(): sudo("A", _do_Approve, True)
 332 def _do_Approve():
 333     # 1. dump advisory in drafts
 334     draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory)
 335     print "Advisory in %s" % (draft)
 336     if not Options["No-Action"]:
 337         adv_file = "./advisory.%s" % (advisory)
 338         if not os.path.exists(adv_file):
 339             adv_file = Cnf["Dir::Templates"]+"/amber.advisory"
 340         adv_fd = os.open(draft, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0664)
 341         os.write(adv_fd, generate_advisory(adv_file))
 342         os.close(adv_fd)
 343         adv_fd = None
 344
 345     # 2. run kelly on changes
 346     print "Accepting packages..."
 347     spawn("%s/kelly -pa %s" % (Cnf["Dir::Katie"], " ".join(changes)))
 348
 349     # 3. run jenna / apt-ftparchve / ziyi / tiffani
 350     if not Options["No-Action"]:
 351         os.chdir(Cnf["Dir::Katie"])
 352
 353     print "Updating file lists for apt-ftparchive..."
 354     spawn("./jenna")
 355     print "Updating Packages and Sources files..."
 356     spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file()))
 357     print "Updating Release files..."
 358     spawn("./ziyi")
 359     print "Triggering security mirrors..."
 360     spawn("sudo -u archvsync /home/archvsync/signal_security")
 361
 362     # 4. chdir to done - do upload
 363     if not Options["No-Action"]:
 364         os.chdir(Cnf["Dir::Queue::Done"])
 365     do_upload()
 366
 367 def do_Disembargo(): sudo("D", _do_Disembargo, True)
 368 def _do_Disembargo():
 369     if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"):
 370         utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"])
 371
 372     dest = Cnf["Dir::Queue::Unembargoed"]
 373     emb_q = db_access.get_or_set_queue_id("embargoed")
 374     une_q = db_access.get_or_set_queue_id("unembargoed")
 375
 376     queuefiles = []
 377     for c in changes:
 378         print "Disembargoing %s" % (c)
 379
 380         Katie.init_vars()
 381         Katie.pkg.changes_file = c
 382         Katie.update_vars()
 383
 384         if "source" in Katie.pkg.changes["architecture"].keys():
 385             print "Adding %s %s to disembargo table" % (Katie.pkg.changes["source"], Katie.pkg.changes["version"])
 386             Katie.projectB.query("INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % (Katie.pkg.changes["source"], Katie.pkg.changes["version"]))
 387
 388         files = {}
 389         for suite in Katie.pkg.changes["distribution"].keys():
 390             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
 391                 continue
 392             dest_dir = Cnf["Dir::QueueBuild"]
 393             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
 394                 dest_dir = os.path.join(dest_dir, suite)
 395             for file in Katie.pkg.files.keys():
 396                 files[os.path.join(dest_dir, file)] = 1
 397
 398         files = files.keys()
 399         Katie.projectB.query("BEGIN WORK")
 400         for f in files:
 401             Katie.projectB.query("UPDATE queue_build SET queue = %s WHERE filename = '%s' AND queue = %s" % (une_q, f, emb_q))
 402         Katie.projectB.query("COMMIT WORK")
 403
 404         for file in Katie.pkg.files.keys():
 405             utils.copy(file, os.path.join(dest, file))
 406             os.unlink(file)
 407
 408     for c in changes:
 409         utils.copy(c, os.path.join(dest, c))
 410         os.unlink(c)
 411         k = c[:8] + ".katie"
 412         utils.copy(k, os.path.join(dest, k))
 413         os.unlink(k)
 414
 415 def do_Reject(): sudo("R", _do_Reject, True)
 416 def _do_Reject():
 417     global changes
 418     for c in changes:
 419         print "Rejecting %s..." % (c)
 420         Katie.init_vars()
 421         Katie.pkg.changes_file = c
 422         Katie.update_vars()
 423         files = {}
 424         for suite in Katie.pkg.changes["distribution"].keys():
 425             if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"):
 426                 continue
 427             dest_dir = Cnf["Dir::QueueBuild"]
 428             if Cnf.FindB("Dinstall::SecurityQueueBuild"):
 429                 dest_dir = os.path.join(dest_dir, suite)
 430             for file in Katie.pkg.files.keys():
 431                 files[os.path.join(dest_dir, file)] = 1
 432
 433         files = files.keys()
 434
 435         aborted = Katie.do_reject()
 436         if not aborted:
 437             os.unlink(c[:-8]+".katie")
 438             for f in files:
 439                 Katie.projectB.query(
 440                     "DELETE FROM queue_build WHERE filename = '%s'" % (f))
 441                 os.unlink(f)
 442
 443     print "Updating buildd information..."
 444     spawn("/org/security.debian.org/katie/cron.buildd-security")
 445
 446     adv_file = "./advisory.%s" % (advisory)
 447     if os.path.exists(adv_file):
 448         os.unlink(adv_file)
 449
 450 def do_DropAdvisory():
 451     for c in changes:
 452         Katie.init_vars()
 453         Katie.pkg.changes_file = c
 454         Katie.update_vars()
 455         del Katie.pkg.changes["adv id"]
 456         Katie.dump_vars(os.getcwd())
 457     quit()
 458
 459 def do_Edit():
 460     adv_file = "./advisory.%s" % (advisory)
 461     if not os.path.exists(adv_file):
 462         utils.copy(Cnf["Dir::Templates"]+"/amber.advisory", adv_file)
 463     editor = os.environ.get("EDITOR","vi")
 464     result = os.system("%s %s" % (editor, adv_file))
 465     if result != 0:
 466         utils.fubar("%s invocation failed for %s." % (editor, adv_file))
 467
 468 def do_Show():
 469     adv_file = "./advisory.%s" % (advisory)
 470     if not os.path.exists(adv_file):
 471         adv_file = Cnf["Dir::Templates"]+"/amber.advisory"
 472     print "====\n%s\n====" % (generate_advisory(adv_file))
 473
 474 def do_Quit():
 475     quit()
 476
 477 def main():
 478     global changes
 479
 480     args = init()
 481     extras = load_args(args)
 482     if advisory:
 483         load_adv_changes()
 484     if extras:
 485         if not advisory:
 486             changes = extras
 487         else:
 488             if srcverarches == {}:
 489                 if not yes_no("Create new advisory %s?" % (advisory)):
 490                     print "Not doing anything, then"
 491                     quit()
 492             else:
 493                 advisory_info()
 494                 doextras = []
 495                 for c in extras:
 496                     if yes_no("Add %s to %s?" % (c, advisory)):
 497                          doextras.append(c)
 498                 extras = doextras
 499             add_changes(extras)
 500
 501     if not advisory:
 502         utils.fubar("Must specify an advisory id")
 503
 504     if not changes:
 505         utils.fubar("No changes specified")
 506
 507     if Options["Approve"]:
 508         advisory_info()
 509         do_Approve()
 510     elif Options["Reject"]:
 511         advisory_info()
 512         do_Reject()
 513     elif Options["Disembargo"]:
 514         advisory_info()
 515         do_Disembargo()
 516     elif Options["Drop-Advisory"]:
 517         advisory_info()
 518         do_DropAdvisory()
 519     else:
 520         while 1:
 521             default = "Q"
 522             opts = ["Approve", "Edit advisory"]
 523             if os.path.exists("./advisory.%s" % advisory):
 524                 default = "A"
 525             else:
 526                 default = "E"
 527             if os.getcwd() == Cnf["Dir::Queue::Embargoed"].rstrip("/"):
 528                 opts.append("Disembargo")
 529             opts += ["Show advisory", "Reject", "Quit"]
 530
 531             advisory_info()
 532             what = prompt(opts, default)
 533
 534             if what == "Quit":
 535                 do_Quit()
 536             elif what == "Approve":
 537                 do_Approve()
 538             elif what == "Edit advisory":
 539                 do_Edit()
 540             elif what == "Show advisory":
 541                 do_Show()
 542             elif what == "Disembargo":
 543                 do_Disembargo()
 544             elif what == "Reject":
 545                 do_Reject()
 546             else:
 547                 utils.fubar("Impossible answer '%s', wtf?" % (what))
 548
 549 main()