dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 # Handles NEW and BYHAND packages
   5 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 # 23:12|<aj> I will not hush!
  24 # 23:12|<elmo> :>
  25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  26 # 23:13|<aj> I shall not be silenced!
  27 # 23:13|<aj> The world shall know!
  28 # 23:13|<aj> The world *must* know!
  29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  30 # 23:13|<aj> yay powerpuff girls!!
  31 # 23:13|<aj> buttercup's my favourite, who's yours?
  32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  33 # 23:14|<aj> *AREN'T YOU*?!
  34 # 23:15|<aj> I will not be treated like this.
  35 # 23:15|<aj> I shall have my revenge.
  36 # 23:15|<aj> I SHALL!!!
  37
  38 ################################################################################
  39
  40 import copy, errno, os, readline, stat, sys, time
  41 import apt_pkg, apt_inst
  42 import examine_package
  43 from daklib import database
  44 from daklib import logging
  45 from daklib import queue
  46 from daklib import utils
  47
  48 # Globals
  49 Cnf = None
  50 Options = None
  51 Upload = None
  52 projectB = None
  53 Logger = None
  54
  55 Priorities = None
  56 Sections = None
  57
  58 reject_message = ""
  59
  60 ################################################################################
  61 ################################################################################
  62 ################################################################################
  63
  64 def reject (str, prefix="Rejected: "):
  65     global reject_message
  66     if str:
  67         reject_message += prefix + str + "\n"
  68
  69 def recheck():
  70     global reject_message
  71     files = Upload.pkg.files
  72     reject_message = ""
  73
  74     for f in files.keys():
  75         # The .orig.tar.gz can disappear out from under us is it's a
  76         # duplicate of one in the archive.
  77         if not files.has_key(f):
  78             continue
  79         # Check that the source still exists
  80         if files[f]["type"] == "deb":
  81             source_version = files[f]["source version"]
  82             source_package = files[f]["source package"]
  83             if not Upload.pkg.changes["architecture"].has_key("source") \
  84                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  85                 source_epochless_version = utils.re_no_epoch.sub('', source_version)
  86                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  87                 found = 0
  88                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  89                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  90                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  91                             found = 1
  92                 if not found:
  93                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  94
  95         # Version and file overwrite checks
  96         if files[f]["type"] == "deb":
  97             reject(Upload.check_binary_against_db(f), "")
  98         elif files[f]["type"] == "dsc":
  99             reject(Upload.check_source_against_db(f), "")
 100             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 101             reject(reject_msg, "")
 102
 103     if reject_message.find("Rejected") != -1:
 104         answer = "XXX"
 105         if Options["No-Action"] or Options["Automatic"]:
 106             answer = 'S'
 107
 108         print "REJECT\n" + reject_message,
 109         prompt = "[R]eject, Skip, Quit ?"
 110
 111         while prompt.find(answer) == -1:
 112             answer = utils.our_raw_input(prompt)
 113             m = queue.re_default_answer.match(prompt)
 114             if answer == "":
 115                 answer = m.group(1)
 116             answer = answer[:1].upper()
 117
 118         if answer == 'R':
 119             Upload.do_reject(0, reject_message)
 120             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 121             return 0
 122         elif answer == 'S':
 123             return 0
 124         elif answer == 'Q':
 125             end()
 126             sys.exit(0)
 127
 128     return 1
 129
 130 ################################################################################
 131
 132 def indiv_sg_compare (a, b):
 133     """Sort by source name, source, version, 'have source', and
 134        finally by filename."""
 135     # Sort by source version
 136     q = apt_pkg.VersionCompare(a["version"], b["version"])
 137     if q:
 138         return -q
 139
 140     # Sort by 'have source'
 141     a_has_source = a["architecture"].get("source")
 142     b_has_source = b["architecture"].get("source")
 143     if a_has_source and not b_has_source:
 144         return -1
 145     elif b_has_source and not a_has_source:
 146         return 1
 147
 148     return cmp(a["filename"], b["filename"])
 149
 150 ############################################################
 151
 152 def sg_compare (a, b):
 153     a = a[1]
 154     b = b[1]
 155     """Sort by have note, source already in database and time of oldest upload."""
 156     # Sort by have note
 157     a_note_state = a["note_state"]
 158     b_note_state = b["note_state"]
 159     if a_note_state < b_note_state:
 160         return -1
 161     elif a_note_state > b_note_state:
 162         return 1
 163     # Sort by source already in database (descending)
 164     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 165     if source_in_database:
 166         return -source_in_database
 167
 168     # Sort by time of oldest upload
 169     return cmp(a["oldest"], b["oldest"])
 170
 171 def sort_changes(changes_files):
 172     """Sort into source groups, then sort each source group by version,
 173     have source, filename.  Finally, sort the source groups by have
 174     note, time of oldest upload of each source upload."""
 175     if len(changes_files) == 1:
 176         return changes_files
 177
 178     sorted_list = []
 179     cache = {}
 180     # Read in all the .changes files
 181     for filename in changes_files:
 182         try:
 183             Upload.pkg.changes_file = filename
 184             Upload.init_vars()
 185             Upload.update_vars()
 186             cache[filename] = copy.copy(Upload.pkg.changes)
 187             cache[filename]["filename"] = filename
 188         except:
 189             sorted_list.append(filename)
 190             break
 191     # Divide the .changes into per-source groups
 192     per_source = {}
 193     for filename in cache.keys():
 194         source = cache[filename]["source"]
 195         if not per_source.has_key(source):
 196             per_source[source] = {}
 197             per_source[source]["list"] = []
 198         per_source[source]["list"].append(cache[filename])
 199     # Determine oldest time and have note status for each source group
 200     for source in per_source.keys():
 201         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 202         ql = q.getresult()
 203         per_source[source]["source_in_database"] = len(ql)>0
 204         source_list = per_source[source]["list"]
 205         first = source_list[0]
 206         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 207         have_note = 0
 208         for d in per_source[source]["list"]:
 209             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 210             if mtime < oldest:
 211                 oldest = mtime
 212             have_note += (d.has_key("process-new note"))
 213         per_source[source]["oldest"] = oldest
 214         if not have_note:
 215             per_source[source]["note_state"] = 0; # none
 216         elif have_note < len(source_list):
 217             per_source[source]["note_state"] = 1; # some
 218         else:
 219             per_source[source]["note_state"] = 2; # all
 220         per_source[source]["list"].sort(indiv_sg_compare)
 221     per_source_items = per_source.items()
 222     per_source_items.sort(sg_compare)
 223     for i in per_source_items:
 224         for j in i[1]["list"]:
 225             sorted_list.append(j["filename"])
 226     return sorted_list
 227
 228 ################################################################################
 229
 230 class Section_Completer:
 231     def __init__ (self):
 232         self.sections = []
 233         q = projectB.query("SELECT section FROM section")
 234         for i in q.getresult():
 235             self.sections.append(i[0])
 236
 237     def complete(self, text, state):
 238         if state == 0:
 239             self.matches = []
 240             n = len(text)
 241             for word in self.sections:
 242                 if word[:n] == text:
 243                     self.matches.append(word)
 244         try:
 245             return self.matches[state]
 246         except IndexError:
 247             return None
 248
 249 ############################################################
 250
 251 class Priority_Completer:
 252     def __init__ (self):
 253         self.priorities = []
 254         q = projectB.query("SELECT priority FROM priority")
 255         for i in q.getresult():
 256             self.priorities.append(i[0])
 257
 258     def complete(self, text, state):
 259         if state == 0:
 260             self.matches = []
 261             n = len(text)
 262             for word in self.priorities:
 263                 if word[:n] == text:
 264                     self.matches.append(word)
 265         try:
 266             return self.matches[state]
 267         except IndexError:
 268             return None
 269
 270 ################################################################################
 271
 272 def print_new (new, indexed, file=sys.stdout):
 273     queue.check_valid(new)
 274     broken = 0
 275     index = 0
 276     for pkg in new.keys():
 277         index += 1
 278         section = new[pkg]["section"]
 279         priority = new[pkg]["priority"]
 280         if new[pkg]["section id"] == -1:
 281             section += "[!]"
 282             broken = 1
 283         if new[pkg]["priority id"] == -1:
 284             priority += "[!]"
 285             broken = 1
 286         if indexed:
 287             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 288         else:
 289             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 290         line = line.strip()+'\n'
 291         file.write(line)
 292     note = Upload.pkg.changes.get("process-new note")
 293     if note:
 294         print "*"*75
 295         print note
 296         print "*"*75
 297     return broken, note
 298
 299 ################################################################################
 300
 301 def index_range (index):
 302     if index == 1:
 303         return "1"
 304     else:
 305         return "1-%s" % (index)
 306
 307 ################################################################################
 308 ################################################################################
 309
 310 def edit_new (new):
 311     # Write the current data to a temporary file
 312     (fd, temp_filename) = utils.temp_filename()
 313     temp_file = os.fdopen(fd, 'w')
 314     print_new (new, 0, temp_file)
 315     temp_file.close()
 316     # Spawn an editor on that file
 317     editor = os.environ.get("EDITOR","vi")
 318     result = os.system("%s %s" % (editor, temp_filename))
 319     if result != 0:
 320         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 321     # Read the edited data back in
 322     temp_file = utils.open_file(temp_filename)
 323     lines = temp_file.readlines()
 324     temp_file.close()
 325     os.unlink(temp_filename)
 326     # Parse the new data
 327     for line in lines:
 328         line = line.strip()
 329         if line == "":
 330             continue
 331         s = line.split()
 332         # Pad the list if necessary
 333         s[len(s):3] = [None] * (3-len(s))
 334         (pkg, priority, section) = s[:3]
 335         if not new.has_key(pkg):
 336             utils.warn("Ignoring unknown package '%s'" % (pkg))
 337         else:
 338             # Strip off any invalid markers, print_new will readd them.
 339             if section.endswith("[!]"):
 340                 section = section[:-3]
 341             if priority.endswith("[!]"):
 342                 priority = priority[:-3]
 343             for f in new[pkg]["files"]:
 344                 Upload.pkg.files[f]["section"] = section
 345                 Upload.pkg.files[f]["priority"] = priority
 346             new[pkg]["section"] = section
 347             new[pkg]["priority"] = priority
 348
 349 ################################################################################
 350
 351 def edit_index (new, index):
 352     priority = new[index]["priority"]
 353     section = new[index]["section"]
 354     ftype = new[index]["type"]
 355     done = 0
 356     while not done:
 357         print "\t".join([index, priority, section])
 358
 359         answer = "XXX"
 360         if ftype != "dsc":
 361             prompt = "[B]oth, Priority, Section, Done ? "
 362         else:
 363             prompt = "[S]ection, Done ? "
 364         edit_priority = edit_section = 0
 365
 366         while prompt.find(answer) == -1:
 367             answer = utils.our_raw_input(prompt)
 368             m = queue.re_default_answer.match(prompt)
 369             if answer == "":
 370                 answer = m.group(1)
 371             answer = answer[:1].upper()
 372
 373         if answer == 'P':
 374             edit_priority = 1
 375         elif answer == 'S':
 376             edit_section = 1
 377         elif answer == 'B':
 378             edit_priority = edit_section = 1
 379         elif answer == 'D':
 380             done = 1
 381
 382         # Edit the priority
 383         if edit_priority:
 384             readline.set_completer(Priorities.complete)
 385             got_priority = 0
 386             while not got_priority:
 387                 new_priority = utils.our_raw_input("New priority: ").strip()
 388                 if new_priority not in Priorities.priorities:
 389                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 390                 else:
 391                     got_priority = 1
 392                     priority = new_priority
 393
 394         # Edit the section
 395         if edit_section:
 396             readline.set_completer(Sections.complete)
 397             got_section = 0
 398             while not got_section:
 399                 new_section = utils.our_raw_input("New section: ").strip()
 400                 if new_section not in Sections.sections:
 401                     print "E: '%s' is not a valid section, try again." % (new_section)
 402                 else:
 403                     got_section = 1
 404                     section = new_section
 405
 406         # Reset the readline completer
 407         readline.set_completer(None)
 408
 409     for f in new[index]["files"]:
 410         Upload.pkg.files[f]["section"] = section
 411         Upload.pkg.files[f]["priority"] = priority
 412     new[index]["priority"] = priority
 413     new[index]["section"] = section
 414     return new
 415
 416 ################################################################################
 417
 418 def edit_overrides (new):
 419     print
 420     done = 0
 421     while not done:
 422         print_new (new, 1)
 423         new_index = {}
 424         index = 0
 425         for i in new.keys():
 426             index += 1
 427             new_index[index] = i
 428
 429         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 430
 431         got_answer = 0
 432         while not got_answer:
 433             answer = utils.our_raw_input(prompt)
 434             if not answer.isdigit():
 435                 answer = answer[:1].upper()
 436             if answer == "E" or answer == "D":
 437                 got_answer = 1
 438             elif queue.re_isanum.match (answer):
 439                 answer = int(answer)
 440                 if (answer < 1) or (answer > index):
 441                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 442                 else:
 443                     got_answer = 1
 444
 445         if answer == 'E':
 446             edit_new(new)
 447         elif answer == 'D':
 448             done = 1
 449         else:
 450             edit_index (new, new_index[answer])
 451
 452     return new
 453
 454 ################################################################################
 455
 456 def edit_note(note):
 457     # Write the current data to a temporary file
 458     (fd, temp_filename) = utils.temp_filename()
 459     temp_file = os.fdopen(temp_filename, 'w')
 460     temp_file.write(note)
 461     temp_file.close()
 462     editor = os.environ.get("EDITOR","vi")
 463     answer = 'E'
 464     while answer == 'E':
 465         os.system("%s %s" % (editor, temp_filename))
 466         temp_file = utils.open_file(temp_filename)
 467         note = temp_file.read().rstrip()
 468         temp_file.close()
 469         print "Note:"
 470         print utils.prefix_multi_line_string(note,"  ")
 471         prompt = "[D]one, Edit, Abandon, Quit ?"
 472         answer = "XXX"
 473         while prompt.find(answer) == -1:
 474             answer = utils.our_raw_input(prompt)
 475             m = queue.re_default_answer.search(prompt)
 476             if answer == "":
 477                 answer = m.group(1)
 478             answer = answer[:1].upper()
 479     os.unlink(temp_filename)
 480     if answer == 'A':
 481         return
 482     elif answer == 'Q':
 483         end()
 484         sys.exit(0)
 485     Upload.pkg.changes["process-new note"] = note
 486     Upload.dump_vars(Cnf["Dir::Queue::New"])
 487
 488 ################################################################################
 489
 490 def check_pkg ():
 491     try:
 492         less_fd = os.popen("less -R -", 'w', 0)
 493         stdout_fd = sys.stdout
 494         try:
 495             sys.stdout = less_fd
 496             changes = utils.parse_changes (Upload.pkg.changes_file)
 497             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 498             files = Upload.pkg.files
 499             for f in files.keys():
 500                 if files[f].has_key("new"):
 501                     ftype = files[f]["type"]
 502                     if ftype == "deb":
 503                         examine_package.check_deb(changes['distribution'], f)
 504                     elif ftype == "dsc":
 505                         examine_package.check_dsc(changes['distribution'], f)
 506         finally:
 507             sys.stdout = stdout_fd
 508     except IOError, e:
 509         if e.errno == errno.EPIPE:
 510             utils.warn("[examine_package] Caught EPIPE; skipping.")
 511             pass
 512         else:
 513             raise
 514     except KeyboardInterrupt:
 515         utils.warn("[examine_package] Caught C-c; skipping.")
 516         pass
 517
 518 ################################################################################
 519
 520 ## FIXME: horribly Debian specific
 521
 522 def do_bxa_notification():
 523     files = Upload.pkg.files
 524     summary = ""
 525     for f in files.keys():
 526         if files[f]["type"] == "deb":
 527             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 528             summary += "\n"
 529             summary += "Package: %s\n" % (control.Find("Package"))
 530             summary += "Description: %s\n" % (control.Find("Description"))
 531     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 532     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 533     utils.send_mail(bxa_mail)
 534
 535 ################################################################################
 536
 537 def add_overrides (new):
 538     changes = Upload.pkg.changes
 539     files = Upload.pkg.files
 540
 541     projectB.query("BEGIN WORK")
 542     for suite in changes["suite"].keys():
 543         suite_id = database.get_suite_id(suite)
 544         for pkg in new.keys():
 545             component_id = database.get_component_id(new[pkg]["component"])
 546             type_id = database.get_override_type_id(new[pkg]["type"])
 547             priority_id = new[pkg]["priority id"]
 548             section_id = new[pkg]["section id"]
 549             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 550             for f in new[pkg]["files"]:
 551                 if files[f].has_key("new"):
 552                     del files[f]["new"]
 553             del new[pkg]
 554
 555     projectB.query("COMMIT WORK")
 556
 557     if Cnf.FindB("Dinstall::BXANotify"):
 558         do_bxa_notification()
 559
 560 ################################################################################
 561
 562 def prod_maintainer ():
 563     # Here we prepare an editor and get them ready to prod...
 564     (fd, temp_filename) = utils.temp_filename()
 565     editor = os.environ.get("EDITOR","vi")
 566     answer = 'E'
 567     while answer == 'E':
 568         os.system("%s %s" % (editor, temp_filename))
 569         f = os.fdopen(fd)
 570         prod_message = "".join(f.readlines())
 571         f.close()
 572         print "Prod message:"
 573         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 574         prompt = "[P]rod, Edit, Abandon, Quit ?"
 575         answer = "XXX"
 576         while prompt.find(answer) == -1:
 577             answer = utils.our_raw_input(prompt)
 578             m = queue.re_default_answer.search(prompt)
 579             if answer == "":
 580                 answer = m.group(1)
 581             answer = answer[:1].upper()
 582         os.unlink(temp_filename)
 583         if answer == 'A':
 584             return
 585         elif answer == 'Q':
 586             end()
 587             sys.exit(0)
 588     # Otherwise, do the proding...
 589     user_email_address = utils.whoami() + " <%s>" % (
 590         Cnf["Dinstall::MyAdminAddress"])
 591
 592     Subst = Upload.Subst
 593
 594     Subst["__FROM_ADDRESS__"] = user_email_address
 595     Subst["__PROD_MESSAGE__"] = prod_message
 596     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 597
 598     prod_mail_message = utils.TemplateSubst(
 599         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 600
 601     # Send the prod mail if appropriate
 602     if not Cnf["Dinstall::Options::No-Mail"]:
 603         utils.send_mail(prod_mail_message)
 604
 605     print "Sent proding message"
 606
 607 ################################################################################
 608
 609 def do_new():
 610     print "NEW\n"
 611     files = Upload.pkg.files
 612     changes = Upload.pkg.changes
 613
 614     # Make a copy of distribution we can happily trample on
 615     changes["suite"] = copy.copy(changes["distribution"])
 616
 617     # Fix up the list of target suites
 618     for suite in changes["suite"].keys():
 619         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 620         if override:
 621             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 622               database.get_suite_id(override) == -1)
 623             if olderr or newerr:
 624                 (oinv, newinv) = ("", "")
 625                 if olderr: oinv = "invalid "
 626                 if newerr: ninv = "invalid "
 627                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 628                         oinv, suite, ninv, override)
 629             del changes["suite"][suite]
 630             changes["suite"][override] = 1
 631     # Validate suites
 632     for suite in changes["suite"].keys():
 633         suite_id = database.get_suite_id(suite)
 634         if suite_id == -1:
 635             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 636
 637     # The main NEW processing loop
 638     done = 0
 639     while not done:
 640         # Find out what's new
 641         new = queue.determine_new(changes, files, projectB)
 642
 643         if not new:
 644             break
 645
 646         answer = "XXX"
 647         if Options["No-Action"] or Options["Automatic"]:
 648             answer = 'S'
 649
 650         (broken, note) = print_new(new, 0)
 651         prompt = ""
 652
 653         if not broken and not note:
 654             prompt = "Add overrides, "
 655         if broken:
 656             print "W: [!] marked entries must be fixed before package can be processed."
 657         if note:
 658             print "W: note must be removed before package can be processed."
 659             prompt += "Remove note, "
 660
 661         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 662
 663         while prompt.find(answer) == -1:
 664             answer = utils.our_raw_input(prompt)
 665             m = queue.re_default_answer.search(prompt)
 666             if answer == "":
 667                 answer = m.group(1)
 668             answer = answer[:1].upper()
 669
 670         if answer == 'A':
 671             done = add_overrides (new)
 672         elif answer == 'C':
 673             check_pkg()
 674         elif answer == 'E':
 675             new = edit_overrides (new)
 676         elif answer == 'M':
 677             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 678             if not aborted:
 679                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 680                 done = 1
 681         elif answer == 'N':
 682             edit_note(changes.get("process-new note", ""))
 683         elif answer == 'P':
 684             prod_maintainer()
 685         elif answer == 'R':
 686             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 687             if confirm == "y":
 688                 del changes["process-new note"]
 689         elif answer == 'S':
 690             done = 1
 691         elif answer == 'Q':
 692             end()
 693             sys.exit(0)
 694
 695 ################################################################################
 696 ################################################################################
 697 ################################################################################
 698
 699 def usage (exit_code=0):
 700     print """Usage: dak process-new [OPTION]... [CHANGES]...
 701   -a, --automatic           automatic run
 702   -h, --help                show this help and exit.
 703   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 704   -m, --manual-reject=MSG   manual reject with `msg'
 705   -n, --no-action           don't do anything
 706   -V, --version             display the version number and exit"""
 707     sys.exit(exit_code)
 708
 709 ################################################################################
 710
 711 def init():
 712     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 713
 714     Cnf = utils.get_conf()
 715
 716     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 717                  ('h',"help","Process-New::Options::Help"),
 718                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 719                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 720                  ('n',"no-action","Process-New::Options::No-Action")]
 721
 722     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 723         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 724             Cnf["Process-New::Options::%s" % (i)] = ""
 725
 726     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 727     Options = Cnf.SubTree("Process-New::Options")
 728
 729     if Options["Help"]:
 730         usage()
 731
 732     Upload = queue.Upload(Cnf)
 733
 734     if not Options["No-Action"]:
 735         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 736
 737     projectB = Upload.projectB
 738
 739     Sections = Section_Completer()
 740     Priorities = Priority_Completer()
 741     readline.parse_and_bind("tab: complete")
 742
 743     return changes_files
 744
 745 ################################################################################
 746
 747 def do_byhand():
 748     done = 0
 749     while not done:
 750         files = Upload.pkg.files
 751         will_install = 1
 752         byhand = []
 753
 754         for f in files.keys():
 755             if files[f]["type"] == "byhand":
 756                 if os.path.exists(f):
 757                     print "W: %s still present; please process byhand components and try again." % (f)
 758                     will_install = 0
 759                 else:
 760                     byhand.append(f)
 761
 762         answer = "XXXX"
 763         if Options["No-Action"]:
 764             answer = "S"
 765         if will_install:
 766             if Options["Automatic"] and not Options["No-Action"]:
 767                 answer = 'A'
 768             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 769         else:
 770             prompt = "Manual reject, [S]kip, Quit ?"
 771
 772         while prompt.find(answer) == -1:
 773             answer = utils.our_raw_input(prompt)
 774             m = queue.re_default_answer.search(prompt)
 775             if answer == "":
 776                 answer = m.group(1)
 777             answer = answer[:1].upper()
 778
 779         if answer == 'A':
 780             done = 1
 781             for f in byhand:
 782                 del files[f]
 783         elif answer == 'M':
 784             Upload.do_reject(1, Options["Manual-Reject"])
 785             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 786             done = 1
 787         elif answer == 'S':
 788             done = 1
 789         elif answer == 'Q':
 790             end()
 791             sys.exit(0)
 792
 793 ################################################################################
 794
 795 def get_accept_lock():
 796     retry = 0
 797     while retry < 10:
 798         try:
 799             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 800             retry = 10
 801         except OSError, e:
 802             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 803                 retry += 1
 804                 if (retry >= 10):
 805                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 806                 else:
 807                     print("Unable to get accepted lock (try %d of 10)" % retry)
 808                 time.sleep(60)
 809             else:
 810                 raise
 811
 812 def move_to_dir (dest, perms=0660, changesperms=0664):
 813     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 814     file_keys = Upload.pkg.files.keys()
 815     for f in file_keys:
 816         utils.move (f, dest, perms=perms)
 817
 818 def is_source_in_queue_dir(qdir):
 819     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 820                 and x.endswith(".changes") ]
 821     for entry in entries:
 822         # read the .dak
 823         u = queue.Upload(Cnf)
 824         u.pkg.changes_file = os.path.join(qdir, entry)
 825         u.update_vars()
 826         if not u.pkg.changes["architecture"].has_key("source"):
 827             # another binary upload, ignore
 828             continue
 829         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 830             # another version, ignore
 831             continue
 832         # found it!
 833         return True
 834     return False
 835
 836 def move_to_holding(suite, queue_dir):
 837     print "Moving to %s holding area." % (suite.upper(),)
 838     if Options["No-Action"]:
 839         return
 840     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 841     Upload.dump_vars(queue_dir)
 842     move_to_dir(queue_dir)
 843     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 844
 845 def _accept():
 846     if Options["No-Action"]:
 847         return
 848     (summary, short_summary) = Upload.build_summaries()
 849     Upload.accept(summary, short_summary)
 850     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 851
 852 def do_accept_stableupdate(suite, q):
 853     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 854     if not Upload.pkg.changes["architecture"].has_key("source"):
 855         # It is not a sourceful upload.  So its source may be either in p-u
 856         # holding, in new, in accepted or already installed.
 857         if is_source_in_queue_dir(queue_dir):
 858             # It's in p-u holding, so move it there.
 859             print "Binary-only upload, source in %s." % (q,)
 860             move_to_holding(suite, queue_dir)
 861         elif Upload.source_exists(Upload.pkg.changes["source"],
 862                 Upload.pkg.changes["version"]):
 863             # dak tells us that there is source available.  At time of
 864             # writing this means that it is installed, so put it into
 865             # accepted.
 866             print "Binary-only upload, source installed."
 867             _accept()
 868         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 869             # The source is in accepted, the binary cleared NEW: accept it.
 870             print "Binary-only upload, source in accepted."
 871             _accept()
 872         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 873             # It's in NEW.  We expect the source to land in p-u holding
 874             # pretty soon.
 875             print "Binary-only upload, source in new."
 876             move_to_holding(suite, queue_dir)
 877         else:
 878             # No case applicable.  Bail out.  Return will cause the upload
 879             # to be skipped.
 880             print "ERROR"
 881             print "Stable update failed.  Source not found."
 882             return
 883     else:
 884         # We are handling a sourceful upload.  Move to accepted if currently
 885         # in p-u holding and to p-u holding otherwise.
 886         if is_source_in_queue_dir(queue_dir):
 887             print "Sourceful upload in %s, accepting." % (q,)
 888             _accept()
 889         else:
 890             move_to_holding(suite, queue_dir)
 891
 892 def do_accept():
 893     print "ACCEPT"
 894     if not Options["No-Action"]:
 895         get_accept_lock()
 896         (summary, short_summary) = Upload.build_summaries()
 897     try:
 898         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 899             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 900             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 901             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 902             # Check for override disparities
 903             Upload.Subst["__SUMMARY__"] = summary
 904         else:
 905             # Stable updates need to be copied to proposed-updates holding
 906             # area instead of accepted.  Sourceful uploads need to go
 907             # to it directly, binaries only if the source has not yet been
 908             # accepted into p-u.
 909             for suite, q in [("proposed-updates", "ProposedUpdates"),
 910                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 911                 if not Upload.pkg.changes["distribution"].has_key(suite):
 912                     continue
 913                 return do_accept_stableupdate(suite, q)
 914             # Just a normal upload, accept it...
 915             _accept()
 916     finally:
 917         if not Options["No-Action"]:
 918             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 919
 920 def check_status(files):
 921     new = byhand = 0
 922     for f in files.keys():
 923         if files[f]["type"] == "byhand":
 924             byhand = 1
 925         elif files[f].has_key("new"):
 926             new = 1
 927     return (new, byhand)
 928
 929 def do_pkg(changes_file):
 930     Upload.pkg.changes_file = changes_file
 931     Upload.init_vars()
 932     Upload.update_vars()
 933     Upload.update_subst()
 934     files = Upload.pkg.files
 935
 936     if not recheck():
 937         return
 938
 939     (new, byhand) = check_status(files)
 940     if new or byhand:
 941         if new:
 942             do_new()
 943         if byhand:
 944             do_byhand()
 945         (new, byhand) = check_status(files)
 946
 947     if not new and not byhand:
 948         do_accept()
 949
 950 ################################################################################
 951
 952 def end():
 953     accept_count = Upload.accept_count
 954     accept_bytes = Upload.accept_bytes
 955
 956     if accept_count:
 957         sets = "set"
 958         if accept_count > 1:
 959             sets = "sets"
 960         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 961         Logger.log(["total",accept_count,accept_bytes])
 962
 963     if not Options["No-Action"]:
 964         Logger.close()
 965
 966 ################################################################################
 967
 968 def do_comments(dir, opref, npref, line, fn):
 969     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 970         lines = open("%s/%s" % (dir, comm)).readlines()
 971         if len(lines) == 0 or lines[0] != line + "\n": continue
 972         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 973                                 and x.endswith(".changes") ]
 974         changes_files = sort_changes(changes_files)
 975         for f in changes_files:
 976             f = utils.validate_changes_file_arg(f, 0)
 977             if not f: continue
 978             print "\n" + f
 979             fn(f, "".join(lines[1:]))
 980
 981         if opref != npref and not Options["No-Action"]:
 982             newcomm = npref + comm[len(opref):]
 983             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 984
 985 ################################################################################
 986
 987 def comment_accept(changes_file, comments):
 988     Upload.pkg.changes_file = changes_file
 989     Upload.init_vars()
 990     Upload.update_vars()
 991     Upload.update_subst()
 992     files = Upload.pkg.files
 993
 994     if not recheck():
 995         return # dak wants to REJECT, crap
 996
 997     (new, byhand) = check_status(files)
 998     if not new and not byhand:
 999         do_accept()
1000
1001 ################################################################################
1002
1003 def comment_reject(changes_file, comments):
1004     Upload.pkg.changes_file = changes_file
1005     Upload.init_vars()
1006     Upload.update_vars()
1007     Upload.update_subst()
1008
1009     if not recheck():
1010         pass # dak has its own reasons to reject as well, which is fine
1011
1012     reject(comments)
1013     print "REJECT\n" + reject_message,
1014     if not Options["No-Action"]:
1015         Upload.do_reject(0, reject_message)
1016         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1017
1018 ################################################################################
1019
1020 def main():
1021     changes_files = init()
1022     if len(changes_files) > 50:
1023         sys.stderr.write("Sorting changes...\n")
1024     changes_files = sort_changes(changes_files)
1025
1026     # Kill me now? **FIXME**
1027     Cnf["Dinstall::Options::No-Mail"] = ""
1028     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1029     if Cnf.has_key("Dinstall::Bcc"):
1030         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1031     else:
1032         Upload.Subst["__BCC__"] = bcc
1033
1034     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1035     if commentsdir:
1036         if changes_files != []:
1037             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1038             sys.exit(1)
1039         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1040         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1041     else:
1042         for changes_file in changes_files:
1043             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1044             if not changes_file:
1045                 continue
1046             print "\n" + changes_file
1047             do_pkg (changes_file)
1048
1049     end()
1050
1051 ################################################################################
1052
1053 if __name__ == '__main__':
1054     main()