dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 # Handles NEW and BYHAND packages
   5 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 # 23:12|<aj> I will not hush!
  24 # 23:12|<elmo> :>
  25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  26 # 23:13|<aj> I shall not be silenced!
  27 # 23:13|<aj> The world shall know!
  28 # 23:13|<aj> The world *must* know!
  29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  30 # 23:13|<aj> yay powerpuff girls!!
  31 # 23:13|<aj> buttercup's my favourite, who's yours?
  32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  33 # 23:14|<aj> *AREN'T YOU*?!
  34 # 23:15|<aj> I will not be treated like this.
  35 # 23:15|<aj> I shall have my revenge.
  36 # 23:15|<aj> I SHALL!!!
  37
  38 ################################################################################
  39
  40 import copy, errno, os, readline, stat, sys, time
  41 import apt_pkg, apt_inst
  42 import examine_package
  43 from daklib import database
  44 from daklib import logging
  45 from daklib import queue
  46 from daklib import utils
  47
  48 # Globals
  49 Cnf = None
  50 Options = None
  51 Upload = None
  52 projectB = None
  53 Logger = None
  54
  55 Priorities = None
  56 Sections = None
  57
  58 reject_message = ""
  59
  60 ################################################################################
  61 ################################################################################
  62 ################################################################################
  63
  64 def reject (str, prefix="Rejected: "):
  65     global reject_message
  66     if str:
  67         reject_message += prefix + str + "\n"
  68
  69 def recheck():
  70     global reject_message
  71     files = Upload.pkg.files
  72     reject_message = ""
  73
  74     for f in files.keys():
  75         # The .orig.tar.gz can disappear out from under us is it's a
  76         # duplicate of one in the archive.
  77         if not files.has_key(f):
  78             continue
  79         # Check that the source still exists
  80         if files[f]["type"] == "deb":
  81             source_version = files[f]["source version"]
  82             source_package = files[f]["source package"]
  83             if not Upload.pkg.changes["architecture"].has_key("source") \
  84                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  85                 source_epochless_version = utils.re_no_epoch.sub('', source_version)
  86                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  87                 found = 0
  88                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  89                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  90                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  91                             found = 1
  92                 if not found:
  93                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  94
  95         # Version and file overwrite checks
  96         if files[f]["type"] == "deb":
  97             reject(Upload.check_binary_against_db(f), "")
  98         elif files[f]["type"] == "dsc":
  99             reject(Upload.check_source_against_db(f), "")
 100             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 101             reject(reject_msg, "")
 102
 103     if reject_message.find("Rejected") != -1:
 104         answer = "XXX"
 105         if Options["No-Action"] or Options["Automatic"]:
 106             answer = 'S'
 107
 108         print "REJECT\n" + reject_message,
 109         prompt = "[R]eject, Skip, Quit ?"
 110
 111         while prompt.find(answer) == -1:
 112             answer = utils.our_raw_input(prompt)
 113             m = queue.re_default_answer.match(prompt)
 114             if answer == "":
 115                 answer = m.group(1)
 116             answer = answer[:1].upper()
 117
 118         if answer == 'R':
 119             Upload.do_reject(0, reject_message)
 120             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 121             return 0
 122         elif answer == 'S':
 123             return 0
 124         elif answer == 'Q':
 125             end()
 126             sys.exit(0)
 127
 128     return 1
 129
 130 ################################################################################
 131
 132 def indiv_sg_compare (a, b):
 133     """Sort by source name, source, version, 'have source', and
 134        finally by filename."""
 135     # Sort by source version
 136     q = apt_pkg.VersionCompare(a["version"], b["version"])
 137     if q:
 138         return -q
 139
 140     # Sort by 'have source'
 141     a_has_source = a["architecture"].get("source")
 142     b_has_source = b["architecture"].get("source")
 143     if a_has_source and not b_has_source:
 144         return -1
 145     elif b_has_source and not a_has_source:
 146         return 1
 147
 148     return cmp(a["filename"], b["filename"])
 149
 150 ############################################################
 151
 152 def sg_compare (a, b):
 153     a = a[1]
 154     b = b[1]
 155     """Sort by have note, source already in database and time of oldest upload."""
 156     # Sort by have note
 157     a_note_state = a["note_state"]
 158     b_note_state = b["note_state"]
 159     if a_note_state < b_note_state:
 160         return -1
 161     elif a_note_state > b_note_state:
 162         return 1
 163     # Sort by source already in database (descending)
 164     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 165     if source_in_database:
 166         return -source_in_database
 167
 168     # Sort by time of oldest upload
 169     return cmp(a["oldest"], b["oldest"])
 170
 171 def sort_changes(changes_files):
 172     """Sort into source groups, then sort each source group by version,
 173     have source, filename.  Finally, sort the source groups by have
 174     note, time of oldest upload of each source upload."""
 175     if len(changes_files) == 1:
 176         return changes_files
 177
 178     sorted_list = []
 179     cache = {}
 180     # Read in all the .changes files
 181     for filename in changes_files:
 182         try:
 183             Upload.pkg.changes_file = filename
 184             Upload.init_vars()
 185             Upload.update_vars()
 186             cache[filename] = copy.copy(Upload.pkg.changes)
 187             cache[filename]["filename"] = filename
 188         except:
 189             sorted_list.append(filename)
 190             break
 191     # Divide the .changes into per-source groups
 192     per_source = {}
 193     for filename in cache.keys():
 194         source = cache[filename]["source"]
 195         if not per_source.has_key(source):
 196             per_source[source] = {}
 197             per_source[source]["list"] = []
 198         per_source[source]["list"].append(cache[filename])
 199     # Determine oldest time and have note status for each source group
 200     for source in per_source.keys():
 201         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 202         ql = q.getresult()
 203         per_source[source]["source_in_database"] = len(ql)>0
 204         source_list = per_source[source]["list"]
 205         first = source_list[0]
 206         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 207         have_note = 0
 208         for d in per_source[source]["list"]:
 209             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 210             if mtime < oldest:
 211                 oldest = mtime
 212             have_note += (d.has_key("process-new note"))
 213         per_source[source]["oldest"] = oldest
 214         if not have_note:
 215             per_source[source]["note_state"] = 0; # none
 216         elif have_note < len(source_list):
 217             per_source[source]["note_state"] = 1; # some
 218         else:
 219             per_source[source]["note_state"] = 2; # all
 220         per_source[source]["list"].sort(indiv_sg_compare)
 221     per_source_items = per_source.items()
 222     per_source_items.sort(sg_compare)
 223     for i in per_source_items:
 224         for j in i[1]["list"]:
 225             sorted_list.append(j["filename"])
 226     return sorted_list
 227
 228 ################################################################################
 229
 230 class Section_Completer:
 231     def __init__ (self):
 232         self.sections = []
 233         self.matches = []
 234         q = projectB.query("SELECT section FROM section")
 235         for i in q.getresult():
 236             self.sections.append(i[0])
 237
 238     def complete(self, text, state):
 239         if state == 0:
 240             self.matches = []
 241             n = len(text)
 242             for word in self.sections:
 243                 if word[:n] == text:
 244                     self.matches.append(word)
 245         try:
 246             return self.matches[state]
 247         except IndexError:
 248             return None
 249
 250 ############################################################
 251
 252 class Priority_Completer:
 253     def __init__ (self):
 254         self.priorities = []
 255         self.matches = []
 256         q = projectB.query("SELECT priority FROM priority")
 257         for i in q.getresult():
 258             self.priorities.append(i[0])
 259
 260     def complete(self, text, state):
 261         if state == 0:
 262             self.matches = []
 263             n = len(text)
 264             for word in self.priorities:
 265                 if word[:n] == text:
 266                     self.matches.append(word)
 267         try:
 268             return self.matches[state]
 269         except IndexError:
 270             return None
 271
 272 ################################################################################
 273
 274 def print_new (new, indexed, file=sys.stdout):
 275     queue.check_valid(new)
 276     broken = 0
 277     index = 0
 278     for pkg in new.keys():
 279         index += 1
 280         section = new[pkg]["section"]
 281         priority = new[pkg]["priority"]
 282         if new[pkg]["section id"] == -1:
 283             section += "[!]"
 284             broken = 1
 285         if new[pkg]["priority id"] == -1:
 286             priority += "[!]"
 287             broken = 1
 288         if indexed:
 289             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 290         else:
 291             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 292         line = line.strip()+'\n'
 293         file.write(line)
 294     note = Upload.pkg.changes.get("process-new note")
 295     if note:
 296         print "*"*75
 297         print note
 298         print "*"*75
 299     return broken, note
 300
 301 ################################################################################
 302
 303 def index_range (index):
 304     if index == 1:
 305         return "1"
 306     else:
 307         return "1-%s" % (index)
 308
 309 ################################################################################
 310 ################################################################################
 311
 312 def edit_new (new):
 313     # Write the current data to a temporary file
 314     (fd, temp_filename) = utils.temp_filename()
 315     temp_file = os.fdopen(fd, 'w')
 316     print_new (new, 0, temp_file)
 317     temp_file.close()
 318     # Spawn an editor on that file
 319     editor = os.environ.get("EDITOR","vi")
 320     result = os.system("%s %s" % (editor, temp_filename))
 321     if result != 0:
 322         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 323     # Read the edited data back in
 324     temp_file = utils.open_file(temp_filename)
 325     lines = temp_file.readlines()
 326     temp_file.close()
 327     os.unlink(temp_filename)
 328     # Parse the new data
 329     for line in lines:
 330         line = line.strip()
 331         if line == "":
 332             continue
 333         s = line.split()
 334         # Pad the list if necessary
 335         s[len(s):3] = [None] * (3-len(s))
 336         (pkg, priority, section) = s[:3]
 337         if not new.has_key(pkg):
 338             utils.warn("Ignoring unknown package '%s'" % (pkg))
 339         else:
 340             # Strip off any invalid markers, print_new will readd them.
 341             if section.endswith("[!]"):
 342                 section = section[:-3]
 343             if priority.endswith("[!]"):
 344                 priority = priority[:-3]
 345             for f in new[pkg]["files"]:
 346                 Upload.pkg.files[f]["section"] = section
 347                 Upload.pkg.files[f]["priority"] = priority
 348             new[pkg]["section"] = section
 349             new[pkg]["priority"] = priority
 350
 351 ################################################################################
 352
 353 def edit_index (new, index):
 354     priority = new[index]["priority"]
 355     section = new[index]["section"]
 356     ftype = new[index]["type"]
 357     done = 0
 358     while not done:
 359         print "\t".join([index, priority, section])
 360
 361         answer = "XXX"
 362         if ftype != "dsc":
 363             prompt = "[B]oth, Priority, Section, Done ? "
 364         else:
 365             prompt = "[S]ection, Done ? "
 366         edit_priority = edit_section = 0
 367
 368         while prompt.find(answer) == -1:
 369             answer = utils.our_raw_input(prompt)
 370             m = queue.re_default_answer.match(prompt)
 371             if answer == "":
 372                 answer = m.group(1)
 373             answer = answer[:1].upper()
 374
 375         if answer == 'P':
 376             edit_priority = 1
 377         elif answer == 'S':
 378             edit_section = 1
 379         elif answer == 'B':
 380             edit_priority = edit_section = 1
 381         elif answer == 'D':
 382             done = 1
 383
 384         # Edit the priority
 385         if edit_priority:
 386             readline.set_completer(Priorities.complete)
 387             got_priority = 0
 388             while not got_priority:
 389                 new_priority = utils.our_raw_input("New priority: ").strip()
 390                 if new_priority not in Priorities.priorities:
 391                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 392                 else:
 393                     got_priority = 1
 394                     priority = new_priority
 395
 396         # Edit the section
 397         if edit_section:
 398             readline.set_completer(Sections.complete)
 399             got_section = 0
 400             while not got_section:
 401                 new_section = utils.our_raw_input("New section: ").strip()
 402                 if new_section not in Sections.sections:
 403                     print "E: '%s' is not a valid section, try again." % (new_section)
 404                 else:
 405                     got_section = 1
 406                     section = new_section
 407
 408         # Reset the readline completer
 409         readline.set_completer(None)
 410
 411     for f in new[index]["files"]:
 412         Upload.pkg.files[f]["section"] = section
 413         Upload.pkg.files[f]["priority"] = priority
 414     new[index]["priority"] = priority
 415     new[index]["section"] = section
 416     return new
 417
 418 ################################################################################
 419
 420 def edit_overrides (new):
 421     print
 422     done = 0
 423     while not done:
 424         print_new (new, 1)
 425         new_index = {}
 426         index = 0
 427         for i in new.keys():
 428             index += 1
 429             new_index[index] = i
 430
 431         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 432
 433         got_answer = 0
 434         while not got_answer:
 435             answer = utils.our_raw_input(prompt)
 436             if not answer.isdigit():
 437                 answer = answer[:1].upper()
 438             if answer == "E" or answer == "D":
 439                 got_answer = 1
 440             elif queue.re_isanum.match (answer):
 441                 answer = int(answer)
 442                 if (answer < 1) or (answer > index):
 443                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 444                 else:
 445                     got_answer = 1
 446
 447         if answer == 'E':
 448             edit_new(new)
 449         elif answer == 'D':
 450             done = 1
 451         else:
 452             edit_index (new, new_index[answer])
 453
 454     return new
 455
 456 ################################################################################
 457
 458 def edit_note(note):
 459     # Write the current data to a temporary file
 460     (fd, temp_filename) = utils.temp_filename()
 461     temp_file = os.fdopen(temp_filename, 'w')
 462     temp_file.write(note)
 463     temp_file.close()
 464     editor = os.environ.get("EDITOR","vi")
 465     answer = 'E'
 466     while answer == 'E':
 467         os.system("%s %s" % (editor, temp_filename))
 468         temp_file = utils.open_file(temp_filename)
 469         note = temp_file.read().rstrip()
 470         temp_file.close()
 471         print "Note:"
 472         print utils.prefix_multi_line_string(note,"  ")
 473         prompt = "[D]one, Edit, Abandon, Quit ?"
 474         answer = "XXX"
 475         while prompt.find(answer) == -1:
 476             answer = utils.our_raw_input(prompt)
 477             m = queue.re_default_answer.search(prompt)
 478             if answer == "":
 479                 answer = m.group(1)
 480             answer = answer[:1].upper()
 481     os.unlink(temp_filename)
 482     if answer == 'A':
 483         return
 484     elif answer == 'Q':
 485         end()
 486         sys.exit(0)
 487     Upload.pkg.changes["process-new note"] = note
 488     Upload.dump_vars(Cnf["Dir::Queue::New"])
 489
 490 ################################################################################
 491
 492 def check_pkg ():
 493     try:
 494         less_fd = os.popen("less -R -", 'w', 0)
 495         stdout_fd = sys.stdout
 496         try:
 497             sys.stdout = less_fd
 498             changes = utils.parse_changes (Upload.pkg.changes_file)
 499             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 500             files = Upload.pkg.files
 501             for f in files.keys():
 502                 if files[f].has_key("new"):
 503                     ftype = files[f]["type"]
 504                     if ftype == "deb":
 505                         examine_package.check_deb(changes['distribution'], f)
 506                     elif ftype == "dsc":
 507                         examine_package.check_dsc(changes['distribution'], f)
 508         finally:
 509             sys.stdout = stdout_fd
 510     except IOError, e:
 511         if e.errno == errno.EPIPE:
 512             utils.warn("[examine_package] Caught EPIPE; skipping.")
 513             pass
 514         else:
 515             raise
 516     except KeyboardInterrupt:
 517         utils.warn("[examine_package] Caught C-c; skipping.")
 518         pass
 519
 520 ################################################################################
 521
 522 ## FIXME: horribly Debian specific
 523
 524 def do_bxa_notification():
 525     files = Upload.pkg.files
 526     summary = ""
 527     for f in files.keys():
 528         if files[f]["type"] == "deb":
 529             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 530             summary += "\n"
 531             summary += "Package: %s\n" % (control.Find("Package"))
 532             summary += "Description: %s\n" % (control.Find("Description"))
 533     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 534     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 535     utils.send_mail(bxa_mail)
 536
 537 ################################################################################
 538
 539 def add_overrides (new):
 540     changes = Upload.pkg.changes
 541     files = Upload.pkg.files
 542
 543     projectB.query("BEGIN WORK")
 544     for suite in changes["suite"].keys():
 545         suite_id = database.get_suite_id(suite)
 546         for pkg in new.keys():
 547             component_id = database.get_component_id(new[pkg]["component"])
 548             type_id = database.get_override_type_id(new[pkg]["type"])
 549             priority_id = new[pkg]["priority id"]
 550             section_id = new[pkg]["section id"]
 551             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 552             for f in new[pkg]["files"]:
 553                 if files[f].has_key("new"):
 554                     del files[f]["new"]
 555             del new[pkg]
 556
 557     projectB.query("COMMIT WORK")
 558
 559     if Cnf.FindB("Dinstall::BXANotify"):
 560         do_bxa_notification()
 561
 562 ################################################################################
 563
 564 def prod_maintainer ():
 565     # Here we prepare an editor and get them ready to prod...
 566     (fd, temp_filename) = utils.temp_filename()
 567     editor = os.environ.get("EDITOR","vi")
 568     answer = 'E'
 569     while answer == 'E':
 570         os.system("%s %s" % (editor, temp_filename))
 571         f = os.fdopen(fd)
 572         prod_message = "".join(f.readlines())
 573         f.close()
 574         print "Prod message:"
 575         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 576         prompt = "[P]rod, Edit, Abandon, Quit ?"
 577         answer = "XXX"
 578         while prompt.find(answer) == -1:
 579             answer = utils.our_raw_input(prompt)
 580             m = queue.re_default_answer.search(prompt)
 581             if answer == "":
 582                 answer = m.group(1)
 583             answer = answer[:1].upper()
 584         os.unlink(temp_filename)
 585         if answer == 'A':
 586             return
 587         elif answer == 'Q':
 588             end()
 589             sys.exit(0)
 590     # Otherwise, do the proding...
 591     user_email_address = utils.whoami() + " <%s>" % (
 592         Cnf["Dinstall::MyAdminAddress"])
 593
 594     Subst = Upload.Subst
 595
 596     Subst["__FROM_ADDRESS__"] = user_email_address
 597     Subst["__PROD_MESSAGE__"] = prod_message
 598     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 599
 600     prod_mail_message = utils.TemplateSubst(
 601         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 602
 603     # Send the prod mail if appropriate
 604     if not Cnf["Dinstall::Options::No-Mail"]:
 605         utils.send_mail(prod_mail_message)
 606
 607     print "Sent proding message"
 608
 609 ################################################################################
 610
 611 def do_new():
 612     print "NEW\n"
 613     files = Upload.pkg.files
 614     changes = Upload.pkg.changes
 615
 616     # Make a copy of distribution we can happily trample on
 617     changes["suite"] = copy.copy(changes["distribution"])
 618
 619     # Fix up the list of target suites
 620     for suite in changes["suite"].keys():
 621         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 622         if override:
 623             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 624               database.get_suite_id(override) == -1)
 625             if olderr or newerr:
 626                 (oinv, newinv) = ("", "")
 627                 if olderr: oinv = "invalid "
 628                 if newerr: ninv = "invalid "
 629                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 630                         oinv, suite, ninv, override)
 631             del changes["suite"][suite]
 632             changes["suite"][override] = 1
 633     # Validate suites
 634     for suite in changes["suite"].keys():
 635         suite_id = database.get_suite_id(suite)
 636         if suite_id == -1:
 637             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 638
 639     # The main NEW processing loop
 640     done = 0
 641     while not done:
 642         # Find out what's new
 643         new = queue.determine_new(changes, files, projectB)
 644
 645         if not new:
 646             break
 647
 648         answer = "XXX"
 649         if Options["No-Action"] or Options["Automatic"]:
 650             answer = 'S'
 651
 652         (broken, note) = print_new(new, 0)
 653         prompt = ""
 654
 655         if not broken and not note:
 656             prompt = "Add overrides, "
 657         if broken:
 658             print "W: [!] marked entries must be fixed before package can be processed."
 659         if note:
 660             print "W: note must be removed before package can be processed."
 661             prompt += "Remove note, "
 662
 663         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 664
 665         while prompt.find(answer) == -1:
 666             answer = utils.our_raw_input(prompt)
 667             m = queue.re_default_answer.search(prompt)
 668             if answer == "":
 669                 answer = m.group(1)
 670             answer = answer[:1].upper()
 671
 672         if answer == 'A':
 673             done = add_overrides (new)
 674         elif answer == 'C':
 675             check_pkg()
 676         elif answer == 'E':
 677             new = edit_overrides (new)
 678         elif answer == 'M':
 679             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 680             if not aborted:
 681                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 682                 done = 1
 683         elif answer == 'N':
 684             edit_note(changes.get("process-new note", ""))
 685         elif answer == 'P':
 686             prod_maintainer()
 687         elif answer == 'R':
 688             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 689             if confirm == "y":
 690                 del changes["process-new note"]
 691         elif answer == 'S':
 692             done = 1
 693         elif answer == 'Q':
 694             end()
 695             sys.exit(0)
 696
 697 ################################################################################
 698 ################################################################################
 699 ################################################################################
 700
 701 def usage (exit_code=0):
 702     print """Usage: dak process-new [OPTION]... [CHANGES]...
 703   -a, --automatic           automatic run
 704   -h, --help                show this help and exit.
 705   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 706   -m, --manual-reject=MSG   manual reject with `msg'
 707   -n, --no-action           don't do anything
 708   -V, --version             display the version number and exit"""
 709     sys.exit(exit_code)
 710
 711 ################################################################################
 712
 713 def init():
 714     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 715
 716     Cnf = utils.get_conf()
 717
 718     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 719                  ('h',"help","Process-New::Options::Help"),
 720                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 721                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 722                  ('n',"no-action","Process-New::Options::No-Action")]
 723
 724     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 725         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 726             Cnf["Process-New::Options::%s" % (i)] = ""
 727
 728     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 729     Options = Cnf.SubTree("Process-New::Options")
 730
 731     if Options["Help"]:
 732         usage()
 733
 734     Upload = queue.Upload(Cnf)
 735
 736     if not Options["No-Action"]:
 737         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 738
 739     projectB = Upload.projectB
 740
 741     Sections = Section_Completer()
 742     Priorities = Priority_Completer()
 743     readline.parse_and_bind("tab: complete")
 744
 745     return changes_files
 746
 747 ################################################################################
 748
 749 def do_byhand():
 750     done = 0
 751     while not done:
 752         files = Upload.pkg.files
 753         will_install = 1
 754         byhand = []
 755
 756         for f in files.keys():
 757             if files[f]["type"] == "byhand":
 758                 if os.path.exists(f):
 759                     print "W: %s still present; please process byhand components and try again." % (f)
 760                     will_install = 0
 761                 else:
 762                     byhand.append(f)
 763
 764         answer = "XXXX"
 765         if Options["No-Action"]:
 766             answer = "S"
 767         if will_install:
 768             if Options["Automatic"] and not Options["No-Action"]:
 769                 answer = 'A'
 770             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 771         else:
 772             prompt = "Manual reject, [S]kip, Quit ?"
 773
 774         while prompt.find(answer) == -1:
 775             answer = utils.our_raw_input(prompt)
 776             m = queue.re_default_answer.search(prompt)
 777             if answer == "":
 778                 answer = m.group(1)
 779             answer = answer[:1].upper()
 780
 781         if answer == 'A':
 782             done = 1
 783             for f in byhand:
 784                 del files[f]
 785         elif answer == 'M':
 786             Upload.do_reject(1, Options["Manual-Reject"])
 787             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 788             done = 1
 789         elif answer == 'S':
 790             done = 1
 791         elif answer == 'Q':
 792             end()
 793             sys.exit(0)
 794
 795 ################################################################################
 796
 797 def get_accept_lock():
 798     retry = 0
 799     while retry < 10:
 800         try:
 801             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 802             retry = 10
 803         except OSError, e:
 804             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 805                 retry += 1
 806                 if (retry >= 10):
 807                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 808                 else:
 809                     print("Unable to get accepted lock (try %d of 10)" % retry)
 810                 time.sleep(60)
 811             else:
 812                 raise
 813
 814 def move_to_dir (dest, perms=0660, changesperms=0664):
 815     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 816     file_keys = Upload.pkg.files.keys()
 817     for f in file_keys:
 818         utils.move (f, dest, perms=perms)
 819
 820 def is_source_in_queue_dir(qdir):
 821     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 822                 and x.endswith(".changes") ]
 823     for entry in entries:
 824         # read the .dak
 825         u = queue.Upload(Cnf)
 826         u.pkg.changes_file = os.path.join(qdir, entry)
 827         u.update_vars()
 828         if not u.pkg.changes["architecture"].has_key("source"):
 829             # another binary upload, ignore
 830             continue
 831         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 832             # another version, ignore
 833             continue
 834         # found it!
 835         return True
 836     return False
 837
 838 def move_to_holding(suite, queue_dir):
 839     print "Moving to %s holding area." % (suite.upper(),)
 840     if Options["No-Action"]:
 841         return
 842     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 843     Upload.dump_vars(queue_dir)
 844     move_to_dir(queue_dir)
 845     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 846
 847 def _accept():
 848     if Options["No-Action"]:
 849         return
 850     (summary, short_summary) = Upload.build_summaries()
 851     Upload.accept(summary, short_summary)
 852     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 853
 854 def do_accept_stableupdate(suite, q):
 855     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 856     if not Upload.pkg.changes["architecture"].has_key("source"):
 857         # It is not a sourceful upload.  So its source may be either in p-u
 858         # holding, in new, in accepted or already installed.
 859         if is_source_in_queue_dir(queue_dir):
 860             # It's in p-u holding, so move it there.
 861             print "Binary-only upload, source in %s." % (q,)
 862             move_to_holding(suite, queue_dir)
 863         elif Upload.source_exists(Upload.pkg.changes["source"],
 864                 Upload.pkg.changes["version"]):
 865             # dak tells us that there is source available.  At time of
 866             # writing this means that it is installed, so put it into
 867             # accepted.
 868             print "Binary-only upload, source installed."
 869             _accept()
 870         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 871             # The source is in accepted, the binary cleared NEW: accept it.
 872             print "Binary-only upload, source in accepted."
 873             _accept()
 874         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 875             # It's in NEW.  We expect the source to land in p-u holding
 876             # pretty soon.
 877             print "Binary-only upload, source in new."
 878             move_to_holding(suite, queue_dir)
 879         else:
 880             # No case applicable.  Bail out.  Return will cause the upload
 881             # to be skipped.
 882             print "ERROR"
 883             print "Stable update failed.  Source not found."
 884             return
 885     else:
 886         # We are handling a sourceful upload.  Move to accepted if currently
 887         # in p-u holding and to p-u holding otherwise.
 888         if is_source_in_queue_dir(queue_dir):
 889             print "Sourceful upload in %s, accepting." % (q,)
 890             _accept()
 891         else:
 892             move_to_holding(suite, queue_dir)
 893
 894 def do_accept():
 895     print "ACCEPT"
 896     if not Options["No-Action"]:
 897         get_accept_lock()
 898         (summary, short_summary) = Upload.build_summaries()
 899     try:
 900         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 901             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 902             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 903             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 904             # Check for override disparities
 905             Upload.Subst["__SUMMARY__"] = summary
 906         else:
 907             # Stable updates need to be copied to proposed-updates holding
 908             # area instead of accepted.  Sourceful uploads need to go
 909             # to it directly, binaries only if the source has not yet been
 910             # accepted into p-u.
 911             for suite, q in [("proposed-updates", "ProposedUpdates"),
 912                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 913                 if not Upload.pkg.changes["distribution"].has_key(suite):
 914                     continue
 915                 return do_accept_stableupdate(suite, q)
 916             # Just a normal upload, accept it...
 917             _accept()
 918     finally:
 919         if not Options["No-Action"]:
 920             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 921
 922 def check_status(files):
 923     new = byhand = 0
 924     for f in files.keys():
 925         if files[f]["type"] == "byhand":
 926             byhand = 1
 927         elif files[f].has_key("new"):
 928             new = 1
 929     return (new, byhand)
 930
 931 def do_pkg(changes_file):
 932     Upload.pkg.changes_file = changes_file
 933     Upload.init_vars()
 934     Upload.update_vars()
 935     Upload.update_subst()
 936     files = Upload.pkg.files
 937
 938     if not recheck():
 939         return
 940
 941     (new, byhand) = check_status(files)
 942     if new or byhand:
 943         if new:
 944             do_new()
 945         if byhand:
 946             do_byhand()
 947         (new, byhand) = check_status(files)
 948
 949     if not new and not byhand:
 950         do_accept()
 951
 952 ################################################################################
 953
 954 def end():
 955     accept_count = Upload.accept_count
 956     accept_bytes = Upload.accept_bytes
 957
 958     if accept_count:
 959         sets = "set"
 960         if accept_count > 1:
 961             sets = "sets"
 962         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 963         Logger.log(["total",accept_count,accept_bytes])
 964
 965     if not Options["No-Action"]:
 966         Logger.close()
 967
 968 ################################################################################
 969
 970 def do_comments(dir, opref, npref, line, fn):
 971     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 972         lines = open("%s/%s" % (dir, comm)).readlines()
 973         if len(lines) == 0 or lines[0] != line + "\n": continue
 974         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 975                                 and x.endswith(".changes") ]
 976         changes_files = sort_changes(changes_files)
 977         for f in changes_files:
 978             f = utils.validate_changes_file_arg(f, 0)
 979             if not f: continue
 980             print "\n" + f
 981             fn(f, "".join(lines[1:]))
 982
 983         if opref != npref and not Options["No-Action"]:
 984             newcomm = npref + comm[len(opref):]
 985             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 986
 987 ################################################################################
 988
 989 def comment_accept(changes_file, comments):
 990     Upload.pkg.changes_file = changes_file
 991     Upload.init_vars()
 992     Upload.update_vars()
 993     Upload.update_subst()
 994     files = Upload.pkg.files
 995
 996     if not recheck():
 997         return # dak wants to REJECT, crap
 998
 999     (new, byhand) = check_status(files)
1000     if not new and not byhand:
1001         do_accept()
1002
1003 ################################################################################
1004
1005 def comment_reject(changes_file, comments):
1006     Upload.pkg.changes_file = changes_file
1007     Upload.init_vars()
1008     Upload.update_vars()
1009     Upload.update_subst()
1010
1011     if not recheck():
1012         pass # dak has its own reasons to reject as well, which is fine
1013
1014     reject(comments)
1015     print "REJECT\n" + reject_message,
1016     if not Options["No-Action"]:
1017         Upload.do_reject(0, reject_message)
1018         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1019
1020 ################################################################################
1021
1022 def main():
1023     changes_files = init()
1024     if len(changes_files) > 50:
1025         sys.stderr.write("Sorting changes...\n")
1026     changes_files = sort_changes(changes_files)
1027
1028     # Kill me now? **FIXME**
1029     Cnf["Dinstall::Options::No-Mail"] = ""
1030     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1031     if Cnf.has_key("Dinstall::Bcc"):
1032         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1033     else:
1034         Upload.Subst["__BCC__"] = bcc
1035
1036     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1037     if commentsdir:
1038         if changes_files != []:
1039             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1040             sys.exit(1)
1041         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1042         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1043     else:
1044         for changes_file in changes_files:
1045             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1046             if not changes_file:
1047                 continue
1048             print "\n" + changes_file
1049             do_pkg (changes_file)
1050
1051     end()
1052
1053 ################################################################################
1054
1055 if __name__ == '__main__':
1056     main()