dak/process_new.py

   1 #!/usr/bin/env python
   2
   3 # Handles NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # 23:12|<aj> I will not hush!
  23 # 23:12|<elmo> :>
  24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  25 # 23:13|<aj> I shall not be silenced!
  26 # 23:13|<aj> The world shall know!
  27 # 23:13|<aj> The world *must* know!
  28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  29 # 23:13|<aj> yay powerpuff girls!!
  30 # 23:13|<aj> buttercup's my favourite, who's yours?
  31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  32 # 23:14|<aj> *AREN'T YOU*?!
  33 # 23:15|<aj> I will not be treated like this.
  34 # 23:15|<aj> I shall have my revenge.
  35 # 23:15|<aj> I SHALL!!!
  36
  37 ################################################################################
  38
  39 import copy, errno, os, readline, stat, sys, time
  40 import apt_pkg, apt_inst
  41 import examine_package
  42 import daklib.database
  43 import daklib.logging
  44 import daklib.queue
  45 import daklib.utils
  46
  47 # Globals
  48 Cnf = None
  49 Options = None
  50 Upload = None
  51 projectB = None
  52 Logger = None
  53
  54 Priorities = None
  55 Sections = None
  56
  57 reject_message = ""
  58
  59 ################################################################################
  60 ################################################################################
  61 ################################################################################
  62
  63 def reject (str, prefix="Rejected: "):
  64     global reject_message
  65     if str:
  66         reject_message += prefix + str + "\n"
  67
  68 def recheck():
  69     global reject_message
  70     files = Upload.pkg.files
  71     reject_message = ""
  72
  73     for file in files.keys():
  74         # The .orig.tar.gz can disappear out from under us is it's a
  75         # duplicate of one in the archive.
  76         if not files.has_key(file):
  77             continue
  78         # Check that the source still exists
  79         if files[file]["type"] == "deb":
  80             source_version = files[file]["source version"]
  81             source_package = files[file]["source package"]
  82             if not Upload.pkg.changes["architecture"].has_key("source") \
  83                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  84                 source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
  85                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  86                 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
  87                     reject("no source found for %s %s (%s)." % (source_package, source_version, file))
  88
  89         # Version and file overwrite checks
  90         if files[file]["type"] == "deb":
  91             reject(Upload.check_binary_against_db(file))
  92         elif files[file]["type"] == "dsc":
  93             reject(Upload.check_source_against_db(file))
  94             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
  95             reject(reject_msg)
  96
  97     if reject_message:
  98         answer = "XXX"
  99         if Options["No-Action"] or Options["Automatic"]:
 100             answer = 'S'
 101
 102         print "REJECT\n" + reject_message,
 103         prompt = "[R]eject, Skip, Quit ?"
 104
 105         while prompt.find(answer) == -1:
 106             answer = daklib.utils.our_raw_input(prompt)
 107             m = daklib.queue.re_default_answer.match(prompt)
 108             if answer == "":
 109                 answer = m.group(1)
 110             answer = answer[:1].upper()
 111
 112         if answer == 'R':
 113             Upload.do_reject(0, reject_message)
 114             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 115             return 0
 116         elif answer == 'S':
 117             return 0
 118         elif answer == 'Q':
 119             sys.exit(0)
 120
 121     return 1
 122
 123 ################################################################################
 124
 125 def determine_new (changes, files):
 126     new = {}
 127
 128     # Build up a list of potentially new things
 129     for file in files.keys():
 130         f = files[file]
 131         # Skip byhand elements
 132         if f["type"] == "byhand":
 133             continue
 134         pkg = f["package"]
 135         priority = f["priority"]
 136         section = f["section"]
 137         # FIXME: unhardcode
 138         if section == "non-US/main":
 139             section = "non-US"
 140         type = get_type(f)
 141         component = f["component"]
 142
 143         if type == "dsc":
 144             priority = "source"
 145         if not new.has_key(pkg):
 146             new[pkg] = {}
 147             new[pkg]["priority"] = priority
 148             new[pkg]["section"] = section
 149             new[pkg]["type"] = type
 150             new[pkg]["component"] = component
 151             new[pkg]["files"] = []
 152         else:
 153             old_type = new[pkg]["type"]
 154             if old_type != type:
 155                 # source gets trumped by deb or udeb
 156                 if old_type == "dsc":
 157                     new[pkg]["priority"] = priority
 158                     new[pkg]["section"] = section
 159                     new[pkg]["type"] = type
 160                     new[pkg]["component"] = component
 161         new[pkg]["files"].append(file)
 162         if f.has_key("othercomponents"):
 163             new[pkg]["othercomponents"] = f["othercomponents"]
 164
 165     for suite in changes["suite"].keys():
 166         suite_id = daklib.database.get_suite_id(suite)
 167         for pkg in new.keys():
 168             component_id = daklib.database.get_component_id(new[pkg]["component"])
 169             type_id = daklib.database.get_override_type_id(new[pkg]["type"])
 170             q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
 171             ql = q.getresult()
 172             if ql:
 173                 for file in new[pkg]["files"]:
 174                     if files[file].has_key("new"):
 175                         del files[file]["new"]
 176                 del new[pkg]
 177
 178     if changes["suite"].has_key("stable"):
 179         print "WARNING: overrides will be added for stable!"
 180     if changes["suite"].has_key("oldstable"):
 181         print "WARNING: overrides will be added for OLDstable!"
 182     for pkg in new.keys():
 183         if new[pkg].has_key("othercomponents"):
 184             print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 185
 186     return new
 187
 188 ################################################################################
 189
 190 def indiv_sg_compare (a, b):
 191     """Sort by source name, source, version, 'have source', and
 192        finally by filename."""
 193     # Sort by source version
 194     q = apt_pkg.VersionCompare(a["version"], b["version"])
 195     if q:
 196         return -q
 197
 198     # Sort by 'have source'
 199     a_has_source = a["architecture"].get("source")
 200     b_has_source = b["architecture"].get("source")
 201     if a_has_source and not b_has_source:
 202         return -1
 203     elif b_has_source and not a_has_source:
 204         return 1
 205
 206     return cmp(a["filename"], b["filename"])
 207
 208 ############################################################
 209
 210 def sg_compare (a, b):
 211     a = a[1]
 212     b = b[1]
 213     """Sort by have note, time of oldest upload."""
 214     # Sort by have note
 215     a_note_state = a["note_state"]
 216     b_note_state = b["note_state"]
 217     if a_note_state < b_note_state:
 218         return -1
 219     elif a_note_state > b_note_state:
 220         return 1
 221
 222     # Sort by time of oldest upload
 223     return cmp(a["oldest"], b["oldest"])
 224
 225 def sort_changes(changes_files):
 226     """Sort into source groups, then sort each source group by version,
 227     have source, filename.  Finally, sort the source groups by have
 228     note, time of oldest upload of each source upload."""
 229     if len(changes_files) == 1:
 230         return changes_files
 231
 232     sorted_list = []
 233     cache = {}
 234     # Read in all the .changes files
 235     for filename in changes_files:
 236         try:
 237             Upload.pkg.changes_file = filename
 238             Upload.init_vars()
 239             Upload.update_vars()
 240             cache[filename] = copy.copy(Upload.pkg.changes)
 241             cache[filename]["filename"] = filename
 242         except:
 243             sorted_list.append(filename)
 244             break
 245     # Divide the .changes into per-source groups
 246     per_source = {}
 247     for filename in cache.keys():
 248         source = cache[filename]["source"]
 249         if not per_source.has_key(source):
 250             per_source[source] = {}
 251             per_source[source]["list"] = []
 252         per_source[source]["list"].append(cache[filename])
 253     # Determine oldest time and have note status for each source group
 254     for source in per_source.keys():
 255         source_list = per_source[source]["list"]
 256         first = source_list[0]
 257         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 258         have_note = 0
 259         for d in per_source[source]["list"]:
 260             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 261             if mtime < oldest:
 262                 oldest = mtime
 263             have_note += (d.has_key("process-new note"))
 264         per_source[source]["oldest"] = oldest
 265         if not have_note:
 266             per_source[source]["note_state"] = 0; # none
 267         elif have_note < len(source_list):
 268             per_source[source]["note_state"] = 1; # some
 269         else:
 270             per_source[source]["note_state"] = 2; # all
 271         per_source[source]["list"].sort(indiv_sg_compare)
 272     per_source_items = per_source.items()
 273     per_source_items.sort(sg_compare)
 274     for i in per_source_items:
 275         for j in i[1]["list"]:
 276             sorted_list.append(j["filename"])
 277     return sorted_list
 278
 279 ################################################################################
 280
 281 class Section_Completer:
 282     def __init__ (self):
 283         self.sections = []
 284         q = projectB.query("SELECT section FROM section")
 285         for i in q.getresult():
 286             self.sections.append(i[0])
 287
 288     def complete(self, text, state):
 289         if state == 0:
 290             self.matches = []
 291             n = len(text)
 292             for word in self.sections:
 293                 if word[:n] == text:
 294                     self.matches.append(word)
 295         try:
 296             return self.matches[state]
 297         except IndexError:
 298             return None
 299
 300 ############################################################
 301
 302 class Priority_Completer:
 303     def __init__ (self):
 304         self.priorities = []
 305         q = projectB.query("SELECT priority FROM priority")
 306         for i in q.getresult():
 307             self.priorities.append(i[0])
 308
 309     def complete(self, text, state):
 310         if state == 0:
 311             self.matches = []
 312             n = len(text)
 313             for word in self.priorities:
 314                 if word[:n] == text:
 315                     self.matches.append(word)
 316         try:
 317             return self.matches[state]
 318         except IndexError:
 319             return None
 320
 321 ################################################################################
 322
 323 def check_valid (new):
 324     for pkg in new.keys():
 325         section = new[pkg]["section"]
 326         priority = new[pkg]["priority"]
 327         type = new[pkg]["type"]
 328         new[pkg]["section id"] = daklib.database.get_section_id(section)
 329         new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"])
 330         # Sanity checks
 331         di = section.find("debian-installer") != -1
 332         if (di and type != "udeb") or (not di and type == "udeb"):
 333             new[pkg]["section id"] = -1
 334         if (priority == "source" and type != "dsc") or \
 335            (priority != "source" and type == "dsc"):
 336             new[pkg]["priority id"] = -1
 337
 338 ################################################################################
 339
 340 def print_new (new, indexed, file=sys.stdout):
 341     check_valid(new)
 342     broken = 0
 343     index = 0
 344     for pkg in new.keys():
 345         index += 1
 346         section = new[pkg]["section"]
 347         priority = new[pkg]["priority"]
 348         if new[pkg]["section id"] == -1:
 349             section += "[!]"
 350             broken = 1
 351         if new[pkg]["priority id"] == -1:
 352             priority += "[!]"
 353             broken = 1
 354         if indexed:
 355             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 356         else:
 357             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 358         line = line.strip()+'\n'
 359         file.write(line)
 360     note = Upload.pkg.changes.get("process-new note")
 361     if note:
 362         print "*"*75
 363         print note
 364         print "*"*75
 365     return broken, note
 366
 367 ################################################################################
 368
 369 def get_type (f):
 370     # Determine the type
 371     if f.has_key("dbtype"):
 372         type = f["dbtype"]
 373     elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
 374         type = "dsc"
 375     else:
 376         daklib.utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (type))
 377
 378     # Validate the override type
 379     type_id = daklib.database.get_override_type_id(type)
 380     if type_id == -1:
 381         daklib.utils.fubar("invalid type (%s) for new.  Say wha?" % (type))
 382
 383     return type
 384
 385 ################################################################################
 386
 387 def index_range (index):
 388     if index == 1:
 389         return "1"
 390     else:
 391         return "1-%s" % (index)
 392
 393 ################################################################################
 394 ################################################################################
 395
 396 def edit_new (new):
 397     # Write the current data to a temporary file
 398     temp_filename = daklib.utils.temp_filename()
 399     temp_file = daklib.utils.open_file(temp_filename, 'w')
 400     print_new (new, 0, temp_file)
 401     temp_file.close()
 402     # Spawn an editor on that file
 403     editor = os.environ.get("EDITOR","vi")
 404     result = os.system("%s %s" % (editor, temp_filename))
 405     if result != 0:
 406         daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 407     # Read the edited data back in
 408     temp_file = daklib.utils.open_file(temp_filename)
 409     lines = temp_file.readlines()
 410     temp_file.close()
 411     os.unlink(temp_filename)
 412     # Parse the new data
 413     for line in lines:
 414         line = line.strip()
 415         if line == "":
 416             continue
 417         s = line.split()
 418         # Pad the list if necessary
 419         s[len(s):3] = [None] * (3-len(s))
 420         (pkg, priority, section) = s[:3]
 421         if not new.has_key(pkg):
 422             daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
 423         else:
 424             # Strip off any invalid markers, print_new will readd them.
 425             if section.endswith("[!]"):
 426                 section = section[:-3]
 427             if priority.endswith("[!]"):
 428                 priority = priority[:-3]
 429             for file in new[pkg]["files"]:
 430                 Upload.pkg.files[file]["section"] = section
 431                 Upload.pkg.files[file]["priority"] = priority
 432             new[pkg]["section"] = section
 433             new[pkg]["priority"] = priority
 434
 435 ################################################################################
 436
 437 def edit_index (new, index):
 438     priority = new[index]["priority"]
 439     section = new[index]["section"]
 440     type = new[index]["type"]
 441     done = 0
 442     while not done:
 443         print "\t".join([index, priority, section])
 444
 445         answer = "XXX"
 446         if type != "dsc":
 447             prompt = "[B]oth, Priority, Section, Done ? "
 448         else:
 449             prompt = "[S]ection, Done ? "
 450         edit_priority = edit_section = 0
 451
 452         while prompt.find(answer) == -1:
 453             answer = daklib.utils.our_raw_input(prompt)
 454             m = daklib.queue.re_default_answer.match(prompt)
 455             if answer == "":
 456                 answer = m.group(1)
 457             answer = answer[:1].upper()
 458
 459         if answer == 'P':
 460             edit_priority = 1
 461         elif answer == 'S':
 462             edit_section = 1
 463         elif answer == 'B':
 464             edit_priority = edit_section = 1
 465         elif answer == 'D':
 466             done = 1
 467
 468         # Edit the priority
 469         if edit_priority:
 470             readline.set_completer(Priorities.complete)
 471             got_priority = 0
 472             while not got_priority:
 473                 new_priority = daklib.utils.our_raw_input("New priority: ").strip()
 474                 if new_priority not in Priorities.priorities:
 475                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 476                 else:
 477                     got_priority = 1
 478                     priority = new_priority
 479
 480         # Edit the section
 481         if edit_section:
 482             readline.set_completer(Sections.complete)
 483             got_section = 0
 484             while not got_section:
 485                 new_section = daklib.utils.our_raw_input("New section: ").strip()
 486                 if new_section not in Sections.sections:
 487                     print "E: '%s' is not a valid section, try again." % (new_section)
 488                 else:
 489                     got_section = 1
 490                     section = new_section
 491
 492         # Reset the readline completer
 493         readline.set_completer(None)
 494
 495     for file in new[index]["files"]:
 496         Upload.pkg.files[file]["section"] = section
 497         Upload.pkg.files[file]["priority"] = priority
 498     new[index]["priority"] = priority
 499     new[index]["section"] = section
 500     return new
 501
 502 ################################################################################
 503
 504 def edit_overrides (new):
 505     print
 506     done = 0
 507     while not done:
 508         print_new (new, 1)
 509         new_index = {}
 510         index = 0
 511         for i in new.keys():
 512             index += 1
 513             new_index[index] = i
 514
 515         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 516
 517         got_answer = 0
 518         while not got_answer:
 519             answer = daklib.utils.our_raw_input(prompt)
 520             if not answer.isdigit():
 521                 answer = answer[:1].upper()
 522             if answer == "E" or answer == "D":
 523                 got_answer = 1
 524             elif daklib.queue.re_isanum.match (answer):
 525                 answer = int(answer)
 526                 if (answer < 1) or (answer > index):
 527                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 528                 else:
 529                     got_answer = 1
 530
 531         if answer == 'E':
 532             edit_new(new)
 533         elif answer == 'D':
 534             done = 1
 535         else:
 536             edit_index (new, new_index[answer])
 537
 538     return new
 539
 540 ################################################################################
 541
 542 def edit_note(note):
 543     # Write the current data to a temporary file
 544     temp_filename = daklib.utils.temp_filename()
 545     temp_file = daklib.utils.open_file(temp_filename, 'w')
 546     temp_file.write(note)
 547     temp_file.close()
 548     editor = os.environ.get("EDITOR","vi")
 549     answer = 'E'
 550     while answer == 'E':
 551         os.system("%s %s" % (editor, temp_filename))
 552         temp_file = daklib.utils.open_file(temp_filename)
 553         note = temp_file.read().rstrip()
 554         temp_file.close()
 555         print "Note:"
 556         print daklib.utils.prefix_multi_line_string(note,"  ")
 557         prompt = "[D]one, Edit, Abandon, Quit ?"
 558         answer = "XXX"
 559         while prompt.find(answer) == -1:
 560             answer = daklib.utils.our_raw_input(prompt)
 561             m = daklib.queue.re_default_answer.search(prompt)
 562             if answer == "":
 563                 answer = m.group(1)
 564             answer = answer[:1].upper()
 565     os.unlink(temp_filename)
 566     if answer == 'A':
 567         return
 568     elif answer == 'Q':
 569         sys.exit(0)
 570     Upload.pkg.changes["process-new note"] = note
 571     Upload.dump_vars(Cnf["Dir::Queue::New"])
 572
 573 ################################################################################
 574
 575 def check_pkg ():
 576     try:
 577         less_fd = os.popen("less -R -", 'w', 0)
 578         stdout_fd = sys.stdout
 579         try:
 580             sys.stdout = less_fd
 581             examine_package.display_changes(Upload.pkg.changes_file)
 582             files = Upload.pkg.files
 583             for file in files.keys():
 584                 if files[file].has_key("new"):
 585                     type = files[file]["type"]
 586                     if type == "deb":
 587                         examine_package.check_deb(file)
 588                     elif type == "dsc":
 589                         examine_package.check_dsc(file)
 590         finally:
 591             sys.stdout = stdout_fd
 592     except IOError, e:
 593         if errno.errorcode[e.errno] == 'EPIPE':
 594             daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
 595             pass
 596         else:
 597             raise
 598     except KeyboardInterrupt:
 599         daklib.utils.warn("[examine_package] Caught C-c; skipping.")
 600         pass
 601
 602 ################################################################################
 603
 604 ## FIXME: horribly Debian specific
 605
 606 def do_bxa_notification():
 607     files = Upload.pkg.files
 608     summary = ""
 609     for file in files.keys():
 610         if files[file]["type"] == "deb":
 611             control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
 612             summary += "\n"
 613             summary += "Package: %s\n" % (control.Find("Package"))
 614             summary += "Description: %s\n" % (control.Find("Description"))
 615     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 616     bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 617     daklib.utils.send_mail(bxa_mail)
 618
 619 ################################################################################
 620
 621 def add_overrides (new):
 622     changes = Upload.pkg.changes
 623     files = Upload.pkg.files
 624
 625     projectB.query("BEGIN WORK")
 626     for suite in changes["suite"].keys():
 627         suite_id = daklib.database.get_suite_id(suite)
 628         for pkg in new.keys():
 629             component_id = daklib.database.get_component_id(new[pkg]["component"])
 630             type_id = daklib.database.get_override_type_id(new[pkg]["type"])
 631             priority_id = new[pkg]["priority id"]
 632             section_id = new[pkg]["section id"]
 633             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 634             for file in new[pkg]["files"]:
 635                 if files[file].has_key("new"):
 636                     del files[file]["new"]
 637             del new[pkg]
 638
 639     projectB.query("COMMIT WORK")
 640
 641     if Cnf.FindB("Dinstall::BXANotify"):
 642         do_bxa_notification()
 643
 644 ################################################################################
 645
 646 def prod_maintainer ():
 647     # Here we prepare an editor and get them ready to prod...
 648     temp_filename = daklib.utils.temp_filename()
 649     editor = os.environ.get("EDITOR","vi")
 650     answer = 'E'
 651     while answer == 'E':
 652         os.system("%s %s" % (editor, temp_filename))
 653         file = daklib.utils.open_file(temp_filename)
 654         prod_message = "".join(file.readlines())
 655         file.close()
 656         print "Prod message:"
 657         print daklib.utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 658         prompt = "[P]rod, Edit, Abandon, Quit ?"
 659         answer = "XXX"
 660         while prompt.find(answer) == -1:
 661             answer = daklib.utils.our_raw_input(prompt)
 662             m = daklib.queue.re_default_answer.search(prompt)
 663             if answer == "":
 664                 answer = m.group(1)
 665             answer = answer[:1].upper()
 666         os.unlink(temp_filename)
 667         if answer == 'A':
 668             return
 669         elif answer == 'Q':
 670             sys.exit(0)
 671     # Otherwise, do the proding...
 672     user_email_address = daklib.utils.whoami() + " <%s>" % (
 673         Cnf["Dinstall::MyAdminAddress"])
 674
 675     Subst = Upload.Subst
 676
 677     Subst["__FROM_ADDRESS__"] = user_email_address
 678     Subst["__PROD_MESSAGE__"] = prod_message
 679     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 680
 681     prod_mail_message = daklib.utils.TemplateSubst(
 682         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 683
 684     # Send the prod mail if appropriate
 685     if not Cnf["Dinstall::Options::No-Mail"]:
 686         daklib.utils.send_mail(prod_mail_message)
 687
 688     print "Sent proding message"
 689
 690 ################################################################################
 691
 692 def do_new():
 693     print "NEW\n"
 694     files = Upload.pkg.files
 695     changes = Upload.pkg.changes
 696
 697     # Make a copy of distribution we can happily trample on
 698     changes["suite"] = copy.copy(changes["distribution"])
 699
 700     # Fix up the list of target suites
 701     for suite in changes["suite"].keys():
 702         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 703         if override:
 704             (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1,
 705               daklib.database.get_suite_id(override) == -1)
 706             if olderr or newerr:
 707                 (oinv, newinv) = ("", "")
 708                 if olderr: oinv = "invalid "
 709                 if newerr: ninv = "invalid "
 710                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 711                         oinv, suite, ninv, override)
 712             del changes["suite"][suite]
 713             changes["suite"][override] = 1
 714     # Validate suites
 715     for suite in changes["suite"].keys():
 716         suite_id = daklib.database.get_suite_id(suite)
 717         if suite_id == -1:
 718             daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 719
 720     # The main NEW processing loop
 721     done = 0
 722     while not done:
 723         # Find out what's new
 724         new = determine_new(changes, files)
 725
 726         if not new:
 727             break
 728
 729         answer = "XXX"
 730         if Options["No-Action"] or Options["Automatic"]:
 731             answer = 'S'
 732
 733         (broken, note) = print_new(new, 0)
 734         prompt = ""
 735
 736         if not broken and not note:
 737             prompt = "Add overrides, "
 738         if broken:
 739             print "W: [!] marked entries must be fixed before package can be processed."
 740         if note:
 741             print "W: note must be removed before package can be processed."
 742             prompt += "Remove note, "
 743
 744         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 745
 746         while prompt.find(answer) == -1:
 747             answer = daklib.utils.our_raw_input(prompt)
 748             m = daklib.queue.re_default_answer.search(prompt)
 749             if answer == "":
 750                 answer = m.group(1)
 751             answer = answer[:1].upper()
 752
 753         if answer == 'A':
 754             done = add_overrides (new)
 755         elif answer == 'C':
 756             check_pkg()
 757         elif answer == 'E':
 758             new = edit_overrides (new)
 759         elif answer == 'M':
 760             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 761             if not aborted:
 762                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 763                 done = 1
 764         elif answer == 'N':
 765             edit_note(changes.get("process-new note", ""))
 766         elif answer == 'P':
 767             prod_maintainer()
 768         elif answer == 'R':
 769             confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
 770             if confirm == "y":
 771                 del changes["process-new note"]
 772         elif answer == 'S':
 773             done = 1
 774         elif answer == 'Q':
 775             sys.exit(0)
 776
 777 ################################################################################
 778 ################################################################################
 779 ################################################################################
 780
 781 def usage (exit_code=0):
 782     print """Usage: dak process-new [OPTION]... [CHANGES]...
 783   -a, --automatic           automatic run
 784   -h, --help                show this help and exit.
 785   -m, --manual-reject=MSG   manual reject with `msg'
 786   -n, --no-action           don't do anything
 787   -V, --version             display the version number and exit"""
 788     sys.exit(exit_code)
 789
 790 ################################################################################
 791
 792 def init():
 793     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 794
 795     Cnf = daklib.utils.get_conf()
 796
 797     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 798                  ('h',"help","Process-New::Options::Help"),
 799                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 800                  ('n',"no-action","Process-New::Options::No-Action")]
 801
 802     for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
 803         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 804             Cnf["Process-New::Options::%s" % (i)] = ""
 805
 806     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 807     Options = Cnf.SubTree("Process-New::Options")
 808
 809     if Options["Help"]:
 810         usage()
 811
 812     Upload = daklib.queue.Upload(Cnf)
 813
 814     if not Options["No-Action"]:
 815         Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
 816
 817     projectB = Upload.projectB
 818
 819     Sections = Section_Completer()
 820     Priorities = Priority_Completer()
 821     readline.parse_and_bind("tab: complete")
 822
 823     return changes_files
 824
 825 ################################################################################
 826
 827 def do_byhand():
 828     done = 0
 829     while not done:
 830         files = Upload.pkg.files
 831         will_install = 1
 832         byhand = []
 833
 834         for file in files.keys():
 835             if files[file]["type"] == "byhand":
 836                 if os.path.exists(file):
 837                     print "W: %s still present; please process byhand components and try again." % (file)
 838                     will_install = 0
 839                 else:
 840                     byhand.append(file)
 841
 842         answer = "XXXX"
 843         if Options["No-Action"]:
 844             answer = "S"
 845         if will_install:
 846             if Options["Automatic"] and not Options["No-Action"]:
 847                 answer = 'A'
 848             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 849         else:
 850             prompt = "Manual reject, [S]kip, Quit ?"
 851
 852         while prompt.find(answer) == -1:
 853             answer = daklib.utils.our_raw_input(prompt)
 854             m = daklib.queue.re_default_answer.search(prompt)
 855             if answer == "":
 856                 answer = m.group(1)
 857             answer = answer[:1].upper()
 858
 859         if answer == 'A':
 860             done = 1
 861             for file in byhand:
 862                 del files[file]
 863         elif answer == 'M':
 864             Upload.do_reject(1, Options["Manual-Reject"])
 865             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 866             done = 1
 867         elif answer == 'S':
 868             done = 1
 869         elif answer == 'Q':
 870             sys.exit(0)
 871
 872 ################################################################################
 873
 874 def do_accept():
 875     print "ACCEPT"
 876     if not Options["No-Action"]:
 877         retry = 0
 878         while retry < 10:
 879             try:
 880                 lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 881                 retry = 10
 882             except OSError, e:
 883                 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
 884                     retry += 1
 885                     if (retry >= 10):
 886                         daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 887                     else:
 888                         print("Unable to get accepted lock (try %d of 10)" % retry)
 889                     time.sleep(60)
 890                 else:
 891                     raise
 892         (summary, short_summary) = Upload.build_summaries()
 893         Upload.accept(summary, short_summary)
 894         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 895         os.unlink(Cnf["Process-New::AcceptedLockFile"])
 896
 897 def check_status(files):
 898     new = byhand = 0
 899     for file in files.keys():
 900         if files[file]["type"] == "byhand":
 901             byhand = 1
 902         elif files[file].has_key("new"):
 903             new = 1
 904     return (new, byhand)
 905
 906 def do_pkg(changes_file):
 907     Upload.pkg.changes_file = changes_file
 908     Upload.init_vars()
 909     Upload.update_vars()
 910     Upload.update_subst()
 911     files = Upload.pkg.files
 912
 913     if not recheck():
 914         return
 915
 916     (new, byhand) = check_status(files)
 917     if new or byhand:
 918         if new:
 919             do_new()
 920         if byhand:
 921             do_byhand()
 922         (new, byhand) = check_status(files)
 923
 924     if not new and not byhand:
 925         do_accept()
 926
 927 ################################################################################
 928
 929 def end():
 930     accept_count = Upload.accept_count
 931     accept_bytes = Upload.accept_bytes
 932
 933     if accept_count:
 934         sets = "set"
 935         if accept_count > 1:
 936             sets = "sets"
 937         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
 938         Logger.log(["total",accept_count,accept_bytes])
 939
 940     if not Options["No-Action"]:
 941         Logger.close()
 942
 943 ################################################################################
 944
 945 def main():
 946     changes_files = init()
 947     if len(changes_files) > 50:
 948         sys.stderr.write("Sorting changes...\n")
 949     changes_files = sort_changes(changes_files)
 950
 951     # Kill me now? **FIXME**
 952     Cnf["Dinstall::Options::No-Mail"] = ""
 953     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
 954     if Cnf.has_key("Dinstall::Bcc"):
 955         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
 956     else:
 957         Upload.Subst["__BCC__"] = bcc
 958
 959     for changes_file in changes_files:
 960         changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
 961         if not changes_file:
 962             continue
 963         print "\n" + changes_file
 964         do_pkg (changes_file)
 965
 966     end()
 967
 968 ################################################################################
 969
 970 if __name__ == '__main__':
 971     main()