dak/process_new.py

   1 #!/usr/bin/env python
   2
   3 # Handles NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # 23:12|<aj> I will not hush!
  23 # 23:12|<elmo> :>
  24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  25 # 23:13|<aj> I shall not be silenced!
  26 # 23:13|<aj> The world shall know!
  27 # 23:13|<aj> The world *must* know!
  28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  29 # 23:13|<aj> yay powerpuff girls!!
  30 # 23:13|<aj> buttercup's my favourite, who's yours?
  31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  32 # 23:14|<aj> *AREN'T YOU*?!
  33 # 23:15|<aj> I will not be treated like this.
  34 # 23:15|<aj> I shall have my revenge.
  35 # 23:15|<aj> I SHALL!!!
  36
  37 ################################################################################
  38
  39 import copy, errno, os, readline, stat, sys, time
  40 import apt_pkg, apt_inst
  41 import examine_package
  42 import daklib.database
  43 import daklib.logging
  44 import daklib.queue
  45 import daklib.utils
  46
  47 # Globals
  48 Cnf = None
  49 Options = None
  50 Upload = None
  51 projectB = None
  52 Logger = None
  53
  54 Priorities = None
  55 Sections = None
  56
  57 reject_message = ""
  58
  59 ################################################################################
  60 ################################################################################
  61 ################################################################################
  62
  63 def reject (str, prefix="Rejected: "):
  64     global reject_message
  65     if str:
  66         reject_message += prefix + str + "\n"
  67
  68 def recheck():
  69     global reject_message
  70     files = Upload.pkg.files
  71     reject_message = ""
  72
  73     for file in files.keys():
  74         # The .orig.tar.gz can disappear out from under us is it's a
  75         # duplicate of one in the archive.
  76         if not files.has_key(file):
  77             continue
  78         # Check that the source still exists
  79         if files[file]["type"] == "deb":
  80             source_version = files[file]["source version"]
  81             source_package = files[file]["source package"]
  82             if not Upload.pkg.changes["architecture"].has_key("source") \
  83                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  84                 source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
  85                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  86                 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
  87                     reject("no source found for %s %s (%s)." % (source_package, source_version, file))
  88
  89         # Version and file overwrite checks
  90         if files[file]["type"] == "deb":
  91             reject(Upload.check_binary_against_db(file))
  92         elif files[file]["type"] == "dsc":
  93             reject(Upload.check_source_against_db(file))
  94             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
  95             reject(reject_msg)
  96
  97     if reject_message:
  98         answer = "XXX"
  99         if Options["No-Action"] or Options["Automatic"]:
 100             answer = 'S'
 101
 102         print "REJECT\n" + reject_message,
 103         prompt = "[R]eject, Skip, Quit ?"
 104
 105         while prompt.find(answer) == -1:
 106             answer = daklib.utils.our_raw_input(prompt)
 107             m = daklib.queue.re_default_answer.match(prompt)
 108             if answer == "":
 109                 answer = m.group(1)
 110             answer = answer[:1].upper()
 111
 112         if answer == 'R':
 113             Upload.do_reject(0, reject_message)
 114             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 115             return 0
 116         elif answer == 'S':
 117             return 0
 118         elif answer == 'Q':
 119             sys.exit(0)
 120
 121     return 1
 122
 123 ################################################################################
 124
 125 def determine_new (changes, files):
 126     new = {}
 127
 128     # Build up a list of potentially new things
 129     for file in files.keys():
 130         f = files[file]
 131         # Skip byhand elements
 132         if f["type"] == "byhand":
 133             continue
 134         pkg = f["package"]
 135         priority = f["priority"]
 136         section = f["section"]
 137         # FIXME: unhardcode
 138         if section == "non-US/main":
 139             section = "non-US"
 140         type = get_type(f)
 141         component = f["component"]
 142
 143         if type == "dsc":
 144             priority = "source"
 145         if not new.has_key(pkg):
 146             new[pkg] = {}
 147             new[pkg]["priority"] = priority
 148             new[pkg]["section"] = section
 149             new[pkg]["type"] = type
 150             new[pkg]["component"] = component
 151             new[pkg]["files"] = []
 152         else:
 153             old_type = new[pkg]["type"]
 154             if old_type != type:
 155                 # source gets trumped by deb or udeb
 156                 if old_type == "dsc":
 157                     new[pkg]["priority"] = priority
 158                     new[pkg]["section"] = section
 159                     new[pkg]["type"] = type
 160                     new[pkg]["component"] = component
 161         new[pkg]["files"].append(file)
 162         if f.has_key("othercomponents"):
 163             new[pkg]["othercomponents"] = f["othercomponents"]
 164
 165     for suite in changes["suite"].keys():
 166         suite_id = daklib.database.get_suite_id(suite)
 167         for pkg in new.keys():
 168             component_id = daklib.database.get_component_id(new[pkg]["component"])
 169             type_id = daklib.database.get_override_type_id(new[pkg]["type"])
 170             q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
 171             ql = q.getresult()
 172             if ql:
 173                 for file in new[pkg]["files"]:
 174                     if files[file].has_key("new"):
 175                         del files[file]["new"]
 176                 del new[pkg]
 177
 178     if changes["suite"].has_key("stable"):
 179         print "WARNING: overrides will be added for stable!"
 180     if changes["suite"].has_key("oldstable"):
 181         print "WARNING: overrides will be added for OLDstable!"
 182     for pkg in new.keys():
 183         if new[pkg].has_key("othercomponents"):
 184             print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 185
 186     return new
 187
 188 ################################################################################
 189
 190 def indiv_sg_compare (a, b):
 191     """Sort by source name, source, version, 'have source', and
 192        finally by filename."""
 193     # Sort by source version
 194     q = apt_pkg.VersionCompare(a["version"], b["version"])
 195     if q:
 196         return -q
 197
 198     # Sort by 'have source'
 199     a_has_source = a["architecture"].get("source")
 200     b_has_source = b["architecture"].get("source")
 201     if a_has_source and not b_has_source:
 202         return -1
 203     elif b_has_source and not a_has_source:
 204         return 1
 205
 206     return cmp(a["filename"], b["filename"])
 207
 208 ############################################################
 209
 210 def sg_compare (a, b):
 211     a = a[1]
 212     b = b[1]
 213     """Sort by have note, time of oldest upload."""
 214     # Sort by have note
 215     a_note_state = a["note_state"]
 216     b_note_state = b["note_state"]
 217     if a_note_state < b_note_state:
 218         return -1
 219     elif a_note_state > b_note_state:
 220         return 1
 221
 222     # Sort by time of oldest upload
 223     return cmp(a["oldest"], b["oldest"])
 224
 225 def sort_changes(changes_files):
 226     """Sort into source groups, then sort each source group by version,
 227     have source, filename.  Finally, sort the source groups by have
 228     note, time of oldest upload of each source upload."""
 229     if len(changes_files) == 1:
 230         return changes_files
 231
 232     sorted_list = []
 233     cache = {}
 234     # Read in all the .changes files
 235     for filename in changes_files:
 236         try:
 237             Upload.pkg.changes_file = filename
 238             Upload.init_vars()
 239             Upload.update_vars()
 240             cache[filename] = copy.copy(Upload.pkg.changes)
 241             cache[filename]["filename"] = filename
 242         except:
 243             sorted_list.append(filename)
 244             break
 245     # Divide the .changes into per-source groups
 246     per_source = {}
 247     for filename in cache.keys():
 248         source = cache[filename]["source"]
 249         if not per_source.has_key(source):
 250             per_source[source] = {}
 251             per_source[source]["list"] = []
 252         per_source[source]["list"].append(cache[filename])
 253     # Determine oldest time and have note status for each source group
 254     for source in per_source.keys():
 255         source_list = per_source[source]["list"]
 256         first = source_list[0]
 257         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 258         have_note = 0
 259         for d in per_source[source]["list"]:
 260             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 261             if mtime < oldest:
 262                 oldest = mtime
 263             have_note += (d.has_key("process-new note"))
 264         per_source[source]["oldest"] = oldest
 265         if not have_note:
 266             per_source[source]["note_state"] = 0; # none
 267         elif have_note < len(source_list):
 268             per_source[source]["note_state"] = 1; # some
 269         else:
 270             per_source[source]["note_state"] = 2; # all
 271         per_source[source]["list"].sort(indiv_sg_compare)
 272     per_source_items = per_source.items()
 273     per_source_items.sort(sg_compare)
 274     for i in per_source_items:
 275         for j in i[1]["list"]:
 276             sorted_list.append(j["filename"])
 277     return sorted_list
 278
 279 ################################################################################
 280
 281 class Section_Completer:
 282     def __init__ (self):
 283         self.sections = []
 284         q = projectB.query("SELECT section FROM section")
 285         for i in q.getresult():
 286             self.sections.append(i[0])
 287
 288     def complete(self, text, state):
 289         if state == 0:
 290             self.matches = []
 291             n = len(text)
 292             for word in self.sections:
 293                 if word[:n] == text:
 294                     self.matches.append(word)
 295         try:
 296             return self.matches[state]
 297         except IndexError:
 298             return None
 299
 300 ############################################################
 301
 302 class Priority_Completer:
 303     def __init__ (self):
 304         self.priorities = []
 305         q = projectB.query("SELECT priority FROM priority")
 306         for i in q.getresult():
 307             self.priorities.append(i[0])
 308
 309     def complete(self, text, state):
 310         if state == 0:
 311             self.matches = []
 312             n = len(text)
 313             for word in self.priorities:
 314                 if word[:n] == text:
 315                     self.matches.append(word)
 316         try:
 317             return self.matches[state]
 318         except IndexError:
 319             return None
 320
 321 ################################################################################
 322
 323 def check_valid (new):
 324     for pkg in new.keys():
 325         section = new[pkg]["section"]
 326         priority = new[pkg]["priority"]
 327         type = new[pkg]["type"]
 328         new[pkg]["section id"] = daklib.database.get_section_id(section)
 329         new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"])
 330         # Sanity checks
 331         di = section.find("debian-installer") != -1
 332         if (di and type != "udeb") or (not di and type == "udeb"):
 333             new[pkg]["section id"] = -1
 334         if (priority == "source" and type != "dsc") or \
 335            (priority != "source" and type == "dsc"):
 336             new[pkg]["priority id"] = -1
 337
 338 ################################################################################
 339
 340 def print_new (new, indexed, file=sys.stdout):
 341     check_valid(new)
 342     broken = 0
 343     index = 0
 344     for pkg in new.keys():
 345         index += 1
 346         section = new[pkg]["section"]
 347         priority = new[pkg]["priority"]
 348         if new[pkg]["section id"] == -1:
 349             section += "[!]"
 350             broken = 1
 351         if new[pkg]["priority id"] == -1:
 352             priority += "[!]"
 353             broken = 1
 354         if indexed:
 355             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 356         else:
 357             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 358         line = line.strip()+'\n'
 359         file.write(line)
 360     note = Upload.pkg.changes.get("process-new note")
 361     if note:
 362         print "*"*75
 363         print note
 364         print "*"*75
 365     return broken, note
 366
 367 ################################################################################
 368
 369 def get_type (f):
 370     # Determine the type
 371     if f.has_key("dbtype"):
 372         type = f["dbtype"]
 373     elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
 374         type = "dsc"
 375     else:
 376         daklib.utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (type))
 377
 378     # Validate the override type
 379     type_id = daklib.database.get_override_type_id(type)
 380     if type_id == -1:
 381         daklib.utils.fubar("invalid type (%s) for new.  Say wha?" % (type))
 382
 383     return type
 384
 385 ################################################################################
 386
 387 def index_range (index):
 388     if index == 1:
 389         return "1"
 390     else:
 391         return "1-%s" % (index)
 392
 393 ################################################################################
 394 ################################################################################
 395
 396 def edit_new (new):
 397     # Write the current data to a temporary file
 398     temp_filename = daklib.utils.temp_filename()
 399     temp_file = daklib.utils.open_file(temp_filename, 'w')
 400     print_new (new, 0, temp_file)
 401     temp_file.close()
 402     # Spawn an editor on that file
 403     editor = os.environ.get("EDITOR","vi")
 404     result = os.system("%s %s" % (editor, temp_filename))
 405     if result != 0:
 406         daklib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 407     # Read the edited data back in
 408     temp_file = daklib.utils.open_file(temp_filename)
 409     lines = temp_file.readlines()
 410     temp_file.close()
 411     os.unlink(temp_filename)
 412     # Parse the new data
 413     for line in lines:
 414         line = line.strip()
 415         if line == "":
 416             continue
 417         s = line.split()
 418         # Pad the list if necessary
 419         s[len(s):3] = [None] * (3-len(s))
 420         (pkg, priority, section) = s[:3]
 421         if not new.has_key(pkg):
 422             daklib.utils.warn("Ignoring unknown package '%s'" % (pkg))
 423         else:
 424             # Strip off any invalid markers, print_new will readd them.
 425             if section.endswith("[!]"):
 426                 section = section[:-3]
 427             if priority.endswith("[!]"):
 428                 priority = priority[:-3]
 429             for file in new[pkg]["files"]:
 430                 Upload.pkg.files[file]["section"] = section
 431                 Upload.pkg.files[file]["priority"] = priority
 432             new[pkg]["section"] = section
 433             new[pkg]["priority"] = priority
 434
 435 ################################################################################
 436
 437 def edit_index (new, index):
 438     priority = new[index]["priority"]
 439     section = new[index]["section"]
 440     type = new[index]["type"]
 441     done = 0
 442     while not done:
 443         print "\t".join([index, priority, section])
 444
 445         answer = "XXX"
 446         if type != "dsc":
 447             prompt = "[B]oth, Priority, Section, Done ? "
 448         else:
 449             prompt = "[S]ection, Done ? "
 450         edit_priority = edit_section = 0
 451
 452         while prompt.find(answer) == -1:
 453             answer = daklib.utils.our_raw_input(prompt)
 454             m = daklib.queue.re_default_answer.match(prompt)
 455             if answer == "":
 456                 answer = m.group(1)
 457             answer = answer[:1].upper()
 458
 459         if answer == 'P':
 460             edit_priority = 1
 461         elif answer == 'S':
 462             edit_section = 1
 463         elif answer == 'B':
 464             edit_priority = edit_section = 1
 465         elif answer == 'D':
 466             done = 1
 467
 468         # Edit the priority
 469         if edit_priority:
 470             readline.set_completer(Priorities.complete)
 471             got_priority = 0
 472             while not got_priority:
 473                 new_priority = daklib.utils.our_raw_input("New priority: ").strip()
 474                 if new_priority not in Priorities.priorities:
 475                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 476                 else:
 477                     got_priority = 1
 478                     priority = new_priority
 479
 480         # Edit the section
 481         if edit_section:
 482             readline.set_completer(Sections.complete)
 483             got_section = 0
 484             while not got_section:
 485                 new_section = daklib.utils.our_raw_input("New section: ").strip()
 486                 if new_section not in Sections.sections:
 487                     print "E: '%s' is not a valid section, try again." % (new_section)
 488                 else:
 489                     got_section = 1
 490                     section = new_section
 491
 492         # Reset the readline completer
 493         readline.set_completer(None)
 494
 495     for file in new[index]["files"]:
 496         Upload.pkg.files[file]["section"] = section
 497         Upload.pkg.files[file]["priority"] = priority
 498     new[index]["priority"] = priority
 499     new[index]["section"] = section
 500     return new
 501
 502 ################################################################################
 503
 504 def edit_overrides (new):
 505     print
 506     done = 0
 507     while not done:
 508         print_new (new, 1)
 509         new_index = {}
 510         index = 0
 511         for i in new.keys():
 512             index += 1
 513             new_index[index] = i
 514
 515         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 516
 517         got_answer = 0
 518         while not got_answer:
 519             answer = daklib.utils.our_raw_input(prompt)
 520             if not answer.isdigit():
 521                 answer = answer[:1].upper()
 522             if answer == "E" or answer == "D":
 523                 got_answer = 1
 524             elif daklib.queue.re_isanum.match (answer):
 525                 answer = int(answer)
 526                 if (answer < 1) or (answer > index):
 527                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 528                 else:
 529                     got_answer = 1
 530
 531         if answer == 'E':
 532             edit_new(new)
 533         elif answer == 'D':
 534             done = 1
 535         else:
 536             edit_index (new, new_index[answer])
 537
 538     return new
 539
 540 ################################################################################
 541
 542 def edit_note(note):
 543     # Write the current data to a temporary file
 544     temp_filename = daklib.utils.temp_filename()
 545     temp_file = daklib.utils.open_file(temp_filename, 'w')
 546     temp_file.write(note)
 547     temp_file.close()
 548     editor = os.environ.get("EDITOR","vi")
 549     answer = 'E'
 550     while answer == 'E':
 551         os.system("%s %s" % (editor, temp_filename))
 552         temp_file = daklib.utils.open_file(temp_filename)
 553         note = temp_file.read().rstrip()
 554         temp_file.close()
 555         print "Note:"
 556         print daklib.utils.prefix_multi_line_string(note,"  ")
 557         prompt = "[D]one, Edit, Abandon, Quit ?"
 558         answer = "XXX"
 559         while prompt.find(answer) == -1:
 560             answer = daklib.utils.our_raw_input(prompt)
 561             m = daklib.queue.re_default_answer.search(prompt)
 562             if answer == "":
 563                 answer = m.group(1)
 564             answer = answer[:1].upper()
 565     os.unlink(temp_filename)
 566     if answer == 'A':
 567         return
 568     elif answer == 'Q':
 569         sys.exit(0)
 570     Upload.pkg.changes["process-new note"] = note
 571     Upload.dump_vars(Cnf["Dir::Queue::New"])
 572
 573 ################################################################################
 574
 575 def check_pkg ():
 576     try:
 577         less_fd = os.popen("less -R -", 'w', 0)
 578         stdout_fd = sys.stdout
 579         try:
 580             sys.stdout = less_fd
 581             examine_package.display_changes(Upload.pkg.changes_file)
 582             files = Upload.pkg.files
 583             for file in files.keys():
 584                 if files[file].has_key("new"):
 585                     type = files[file]["type"]
 586                     if type == "deb":
 587                         examine_package.check_deb(file)
 588                     elif type == "dsc":
 589                         examine_package.check_dsc(file)
 590         finally:
 591             sys.stdout = stdout_fd
 592     except IOError, e:
 593         if errno.errorcode[e.errno] == 'EPIPE':
 594             daklib.utils.warn("[examine_package] Caught EPIPE; skipping.")
 595             pass
 596         else:
 597             raise
 598     except KeyboardInterrupt:
 599         daklib.utils.warn("[examine_package] Caught C-c; skipping.")
 600         pass
 601
 602 ################################################################################
 603
 604 ## FIXME: horribly Debian specific
 605
 606 def do_bxa_notification():
 607     files = Upload.pkg.files
 608     summary = ""
 609     for file in files.keys():
 610         if files[file]["type"] == "deb":
 611             control = apt_pkg.ParseSection(apt_inst.debExtractControl(daklib.utils.open_file(file)))
 612             summary += "\n"
 613             summary += "Package: %s\n" % (control.Find("Package"))
 614             summary += "Description: %s\n" % (control.Find("Description"))
 615     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 616     bxa_mail = daklib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 617     daklib.utils.send_mail(bxa_mail)
 618
 619 ################################################################################
 620
 621 def add_overrides (new):
 622     changes = Upload.pkg.changes
 623     files = Upload.pkg.files
 624
 625     projectB.query("BEGIN WORK")
 626     for suite in changes["suite"].keys():
 627         suite_id = daklib.database.get_suite_id(suite)
 628         for pkg in new.keys():
 629             component_id = daklib.database.get_component_id(new[pkg]["component"])
 630             type_id = daklib.database.get_override_type_id(new[pkg]["type"])
 631             priority_id = new[pkg]["priority id"]
 632             section_id = new[pkg]["section id"]
 633             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 634             for file in new[pkg]["files"]:
 635                 if files[file].has_key("new"):
 636                     del files[file]["new"]
 637             del new[pkg]
 638
 639     projectB.query("COMMIT WORK")
 640
 641     if Cnf.FindB("Dinstall::BXANotify"):
 642         do_bxa_notification()
 643
 644 ################################################################################
 645
 646 def prod_maintainer ():
 647     # Here we prepare an editor and get them ready to prod...
 648     temp_filename = daklib.utils.temp_filename()
 649     editor = os.environ.get("EDITOR","vi")
 650     answer = 'E'
 651     while answer == 'E':
 652         os.system("%s %s" % (editor, temp_filename))
 653         file = daklib.utils.open_file(temp_filename)
 654         prod_message = "".join(file.readlines())
 655         file.close()
 656         print "Prod message:"
 657         print daklib.utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 658         prompt = "[P]rod, Edit, Abandon, Quit ?"
 659         answer = "XXX"
 660         while prompt.find(answer) == -1:
 661             answer = daklib.utils.our_raw_input(prompt)
 662             m = daklib.queue.re_default_answer.search(prompt)
 663             if answer == "":
 664                 answer = m.group(1)
 665             answer = answer[:1].upper()
 666         os.unlink(temp_filename)
 667         if answer == 'A':
 668             return
 669         elif answer == 'Q':
 670             sys.exit(0)
 671     # Otherwise, do the proding...
 672     user_email_address = daklib.utils.whoami() + " <%s>" % (
 673         Cnf["Dinstall::MyAdminAddress"])
 674
 675     Subst = Upload.Subst
 676
 677     Subst["__FROM_ADDRESS__"] = user_email_address
 678     Subst["__PROD_MESSAGE__"] = prod_message
 679     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 680
 681     prod_mail_message = daklib.utils.TemplateSubst(
 682         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 683
 684     # Send the prod mail if appropriate
 685     if not Cnf["Dinstall::Options::No-Mail"]:
 686         daklib.utils.send_mail(prod_mail_message)
 687
 688     print "Sent proding message"
 689
 690 ################################################################################
 691
 692 def do_new():
 693     print "NEW\n"
 694     files = Upload.pkg.files
 695     changes = Upload.pkg.changes
 696
 697     # Make a copy of distribution we can happily trample on
 698     changes["suite"] = copy.copy(changes["distribution"])
 699
 700     # Fix up the list of target suites
 701     for suite in changes["suite"].keys():
 702         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 703         if override:
 704             (olderr, newerr) = (daklib.database.get_suite_id(suite) == -1,
 705               daklib.database.get_suite_id(override) == -1)
 706             if olderr or newerr:
 707                 (oinv, newinv) = ("", "")
 708                 if olderr: oinv = "invalid "
 709                 if newerr: ninv = "invalid "
 710                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 711                         oinv, suite, ninv, override)
 712             del changes["suite"][suite]
 713             changes["suite"][override] = 1
 714     # Validate suites
 715     for suite in changes["suite"].keys():
 716         suite_id = daklib.database.get_suite_id(suite)
 717         if suite_id == -1:
 718             daklib.utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 719
 720     # The main NEW processing loop
 721     done = 0
 722     while not done:
 723         # Find out what's new
 724         new = determine_new(changes, files)
 725
 726         if not new:
 727             break
 728
 729         answer = "XXX"
 730         if Options["No-Action"] or Options["Automatic"]:
 731             answer = 'S'
 732
 733         (broken, note) = print_new(new, 0)
 734         prompt = ""
 735
 736         if not broken and not note:
 737             prompt = "Add overrides, "
 738         if broken:
 739             print "W: [!] marked entries must be fixed before package can be processed."
 740         if note:
 741             print "W: note must be removed before package can be processed."
 742             prompt += "Remove note, "
 743
 744         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 745
 746         while prompt.find(answer) == -1:
 747             answer = daklib.utils.our_raw_input(prompt)
 748             m = daklib.queue.re_default_answer.search(prompt)
 749             if answer == "":
 750                 answer = m.group(1)
 751             answer = answer[:1].upper()
 752
 753         if answer == 'A':
 754             done = add_overrides (new)
 755         elif answer == 'C':
 756             check_pkg()
 757         elif answer == 'E':
 758             new = edit_overrides (new)
 759         elif answer == 'M':
 760             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 761             if not aborted:
 762                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 763                 done = 1
 764         elif answer == 'N':
 765             edit_note(changes.get("process-new note", ""))
 766         elif answer == 'P':
 767             prod_maintainer()
 768         elif answer == 'R':
 769             confirm = daklib.utils.our_raw_input("Really clear note (y/N)? ").lower()
 770             if confirm == "y":
 771                 del changes["process-new note"]
 772         elif answer == 'S':
 773             done = 1
 774         elif answer == 'Q':
 775             sys.exit(0)
 776
 777 ################################################################################
 778 ################################################################################
 779 ################################################################################
 780
 781 def usage (exit_code=0):
 782     print """Usage: dak process-new [OPTION]... [CHANGES]...
 783   -a, --automatic           automatic run
 784   -h, --help                show this help and exit.
 785   -m, --manual-reject=MSG   manual reject with `msg'
 786   -n, --no-action           don't do anything
 787   -V, --version             display the version number and exit"""
 788     sys.exit(exit_code)
 789
 790 ################################################################################
 791
 792 def init():
 793     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 794
 795     Cnf = daklib.utils.get_conf()
 796
 797     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 798                  ('h',"help","Process-New::Options::Help"),
 799                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 800                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 801                  ('n',"no-action","Process-New::Options::No-Action")]
 802
 803     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 804         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 805             Cnf["Process-New::Options::%s" % (i)] = ""
 806
 807     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 808     Options = Cnf.SubTree("Process-New::Options")
 809
 810     if Options["Help"]:
 811         usage()
 812
 813     Upload = daklib.queue.Upload(Cnf)
 814
 815     if not Options["No-Action"]:
 816         Logger = Upload.Logger = daklib.logging.Logger(Cnf, "process-new")
 817
 818     projectB = Upload.projectB
 819
 820     Sections = Section_Completer()
 821     Priorities = Priority_Completer()
 822     readline.parse_and_bind("tab: complete")
 823
 824     return changes_files
 825
 826 ################################################################################
 827
 828 def do_byhand():
 829     done = 0
 830     while not done:
 831         files = Upload.pkg.files
 832         will_install = 1
 833         byhand = []
 834
 835         for file in files.keys():
 836             if files[file]["type"] == "byhand":
 837                 if os.path.exists(file):
 838                     print "W: %s still present; please process byhand components and try again." % (file)
 839                     will_install = 0
 840                 else:
 841                     byhand.append(file)
 842
 843         answer = "XXXX"
 844         if Options["No-Action"]:
 845             answer = "S"
 846         if will_install:
 847             if Options["Automatic"] and not Options["No-Action"]:
 848                 answer = 'A'
 849             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 850         else:
 851             prompt = "Manual reject, [S]kip, Quit ?"
 852
 853         while prompt.find(answer) == -1:
 854             answer = daklib.utils.our_raw_input(prompt)
 855             m = daklib.queue.re_default_answer.search(prompt)
 856             if answer == "":
 857                 answer = m.group(1)
 858             answer = answer[:1].upper()
 859
 860         if answer == 'A':
 861             done = 1
 862             for file in byhand:
 863                 del files[file]
 864         elif answer == 'M':
 865             Upload.do_reject(1, Options["Manual-Reject"])
 866             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 867             done = 1
 868         elif answer == 'S':
 869             done = 1
 870         elif answer == 'Q':
 871             sys.exit(0)
 872
 873 ################################################################################
 874
 875 def do_accept():
 876     print "ACCEPT"
 877     if not Options["No-Action"]:
 878         retry = 0
 879         while retry < 10:
 880             try:
 881                 lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 882                 retry = 10
 883             except OSError, e:
 884                 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
 885                     retry += 1
 886                     if (retry >= 10):
 887                         daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 888                     else:
 889                         print("Unable to get accepted lock (try %d of 10)" % retry)
 890                     time.sleep(60)
 891                 else:
 892                     raise
 893         (summary, short_summary) = Upload.build_summaries()
 894         Upload.accept(summary, short_summary)
 895         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 896         os.unlink(Cnf["Process-New::AcceptedLockFile"])
 897
 898 def check_status(files):
 899     new = byhand = 0
 900     for file in files.keys():
 901         if files[file]["type"] == "byhand":
 902             byhand = 1
 903         elif files[file].has_key("new"):
 904             new = 1
 905     return (new, byhand)
 906
 907 def do_pkg(changes_file):
 908     Upload.pkg.changes_file = changes_file
 909     Upload.init_vars()
 910     Upload.update_vars()
 911     Upload.update_subst()
 912     files = Upload.pkg.files
 913
 914     if not recheck():
 915         return
 916
 917     (new, byhand) = check_status(files)
 918     if new or byhand:
 919         if new:
 920             do_new()
 921         if byhand:
 922             do_byhand()
 923         (new, byhand) = check_status(files)
 924
 925     if not new and not byhand:
 926         do_accept()
 927
 928 ################################################################################
 929
 930 def end():
 931     accept_count = Upload.accept_count
 932     accept_bytes = Upload.accept_bytes
 933
 934     if accept_count:
 935         sets = "set"
 936         if accept_count > 1:
 937             sets = "sets"
 938         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, daklib.utils.size_type(int(accept_bytes))))
 939         Logger.log(["total",accept_count,accept_bytes])
 940
 941     if not Options["No-Action"]:
 942         Logger.close()
 943
 944 ################################################################################
 945
 946 def do_comments(dir, opref, npref, line, fn):
 947     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 948         lines = open("%s/%s" % (dir, comm)).readlines()
 949         if len(lines) == 0 or lines[0] != line + "\n": continue
 950         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 951                                 and x.endswith(".changes") ]
 952         changes_files = sort_changes(changes_files)
 953         for f in changes_files:
 954                 f = daklib.utils.validate_changes_file_arg(f, 0)
 955                 if not f: continue
 956                 print "\n" + f
 957                 fn(f, "".join(lines[1:]))
 958
 959         if opref != npref and not Options["No-Action"]:
 960                 newcomm = npref + comm[len(opref):]
 961                 os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 962
 963 ################################################################################
 964
 965 def comment_accept(changes_file, comments):
 966     Upload.pkg.changes_file = changes_file
 967     Upload.init_vars()
 968     Upload.update_vars()
 969     Upload.update_subst()
 970     files = Upload.pkg.files
 971
 972     if not recheck():
 973         return
 974
 975     (new, byhand) = check_status(files)
 976     if not new and not byhand:
 977         do_accept()
 978
 979 ################################################################################
 980
 981 def comment_reject(changes_file, comments):
 982     Upload.pkg.changes_file = changes_file
 983     Upload.init_vars()
 984     Upload.update_vars()
 985     Upload.update_subst()
 986     files = Upload.pkg.files
 987
 988     if not recheck():
 989         return
 990
 991     reject(comments)
 992     print "REJECT\n" + reject_message,
 993     if not Options["No-Action"]:
 994         Upload.do_reject(0, reject_message)
 995         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 996
 997 ################################################################################
 998
 999 def main():
1000     changes_files = init()
1001     if len(changes_files) > 50:
1002         sys.stderr.write("Sorting changes...\n")
1003     changes_files = sort_changes(changes_files)
1004
1005     # Kill me now? **FIXME**
1006     Cnf["Dinstall::Options::No-Mail"] = ""
1007     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1008     if Cnf.has_key("Dinstall::Bcc"):
1009         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1010     else:
1011         Upload.Subst["__BCC__"] = bcc
1012
1013     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1014     if commentsdir:
1015         if changes_files != []:
1016                 sys.stderr.write("Can't specify any changes files if working with comments-dir")
1017                 sys.exit(1)
1018         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1019         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1020     else:
1021         for changes_file in changes_files:
1022             changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
1023             if not changes_file:
1024                 continue
1025             print "\n" + changes_file
1026             do_pkg (changes_file)
1027
1028     end()
1029
1030 ################################################################################
1031
1032 if __name__ == '__main__':
1033     main()