dak/process_new.py

   1 #!/usr/bin/env python
   2
   3 # Handles NEW and BYHAND packages
   4 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 ################################################################################
  21
  22 # 23:12|<aj> I will not hush!
  23 # 23:12|<elmo> :>
  24 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  25 # 23:13|<aj> I shall not be silenced!
  26 # 23:13|<aj> The world shall know!
  27 # 23:13|<aj> The world *must* know!
  28 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  29 # 23:13|<aj> yay powerpuff girls!!
  30 # 23:13|<aj> buttercup's my favourite, who's yours?
  31 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  32 # 23:14|<aj> *AREN'T YOU*?!
  33 # 23:15|<aj> I will not be treated like this.
  34 # 23:15|<aj> I shall have my revenge.
  35 # 23:15|<aj> I SHALL!!!
  36
  37 ################################################################################
  38
  39 import copy, errno, os, readline, stat, sys, time
  40 import apt_pkg, apt_inst
  41 import dak.lib.database, examine_package, dak.lib.queue, dak.lib.logging, dak.lib.utils
  42
  43 # Globals
  44 Cnf = None
  45 Options = None
  46 Upload = None
  47 projectB = None
  48 Logger = None
  49
  50 Priorities = None
  51 Sections = None
  52
  53 reject_message = ""
  54
  55 ################################################################################
  56 ################################################################################
  57 ################################################################################
  58
  59 def reject (str, prefix="Rejected: "):
  60     global reject_message
  61     if str:
  62         reject_message += prefix + str + "\n"
  63
  64 def recheck():
  65     global reject_message
  66     files = Upload.pkg.files
  67     reject_message = ""
  68
  69     for file in files.keys():
  70         # The .orig.tar.gz can disappear out from under us is it's a
  71         # duplicate of one in the archive.
  72         if not files.has_key(file):
  73             continue
  74         # Check that the source still exists
  75         if files[file]["type"] == "deb":
  76             source_version = files[file]["source version"]
  77             source_package = files[file]["source package"]
  78             if not Upload.pkg.changes["architecture"].has_key("source") \
  79                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  80                 source_epochless_version = dak.lib.utils.re_no_epoch.sub('', source_version)
  81                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  82                 if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
  83                     reject("no source found for %s %s (%s)." % (source_package, source_version, file))
  84
  85         # Version and file overwrite checks
  86         if files[file]["type"] == "deb":
  87             reject(Upload.check_binary_against_db(file))
  88         elif files[file]["type"] == "dsc":
  89             reject(Upload.check_source_against_db(file))
  90             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
  91             reject(reject_msg)
  92
  93     if reject_message:
  94         answer = "XXX"
  95         if Options["No-Action"] or Options["Automatic"]:
  96             answer = 'S'
  97
  98         print "REJECT\n" + reject_message,
  99         prompt = "[R]eject, Skip, Quit ?"
 100
 101         while prompt.find(answer) == -1:
 102             answer = dak.lib.utils.our_raw_input(prompt)
 103             m = dak.lib.queue.re_default_answer.match(prompt)
 104             if answer == "":
 105                 answer = m.group(1)
 106             answer = answer[:1].upper()
 107
 108         if answer == 'R':
 109             Upload.do_reject(0, reject_message)
 110             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 111             return 0
 112         elif answer == 'S':
 113             return 0
 114         elif answer == 'Q':
 115             sys.exit(0)
 116
 117     return 1
 118
 119 ################################################################################
 120
 121 def determine_new (changes, files):
 122     new = {}
 123
 124     # Build up a list of potentially new things
 125     for file in files.keys():
 126         f = files[file]
 127         # Skip byhand elements
 128         if f["type"] == "byhand":
 129             continue
 130         pkg = f["package"]
 131         priority = f["priority"]
 132         section = f["section"]
 133         # FIXME: unhardcode
 134         if section == "non-US/main":
 135             section = "non-US"
 136         type = get_type(f)
 137         component = f["component"]
 138
 139         if type == "dsc":
 140             priority = "source"
 141         if not new.has_key(pkg):
 142             new[pkg] = {}
 143             new[pkg]["priority"] = priority
 144             new[pkg]["section"] = section
 145             new[pkg]["type"] = type
 146             new[pkg]["component"] = component
 147             new[pkg]["files"] = []
 148         else:
 149             old_type = new[pkg]["type"]
 150             if old_type != type:
 151                 # source gets trumped by deb or udeb
 152                 if old_type == "dsc":
 153                     new[pkg]["priority"] = priority
 154                     new[pkg]["section"] = section
 155                     new[pkg]["type"] = type
 156                     new[pkg]["component"] = component
 157         new[pkg]["files"].append(file)
 158         if f.has_key("othercomponents"):
 159             new[pkg]["othercomponents"] = f["othercomponents"]
 160
 161     for suite in changes["suite"].keys():
 162         suite_id = dak.lib.database.get_suite_id(suite)
 163         for pkg in new.keys():
 164             component_id = dak.lib.database.get_component_id(new[pkg]["component"])
 165             type_id = dak.lib.database.get_override_type_id(new[pkg]["type"])
 166             q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
 167             ql = q.getresult()
 168             if ql:
 169                 for file in new[pkg]["files"]:
 170                     if files[file].has_key("new"):
 171                         del files[file]["new"]
 172                 del new[pkg]
 173
 174     if changes["suite"].has_key("stable"):
 175         print "WARNING: overrides will be added for stable!"
 176     if changes["suite"].has_key("oldstable"):
 177         print "WARNING: overrides will be added for OLDstable!"
 178     for pkg in new.keys():
 179         if new[pkg].has_key("othercomponents"):
 180             print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
 181
 182     return new
 183
 184 ################################################################################
 185
 186 def indiv_sg_compare (a, b):
 187     """Sort by source name, source, version, 'have source', and
 188        finally by filename."""
 189     # Sort by source version
 190     q = apt_pkg.VersionCompare(a["version"], b["version"])
 191     if q:
 192         return -q
 193
 194     # Sort by 'have source'
 195     a_has_source = a["architecture"].get("source")
 196     b_has_source = b["architecture"].get("source")
 197     if a_has_source and not b_has_source:
 198         return -1
 199     elif b_has_source and not a_has_source:
 200         return 1
 201
 202     return cmp(a["filename"], b["filename"])
 203
 204 ############################################################
 205
 206 def sg_compare (a, b):
 207     a = a[1]
 208     b = b[1]
 209     """Sort by have note, time of oldest upload."""
 210     # Sort by have note
 211     a_note_state = a["note_state"]
 212     b_note_state = b["note_state"]
 213     if a_note_state < b_note_state:
 214         return -1
 215     elif a_note_state > b_note_state:
 216         return 1
 217
 218     # Sort by time of oldest upload
 219     return cmp(a["oldest"], b["oldest"])
 220
 221 def sort_changes(changes_files):
 222     """Sort into source groups, then sort each source group by version,
 223     have source, filename.  Finally, sort the source groups by have
 224     note, time of oldest upload of each source upload."""
 225     if len(changes_files) == 1:
 226         return changes_files
 227
 228     sorted_list = []
 229     cache = {}
 230     # Read in all the .changes files
 231     for filename in changes_files:
 232         try:
 233             Upload.pkg.changes_file = filename
 234             Upload.init_vars()
 235             Upload.update_vars()
 236             cache[filename] = copy.copy(Upload.pkg.changes)
 237             cache[filename]["filename"] = filename
 238         except:
 239             sorted_list.append(filename)
 240             break
 241     # Divide the .changes into per-source groups
 242     per_source = {}
 243     for filename in cache.keys():
 244         source = cache[filename]["source"]
 245         if not per_source.has_key(source):
 246             per_source[source] = {}
 247             per_source[source]["list"] = []
 248         per_source[source]["list"].append(cache[filename])
 249     # Determine oldest time and have note status for each source group
 250     for source in per_source.keys():
 251         source_list = per_source[source]["list"]
 252         first = source_list[0]
 253         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 254         have_note = 0
 255         for d in per_source[source]["list"]:
 256             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 257             if mtime < oldest:
 258                 oldest = mtime
 259             have_note += (d.has_key("process-new note"))
 260         per_source[source]["oldest"] = oldest
 261         if not have_note:
 262             per_source[source]["note_state"] = 0; # none
 263         elif have_note < len(source_list):
 264             per_source[source]["note_state"] = 1; # some
 265         else:
 266             per_source[source]["note_state"] = 2; # all
 267         per_source[source]["list"].sort(indiv_sg_compare)
 268     per_source_items = per_source.items()
 269     per_source_items.sort(sg_compare)
 270     for i in per_source_items:
 271         for j in i[1]["list"]:
 272             sorted_list.append(j["filename"])
 273     return sorted_list
 274
 275 ################################################################################
 276
 277 class Section_Completer:
 278     def __init__ (self):
 279         self.sections = []
 280         q = projectB.query("SELECT section FROM section")
 281         for i in q.getresult():
 282             self.sections.append(i[0])
 283
 284     def complete(self, text, state):
 285         if state == 0:
 286             self.matches = []
 287             n = len(text)
 288             for word in self.sections:
 289                 if word[:n] == text:
 290                     self.matches.append(word)
 291         try:
 292             return self.matches[state]
 293         except IndexError:
 294             return None
 295
 296 ############################################################
 297
 298 class Priority_Completer:
 299     def __init__ (self):
 300         self.priorities = []
 301         q = projectB.query("SELECT priority FROM priority")
 302         for i in q.getresult():
 303             self.priorities.append(i[0])
 304
 305     def complete(self, text, state):
 306         if state == 0:
 307             self.matches = []
 308             n = len(text)
 309             for word in self.priorities:
 310                 if word[:n] == text:
 311                     self.matches.append(word)
 312         try:
 313             return self.matches[state]
 314         except IndexError:
 315             return None
 316
 317 ################################################################################
 318
 319 def check_valid (new):
 320     for pkg in new.keys():
 321         section = new[pkg]["section"]
 322         priority = new[pkg]["priority"]
 323         type = new[pkg]["type"]
 324         new[pkg]["section id"] = dak.lib.database.get_section_id(section)
 325         new[pkg]["priority id"] = dak.lib.database.get_priority_id(new[pkg]["priority"])
 326         # Sanity checks
 327         if (section == "debian-installer" and type != "udeb") or \
 328            (section != "debian-installer" and type == "udeb"):
 329             new[pkg]["section id"] = -1
 330         if (priority == "source" and type != "dsc") or \
 331            (priority != "source" and type == "dsc"):
 332             new[pkg]["priority id"] = -1
 333
 334 ################################################################################
 335
 336 def print_new (new, indexed, file=sys.stdout):
 337     check_valid(new)
 338     broken = 0
 339     index = 0
 340     for pkg in new.keys():
 341         index += 1
 342         section = new[pkg]["section"]
 343         priority = new[pkg]["priority"]
 344         if new[pkg]["section id"] == -1:
 345             section += "[!]"
 346             broken = 1
 347         if new[pkg]["priority id"] == -1:
 348             priority += "[!]"
 349             broken = 1
 350         if indexed:
 351             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 352         else:
 353             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 354         line = line.strip()+'\n'
 355         file.write(line)
 356     note = Upload.pkg.changes.get("process-new note")
 357     if note:
 358         print "*"*75
 359         print note
 360         print "*"*75
 361     return broken, note
 362
 363 ################################################################################
 364
 365 def get_type (f):
 366     # Determine the type
 367     if f.has_key("dbtype"):
 368         type = f["dbtype"]
 369     elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
 370         type = "dsc"
 371     else:
 372         dak.lib.utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (type))
 373
 374     # Validate the override type
 375     type_id = dak.lib.database.get_override_type_id(type)
 376     if type_id == -1:
 377         dak.lib.utils.fubar("invalid type (%s) for new.  Say wha?" % (type))
 378
 379     return type
 380
 381 ################################################################################
 382
 383 def index_range (index):
 384     if index == 1:
 385         return "1"
 386     else:
 387         return "1-%s" % (index)
 388
 389 ################################################################################
 390 ################################################################################
 391
 392 def edit_new (new):
 393     # Write the current data to a temporary file
 394     temp_filename = dak.lib.utils.temp_filename()
 395     temp_file = dak.lib.utils.open_file(temp_filename, 'w')
 396     print_new (new, 0, temp_file)
 397     temp_file.close()
 398     # Spawn an editor on that file
 399     editor = os.environ.get("EDITOR","vi")
 400     result = os.system("%s %s" % (editor, temp_filename))
 401     if result != 0:
 402         dak.lib.utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 403     # Read the edited data back in
 404     temp_file = dak.lib.utils.open_file(temp_filename)
 405     lines = temp_file.readlines()
 406     temp_file.close()
 407     os.unlink(temp_filename)
 408     # Parse the new data
 409     for line in lines:
 410         line = line.strip()
 411         if line == "":
 412             continue
 413         s = line.split()
 414         # Pad the list if necessary
 415         s[len(s):3] = [None] * (3-len(s))
 416         (pkg, priority, section) = s[:3]
 417         if not new.has_key(pkg):
 418             dak.lib.utils.warn("Ignoring unknown package '%s'" % (pkg))
 419         else:
 420             # Strip off any invalid markers, print_new will readd them.
 421             if section.endswith("[!]"):
 422                 section = section[:-3]
 423             if priority.endswith("[!]"):
 424                 priority = priority[:-3]
 425             for file in new[pkg]["files"]:
 426                 Upload.pkg.files[file]["section"] = section
 427                 Upload.pkg.files[file]["priority"] = priority
 428             new[pkg]["section"] = section
 429             new[pkg]["priority"] = priority
 430
 431 ################################################################################
 432
 433 def edit_index (new, index):
 434     priority = new[index]["priority"]
 435     section = new[index]["section"]
 436     type = new[index]["type"]
 437     done = 0
 438     while not done:
 439         print "\t".join([index, priority, section])
 440
 441         answer = "XXX"
 442         if type != "dsc":
 443             prompt = "[B]oth, Priority, Section, Done ? "
 444         else:
 445             prompt = "[S]ection, Done ? "
 446         edit_priority = edit_section = 0
 447
 448         while prompt.find(answer) == -1:
 449             answer = dak.lib.utils.our_raw_input(prompt)
 450             m = dak.lib.queue.re_default_answer.match(prompt)
 451             if answer == "":
 452                 answer = m.group(1)
 453             answer = answer[:1].upper()
 454
 455         if answer == 'P':
 456             edit_priority = 1
 457         elif answer == 'S':
 458             edit_section = 1
 459         elif answer == 'B':
 460             edit_priority = edit_section = 1
 461         elif answer == 'D':
 462             done = 1
 463
 464         # Edit the priority
 465         if edit_priority:
 466             readline.set_completer(Priorities.complete)
 467             got_priority = 0
 468             while not got_priority:
 469                 new_priority = dak.lib.utils.our_raw_input("New priority: ").strip()
 470                 if new_priority not in Priorities.priorities:
 471                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 472                 else:
 473                     got_priority = 1
 474                     priority = new_priority
 475
 476         # Edit the section
 477         if edit_section:
 478             readline.set_completer(Sections.complete)
 479             got_section = 0
 480             while not got_section:
 481                 new_section = dak.lib.utils.our_raw_input("New section: ").strip()
 482                 if new_section not in Sections.sections:
 483                     print "E: '%s' is not a valid section, try again." % (new_section)
 484                 else:
 485                     got_section = 1
 486                     section = new_section
 487
 488         # Reset the readline completer
 489         readline.set_completer(None)
 490
 491     for file in new[index]["files"]:
 492         Upload.pkg.files[file]["section"] = section
 493         Upload.pkg.files[file]["priority"] = priority
 494     new[index]["priority"] = priority
 495     new[index]["section"] = section
 496     return new
 497
 498 ################################################################################
 499
 500 def edit_overrides (new):
 501     print
 502     done = 0
 503     while not done:
 504         print_new (new, 1)
 505         new_index = {}
 506         index = 0
 507         for i in new.keys():
 508             index += 1
 509             new_index[index] = i
 510
 511         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 512
 513         got_answer = 0
 514         while not got_answer:
 515             answer = dak.lib.utils.our_raw_input(prompt)
 516             if not dak.lib.utils.str_isnum(answer):
 517                 answer = answer[:1].upper()
 518             if answer == "E" or answer == "D":
 519                 got_answer = 1
 520             elif dak.lib.queue.re_isanum.match (answer):
 521                 answer = int(answer)
 522                 if (answer < 1) or (answer > index):
 523                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 524                 else:
 525                     got_answer = 1
 526
 527         if answer == 'E':
 528             edit_new(new)
 529         elif answer == 'D':
 530             done = 1
 531         else:
 532             edit_index (new, new_index[answer])
 533
 534     return new
 535
 536 ################################################################################
 537
 538 def edit_note(note):
 539     # Write the current data to a temporary file
 540     temp_filename = dak.lib.utils.temp_filename()
 541     temp_file = dak.lib.utils.open_file(temp_filename, 'w')
 542     temp_file.write(note)
 543     temp_file.close()
 544     editor = os.environ.get("EDITOR","vi")
 545     answer = 'E'
 546     while answer == 'E':
 547         os.system("%s %s" % (editor, temp_filename))
 548         temp_file = dak.lib.utils.open_file(temp_filename)
 549         note = temp_file.read().rstrip()
 550         temp_file.close()
 551         print "Note:"
 552         print dak.lib.utils.prefix_multi_line_string(note,"  ")
 553         prompt = "[D]one, Edit, Abandon, Quit ?"
 554         answer = "XXX"
 555         while prompt.find(answer) == -1:
 556             answer = dak.lib.utils.our_raw_input(prompt)
 557             m = dak.lib.queue.re_default_answer.search(prompt)
 558             if answer == "":
 559                 answer = m.group(1)
 560             answer = answer[:1].upper()
 561     os.unlink(temp_filename)
 562     if answer == 'A':
 563         return
 564     elif answer == 'Q':
 565         sys.exit(0)
 566     Upload.pkg.changes["process-new note"] = note
 567     Upload.dump_vars(Cnf["Dir::Queue::New"])
 568
 569 ################################################################################
 570
 571 def check_pkg ():
 572     try:
 573         less_fd = os.popen("less -R -", 'w', 0)
 574         stdout_fd = sys.stdout
 575         try:
 576             sys.stdout = less_fd
 577             examine_package.display_changes(Upload.pkg.changes_file)
 578             files = Upload.pkg.files
 579             for file in files.keys():
 580                 if files[file].has_key("new"):
 581                     type = files[file]["type"]
 582                     if type == "deb":
 583                         examine_package.check_deb(file)
 584                     elif type == "dsc":
 585                         examine_package.check_dsc(file)
 586         finally:
 587             sys.stdout = stdout_fd
 588     except IOError, e:
 589         if errno.errorcode[e.errno] == 'EPIPE':
 590             dak.lib.utils.warn("[examine_package] Caught EPIPE; skipping.")
 591             pass
 592         else:
 593             raise
 594     except KeyboardInterrupt:
 595         dak.lib.utils.warn("[examine_package] Caught C-c; skipping.")
 596         pass
 597
 598 ################################################################################
 599
 600 ## FIXME: horribly Debian specific
 601
 602 def do_bxa_notification():
 603     files = Upload.pkg.files
 604     summary = ""
 605     for file in files.keys():
 606         if files[file]["type"] == "deb":
 607             control = apt_pkg.ParseSection(apt_inst.debExtractControl(dak.lib.utils.open_file(file)))
 608             summary += "\n"
 609             summary += "Package: %s\n" % (control.Find("Package"))
 610             summary += "Description: %s\n" % (control.Find("Description"))
 611     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 612     bxa_mail = dak.lib.utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 613     dak.lib.utils.send_mail(bxa_mail)
 614
 615 ################################################################################
 616
 617 def add_overrides (new):
 618     changes = Upload.pkg.changes
 619     files = Upload.pkg.files
 620
 621     projectB.query("BEGIN WORK")
 622     for suite in changes["suite"].keys():
 623         suite_id = dak.lib.database.get_suite_id(suite)
 624         for pkg in new.keys():
 625             component_id = dak.lib.database.get_component_id(new[pkg]["component"])
 626             type_id = dak.lib.database.get_override_type_id(new[pkg]["type"])
 627             priority_id = new[pkg]["priority id"]
 628             section_id = new[pkg]["section id"]
 629             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 630             for file in new[pkg]["files"]:
 631                 if files[file].has_key("new"):
 632                     del files[file]["new"]
 633             del new[pkg]
 634
 635     projectB.query("COMMIT WORK")
 636
 637     if Cnf.FindB("Dinstall::BXANotify"):
 638         do_bxa_notification()
 639
 640 ################################################################################
 641
 642 def prod_maintainer ():
 643     # Here we prepare an editor and get them ready to prod...
 644     temp_filename = dak.lib.utils.temp_filename()
 645     editor = os.environ.get("EDITOR","vi")
 646     answer = 'E'
 647     while answer == 'E':
 648         os.system("%s %s" % (editor, temp_filename))
 649         file = dak.lib.utils.open_file(temp_filename)
 650         prod_message = "".join(file.readlines())
 651         file.close()
 652         print "Prod message:"
 653         print dak.lib.utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 654         prompt = "[P]rod, Edit, Abandon, Quit ?"
 655         answer = "XXX"
 656         while prompt.find(answer) == -1:
 657             answer = dak.lib.utils.our_raw_input(prompt)
 658             m = dak.lib.queue.re_default_answer.search(prompt)
 659             if answer == "":
 660                 answer = m.group(1)
 661             answer = answer[:1].upper()
 662         os.unlink(temp_filename)
 663         if answer == 'A':
 664             return
 665         elif answer == 'Q':
 666             sys.exit(0)
 667     # Otherwise, do the proding...
 668     user_email_address = dak.lib.utils.whoami() + " <%s>" % (
 669         Cnf["Dinstall::MyAdminAddress"])
 670
 671     Subst = Upload.Subst
 672
 673     Subst["__FROM_ADDRESS__"] = user_email_address
 674     Subst["__PROD_MESSAGE__"] = prod_message
 675     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 676
 677     prod_mail_message = dak.lib.utils.TemplateSubst(
 678         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 679
 680     # Send the prod mail if appropriate
 681     if not Cnf["Dinstall::Options::No-Mail"]:
 682         dak.lib.utils.send_mail(prod_mail_message)
 683
 684     print "Sent proding message"
 685
 686 ################################################################################
 687
 688 def do_new():
 689     print "NEW\n"
 690     files = Upload.pkg.files
 691     changes = Upload.pkg.changes
 692
 693     # Make a copy of distribution we can happily trample on
 694     changes["suite"] = copy.copy(changes["distribution"])
 695
 696     # Fix up the list of target suites
 697     for suite in changes["suite"].keys():
 698         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 699         if override:
 700             del changes["suite"][suite]
 701             changes["suite"][override] = 1
 702     # Validate suites
 703     for suite in changes["suite"].keys():
 704         suite_id = dak.lib.database.get_suite_id(suite)
 705         if suite_id == -1:
 706             dak.lib.utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 707
 708     # The main NEW processing loop
 709     done = 0
 710     while not done:
 711         # Find out what's new
 712         new = determine_new(changes, files)
 713
 714         if not new:
 715             break
 716
 717         answer = "XXX"
 718         if Options["No-Action"] or Options["Automatic"]:
 719             answer = 'S'
 720
 721         (broken, note) = print_new(new, 0)
 722         prompt = ""
 723
 724         if not broken and not note:
 725             prompt = "Add overrides, "
 726         if broken:
 727             print "W: [!] marked entries must be fixed before package can be processed."
 728         if note:
 729             print "W: note must be removed before package can be processed."
 730             prompt += "Remove note, "
 731
 732         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 733
 734         while prompt.find(answer) == -1:
 735             answer = dak.lib.utils.our_raw_input(prompt)
 736             m = dak.lib.queue.re_default_answer.search(prompt)
 737             if answer == "":
 738                 answer = m.group(1)
 739             answer = answer[:1].upper()
 740
 741         if answer == 'A':
 742             done = add_overrides (new)
 743         elif answer == 'C':
 744             check_pkg()
 745         elif answer == 'E':
 746             new = edit_overrides (new)
 747         elif answer == 'M':
 748             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 749             if not aborted:
 750                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 751                 done = 1
 752         elif answer == 'N':
 753             edit_note(changes.get("process-new note", ""))
 754         elif answer == 'P':
 755             prod_maintainer()
 756         elif answer == 'R':
 757             confirm = dak.lib.utils.our_raw_input("Really clear note (y/N)? ").lower()
 758             if confirm == "y":
 759                 del changes["process-new note"]
 760         elif answer == 'S':
 761             done = 1
 762         elif answer == 'Q':
 763             sys.exit(0)
 764
 765 ################################################################################
 766 ################################################################################
 767 ################################################################################
 768
 769 def usage (exit_code=0):
 770     print """Usage: dak process-new [OPTION]... [CHANGES]...
 771   -a, --automatic           automatic run
 772   -h, --help                show this help and exit.
 773   -m, --manual-reject=MSG   manual reject with `msg'
 774   -n, --no-action           don't do anything
 775   -V, --version             display the version number and exit"""
 776     sys.exit(exit_code)
 777
 778 ################################################################################
 779
 780 def init():
 781     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 782
 783     Cnf = dak.lib.utils.get_conf()
 784
 785     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 786                  ('h',"help","Process-New::Options::Help"),
 787                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 788                  ('n',"no-action","Process-New::Options::No-Action")]
 789
 790     for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
 791         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 792             Cnf["Process-New::Options::%s" % (i)] = ""
 793
 794     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 795     Options = Cnf.SubTree("Process-New::Options")
 796
 797     if Options["Help"]:
 798         usage()
 799
 800     Upload = dak.lib.queue.Upload(Cnf)
 801
 802     if not Options["No-Action"]:
 803         Logger = Upload.Logger = dak.lib.logging.Logger(Cnf, "process-new")
 804
 805     projectB = Upload.projectB
 806
 807     Sections = Section_Completer()
 808     Priorities = Priority_Completer()
 809     readline.parse_and_bind("tab: complete")
 810
 811     return changes_files
 812
 813 ################################################################################
 814
 815 def do_byhand():
 816     done = 0
 817     while not done:
 818         files = Upload.pkg.files
 819         will_install = 1
 820         byhand = []
 821
 822         for file in files.keys():
 823             if files[file]["type"] == "byhand":
 824                 if os.path.exists(file):
 825                     print "W: %s still present; please process byhand components and try again." % (file)
 826                     will_install = 0
 827                 else:
 828                     byhand.append(file)
 829
 830         answer = "XXXX"
 831         if Options["No-Action"]:
 832             answer = "S"
 833         if will_install:
 834             if Options["Automatic"] and not Options["No-Action"]:
 835                 answer = 'A'
 836             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 837         else:
 838             prompt = "Manual reject, [S]kip, Quit ?"
 839
 840         while prompt.find(answer) == -1:
 841             answer = dak.lib.utils.our_raw_input(prompt)
 842             m = dak.lib.queue.re_default_answer.search(prompt)
 843             if answer == "":
 844                 answer = m.group(1)
 845             answer = answer[:1].upper()
 846
 847         if answer == 'A':
 848             done = 1
 849             for file in byhand:
 850                 del files[file]
 851         elif answer == 'M':
 852             Upload.do_reject(1, Options["Manual-Reject"])
 853             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 854             done = 1
 855         elif answer == 'S':
 856             done = 1
 857         elif answer == 'Q':
 858             sys.exit(0)
 859
 860 ################################################################################
 861
 862 def do_accept():
 863     print "ACCEPT"
 864     if not Options["No-Action"]:
 865         retry = 0
 866         while retry < 10:
 867             try:
 868                 lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 869                 retry = 10
 870             except OSError, e:
 871                 if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
 872                     retry += 1
 873                     if (retry >= 10):
 874                         dak.lib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 875                     else:
 876                         print("Unable to get accepted lock (try %d of 10)" % retry)
 877                     time.sleep(60)
 878                 else:
 879                     raise
 880         (summary, short_summary) = Upload.build_summaries()
 881         Upload.accept(summary, short_summary)
 882         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 883         os.unlink(Cnf["Process-New::AcceptedLockFile"])
 884
 885 def check_status(files):
 886     new = byhand = 0
 887     for file in files.keys():
 888         if files[file]["type"] == "byhand":
 889             byhand = 1
 890         elif files[file].has_key("new"):
 891             new = 1
 892     return (new, byhand)
 893
 894 def do_pkg(changes_file):
 895     Upload.pkg.changes_file = changes_file
 896     Upload.init_vars()
 897     Upload.update_vars()
 898     Upload.update_subst()
 899     files = Upload.pkg.files
 900
 901     if not recheck():
 902         return
 903
 904     (new, byhand) = check_status(files)
 905     if new or byhand:
 906         if new:
 907             do_new()
 908         if byhand:
 909             do_byhand()
 910         (new, byhand) = check_status(files)
 911
 912     if not new and not byhand:
 913         do_accept()
 914
 915 ################################################################################
 916
 917 def end():
 918     accept_count = Upload.accept_count
 919     accept_bytes = Upload.accept_bytes
 920
 921     if accept_count:
 922         sets = "set"
 923         if accept_count > 1:
 924             sets = "sets"
 925         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, dak.lib.utils.size_type(int(accept_bytes))))
 926         Logger.log(["total",accept_count,accept_bytes])
 927
 928     if not Options["No-Action"]:
 929         Logger.close()
 930
 931 ################################################################################
 932
 933 def main():
 934     changes_files = init()
 935     if len(changes_files) > 50:
 936         sys.stderr.write("Sorting changes...\n")
 937     changes_files = sort_changes(changes_files)
 938
 939     # Kill me now? **FIXME**
 940     Cnf["Dinstall::Options::No-Mail"] = ""
 941     bcc = "X-DAK: dak process-new\nX-Katie: this header is obsolete"
 942     if Cnf.has_key("Dinstall::Bcc"):
 943         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
 944     else:
 945         Upload.Subst["__BCC__"] = bcc
 946
 947     for changes_file in changes_files:
 948         changes_file = dak.lib.utils.validate_changes_file_arg(changes_file, 0)
 949         if not changes_file:
 950             continue
 951         print "\n" + changes_file
 952         do_pkg (changes_file)
 953
 954     end()
 955
 956 ################################################################################
 957
 958 if __name__ == '__main__':
 959     main()