dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 # Handles NEW and BYHAND packages
   5 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 # 23:12|<aj> I will not hush!
  24 # 23:12|<elmo> :>
  25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  26 # 23:13|<aj> I shall not be silenced!
  27 # 23:13|<aj> The world shall know!
  28 # 23:13|<aj> The world *must* know!
  29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  30 # 23:13|<aj> yay powerpuff girls!!
  31 # 23:13|<aj> buttercup's my favourite, who's yours?
  32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  33 # 23:14|<aj> *AREN'T YOU*?!
  34 # 23:15|<aj> I will not be treated like this.
  35 # 23:15|<aj> I shall have my revenge.
  36 # 23:15|<aj> I SHALL!!!
  37
  38 ################################################################################
  39
  40 import copy, errno, os, readline, stat, sys, time
  41 import apt_pkg, apt_inst
  42 import examine_package
  43 from daklib import database
  44 from daklib import logging
  45 from daklib import queue
  46 from daklib import utils
  47 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  48
  49 # Globals
  50 Cnf = None
  51 Options = None
  52 Upload = None
  53 projectB = None
  54 Logger = None
  55
  56 Priorities = None
  57 Sections = None
  58
  59 reject_message = ""
  60
  61 ################################################################################
  62 ################################################################################
  63 ################################################################################
  64
  65 def reject (str, prefix="Rejected: "):
  66     global reject_message
  67     if str:
  68         reject_message += prefix + str + "\n"
  69
  70 def recheck():
  71     global reject_message
  72     files = Upload.pkg.files
  73     reject_message = ""
  74
  75     for f in files.keys():
  76         # The .orig.tar.gz can disappear out from under us is it's a
  77         # duplicate of one in the archive.
  78         if not files.has_key(f):
  79             continue
  80         # Check that the source still exists
  81         if files[f]["type"] == "deb":
  82             source_version = files[f]["source version"]
  83             source_package = files[f]["source package"]
  84             if not Upload.pkg.changes["architecture"].has_key("source") \
  85                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  86                 source_epochless_version = re_no_epoch.sub('', source_version)
  87                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  88                 found = 0
  89                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  90                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  91                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  92                             found = 1
  93                 if not found:
  94                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  95
  96         # Version and file overwrite checks
  97         if files[f]["type"] == "deb":
  98             reject(Upload.check_binary_against_db(f), "")
  99         elif files[f]["type"] == "dsc":
 100             reject(Upload.check_source_against_db(f), "")
 101             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 102             reject(reject_msg, "")
 103
 104     if reject_message.find("Rejected") != -1:
 105         answer = "XXX"
 106         if Options["No-Action"] or Options["Automatic"]:
 107             answer = 'S'
 108
 109         print "REJECT\n" + reject_message,
 110         prompt = "[R]eject, Skip, Quit ?"
 111
 112         while prompt.find(answer) == -1:
 113             answer = utils.our_raw_input(prompt)
 114             m = re_default_answer.match(prompt)
 115             if answer == "":
 116                 answer = m.group(1)
 117             answer = answer[:1].upper()
 118
 119         if answer == 'R':
 120             Upload.do_reject(0, reject_message)
 121             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 122             return 0
 123         elif answer == 'S':
 124             return 0
 125         elif answer == 'Q':
 126             end()
 127             sys.exit(0)
 128
 129     return 1
 130
 131 ################################################################################
 132
 133 def indiv_sg_compare (a, b):
 134     """Sort by source name, source, version, 'have source', and
 135        finally by filename."""
 136     # Sort by source version
 137     q = apt_pkg.VersionCompare(a["version"], b["version"])
 138     if q:
 139         return -q
 140
 141     # Sort by 'have source'
 142     a_has_source = a["architecture"].get("source")
 143     b_has_source = b["architecture"].get("source")
 144     if a_has_source and not b_has_source:
 145         return -1
 146     elif b_has_source and not a_has_source:
 147         return 1
 148
 149     return cmp(a["filename"], b["filename"])
 150
 151 ############################################################
 152
 153 def sg_compare (a, b):
 154     a = a[1]
 155     b = b[1]
 156     """Sort by have note, source already in database and time of oldest upload."""
 157     # Sort by have note
 158     a_note_state = a["note_state"]
 159     b_note_state = b["note_state"]
 160     if a_note_state < b_note_state:
 161         return -1
 162     elif a_note_state > b_note_state:
 163         return 1
 164     # Sort by source already in database (descending)
 165     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 166     if source_in_database:
 167         return -source_in_database
 168
 169     # Sort by time of oldest upload
 170     return cmp(a["oldest"], b["oldest"])
 171
 172 def sort_changes(changes_files):
 173     """Sort into source groups, then sort each source group by version,
 174     have source, filename.  Finally, sort the source groups by have
 175     note, time of oldest upload of each source upload."""
 176     if len(changes_files) == 1:
 177         return changes_files
 178
 179     sorted_list = []
 180     cache = {}
 181     # Read in all the .changes files
 182     for filename in changes_files:
 183         try:
 184             Upload.pkg.changes_file = filename
 185             Upload.init_vars()
 186             Upload.update_vars()
 187             cache[filename] = copy.copy(Upload.pkg.changes)
 188             cache[filename]["filename"] = filename
 189         except:
 190             sorted_list.append(filename)
 191             break
 192     # Divide the .changes into per-source groups
 193     per_source = {}
 194     for filename in cache.keys():
 195         source = cache[filename]["source"]
 196         if not per_source.has_key(source):
 197             per_source[source] = {}
 198             per_source[source]["list"] = []
 199         per_source[source]["list"].append(cache[filename])
 200     # Determine oldest time and have note status for each source group
 201     for source in per_source.keys():
 202         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 203         ql = q.getresult()
 204         per_source[source]["source_in_database"] = len(ql)>0
 205         source_list = per_source[source]["list"]
 206         first = source_list[0]
 207         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 208         have_note = 0
 209         for d in per_source[source]["list"]:
 210             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 211             if mtime < oldest:
 212                 oldest = mtime
 213             have_note += (d.has_key("process-new note"))
 214         per_source[source]["oldest"] = oldest
 215         if not have_note:
 216             per_source[source]["note_state"] = 0; # none
 217         elif have_note < len(source_list):
 218             per_source[source]["note_state"] = 1; # some
 219         else:
 220             per_source[source]["note_state"] = 2; # all
 221         per_source[source]["list"].sort(indiv_sg_compare)
 222     per_source_items = per_source.items()
 223     per_source_items.sort(sg_compare)
 224     for i in per_source_items:
 225         for j in i[1]["list"]:
 226             sorted_list.append(j["filename"])
 227     return sorted_list
 228
 229 ################################################################################
 230
 231 class Section_Completer:
 232     def __init__ (self):
 233         self.sections = []
 234         q = projectB.query("SELECT section FROM section")
 235         for i in q.getresult():
 236             self.sections.append(i[0])
 237
 238     def complete(self, text, state):
 239         if state == 0:
 240             self.matches = []
 241             n = len(text)
 242             for word in self.sections:
 243                 if word[:n] == text:
 244                     self.matches.append(word)
 245         try:
 246             return self.matches[state]
 247         except IndexError:
 248             return None
 249
 250 ############################################################
 251
 252 class Priority_Completer:
 253     def __init__ (self):
 254         self.priorities = []
 255         q = projectB.query("SELECT priority FROM priority")
 256         for i in q.getresult():
 257             self.priorities.append(i[0])
 258
 259     def complete(self, text, state):
 260         if state == 0:
 261             self.matches = []
 262             n = len(text)
 263             for word in self.priorities:
 264                 if word[:n] == text:
 265                     self.matches.append(word)
 266         try:
 267             return self.matches[state]
 268         except IndexError:
 269             return None
 270
 271 ################################################################################
 272
 273 def print_new (new, indexed, file=sys.stdout):
 274     queue.check_valid(new)
 275     broken = 0
 276     index = 0
 277     for pkg in new.keys():
 278         index += 1
 279         section = new[pkg]["section"]
 280         priority = new[pkg]["priority"]
 281         if new[pkg]["section id"] == -1:
 282             section += "[!]"
 283             broken = 1
 284         if new[pkg]["priority id"] == -1:
 285             priority += "[!]"
 286             broken = 1
 287         if indexed:
 288             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 289         else:
 290             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 291         line = line.strip()+'\n'
 292         file.write(line)
 293     note = Upload.pkg.changes.get("process-new note")
 294     if note:
 295         print "*"*75
 296         print note
 297         print "*"*75
 298     return broken, note
 299
 300 ################################################################################
 301
 302 def index_range (index):
 303     if index == 1:
 304         return "1"
 305     else:
 306         return "1-%s" % (index)
 307
 308 ################################################################################
 309 ################################################################################
 310
 311 def edit_new (new):
 312     # Write the current data to a temporary file
 313     (fd, temp_filename) = utils.temp_filename()
 314     temp_file = os.fdopen(fd, 'w')
 315     print_new (new, 0, temp_file)
 316     temp_file.close()
 317     # Spawn an editor on that file
 318     editor = os.environ.get("EDITOR","vi")
 319     result = os.system("%s %s" % (editor, temp_filename))
 320     if result != 0:
 321         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 322     # Read the edited data back in
 323     temp_file = utils.open_file(temp_filename)
 324     lines = temp_file.readlines()
 325     temp_file.close()
 326     os.unlink(temp_filename)
 327     # Parse the new data
 328     for line in lines:
 329         line = line.strip()
 330         if line == "":
 331             continue
 332         s = line.split()
 333         # Pad the list if necessary
 334         s[len(s):3] = [None] * (3-len(s))
 335         (pkg, priority, section) = s[:3]
 336         if not new.has_key(pkg):
 337             utils.warn("Ignoring unknown package '%s'" % (pkg))
 338         else:
 339             # Strip off any invalid markers, print_new will readd them.
 340             if section.endswith("[!]"):
 341                 section = section[:-3]
 342             if priority.endswith("[!]"):
 343                 priority = priority[:-3]
 344             for f in new[pkg]["files"]:
 345                 Upload.pkg.files[f]["section"] = section
 346                 Upload.pkg.files[f]["priority"] = priority
 347             new[pkg]["section"] = section
 348             new[pkg]["priority"] = priority
 349
 350 ################################################################################
 351
 352 def edit_index (new, index):
 353     priority = new[index]["priority"]
 354     section = new[index]["section"]
 355     ftype = new[index]["type"]
 356     done = 0
 357     while not done:
 358         print "\t".join([index, priority, section])
 359
 360         answer = "XXX"
 361         if ftype != "dsc":
 362             prompt = "[B]oth, Priority, Section, Done ? "
 363         else:
 364             prompt = "[S]ection, Done ? "
 365         edit_priority = edit_section = 0
 366
 367         while prompt.find(answer) == -1:
 368             answer = utils.our_raw_input(prompt)
 369             m = re_default_answer.match(prompt)
 370             if answer == "":
 371                 answer = m.group(1)
 372             answer = answer[:1].upper()
 373
 374         if answer == 'P':
 375             edit_priority = 1
 376         elif answer == 'S':
 377             edit_section = 1
 378         elif answer == 'B':
 379             edit_priority = edit_section = 1
 380         elif answer == 'D':
 381             done = 1
 382
 383         # Edit the priority
 384         if edit_priority:
 385             readline.set_completer(Priorities.complete)
 386             got_priority = 0
 387             while not got_priority:
 388                 new_priority = utils.our_raw_input("New priority: ").strip()
 389                 if new_priority not in Priorities.priorities:
 390                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 391                 else:
 392                     got_priority = 1
 393                     priority = new_priority
 394
 395         # Edit the section
 396         if edit_section:
 397             readline.set_completer(Sections.complete)
 398             got_section = 0
 399             while not got_section:
 400                 new_section = utils.our_raw_input("New section: ").strip()
 401                 if new_section not in Sections.sections:
 402                     print "E: '%s' is not a valid section, try again." % (new_section)
 403                 else:
 404                     got_section = 1
 405                     section = new_section
 406
 407         # Reset the readline completer
 408         readline.set_completer(None)
 409
 410     for f in new[index]["files"]:
 411         Upload.pkg.files[f]["section"] = section
 412         Upload.pkg.files[f]["priority"] = priority
 413     new[index]["priority"] = priority
 414     new[index]["section"] = section
 415     return new
 416
 417 ################################################################################
 418
 419 def edit_overrides (new):
 420     print
 421     done = 0
 422     while not done:
 423         print_new (new, 1)
 424         new_index = {}
 425         index = 0
 426         for i in new.keys():
 427             index += 1
 428             new_index[index] = i
 429
 430         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 431
 432         got_answer = 0
 433         while not got_answer:
 434             answer = utils.our_raw_input(prompt)
 435             if not answer.isdigit():
 436                 answer = answer[:1].upper()
 437             if answer == "E" or answer == "D":
 438                 got_answer = 1
 439             elif re_isanum.match (answer):
 440                 answer = int(answer)
 441                 if (answer < 1) or (answer > index):
 442                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 443                 else:
 444                     got_answer = 1
 445
 446         if answer == 'E':
 447             edit_new(new)
 448         elif answer == 'D':
 449             done = 1
 450         else:
 451             edit_index (new, new_index[answer])
 452
 453     return new
 454
 455 ################################################################################
 456
 457 def edit_note(note):
 458     # Write the current data to a temporary file
 459     (fd, temp_filename) = utils.temp_filename()
 460     temp_file = os.fdopen(temp_filename, 'w')
 461     temp_file.write(note)
 462     temp_file.close()
 463     editor = os.environ.get("EDITOR","vi")
 464     answer = 'E'
 465     while answer == 'E':
 466         os.system("%s %s" % (editor, temp_filename))
 467         temp_file = utils.open_file(temp_filename)
 468         note = temp_file.read().rstrip()
 469         temp_file.close()
 470         print "Note:"
 471         print utils.prefix_multi_line_string(note,"  ")
 472         prompt = "[D]one, Edit, Abandon, Quit ?"
 473         answer = "XXX"
 474         while prompt.find(answer) == -1:
 475             answer = utils.our_raw_input(prompt)
 476             m = re_default_answer.search(prompt)
 477             if answer == "":
 478                 answer = m.group(1)
 479             answer = answer[:1].upper()
 480     os.unlink(temp_filename)
 481     if answer == 'A':
 482         return
 483     elif answer == 'Q':
 484         end()
 485         sys.exit(0)
 486     Upload.pkg.changes["process-new note"] = note
 487     Upload.dump_vars(Cnf["Dir::Queue::New"])
 488
 489 ################################################################################
 490
 491 def check_pkg ():
 492     try:
 493         less_fd = os.popen("less -R -", 'w', 0)
 494         stdout_fd = sys.stdout
 495         try:
 496             sys.stdout = less_fd
 497             changes = utils.parse_changes (Upload.pkg.changes_file)
 498             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 499             files = Upload.pkg.files
 500             for f in files.keys():
 501                 if files[f].has_key("new"):
 502                     ftype = files[f]["type"]
 503                     if ftype == "deb":
 504                         examine_package.check_deb(changes['distribution'], f)
 505                     elif ftype == "dsc":
 506                         examine_package.check_dsc(changes['distribution'], f)
 507         finally:
 508             sys.stdout = stdout_fd
 509     except IOError, e:
 510         if e.errno == errno.EPIPE:
 511             utils.warn("[examine_package] Caught EPIPE; skipping.")
 512             pass
 513         else:
 514             raise
 515     except KeyboardInterrupt:
 516         utils.warn("[examine_package] Caught C-c; skipping.")
 517         pass
 518
 519 ################################################################################
 520
 521 ## FIXME: horribly Debian specific
 522
 523 def do_bxa_notification():
 524     files = Upload.pkg.files
 525     summary = ""
 526     for f in files.keys():
 527         if files[f]["type"] == "deb":
 528             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 529             summary += "\n"
 530             summary += "Package: %s\n" % (control.Find("Package"))
 531             summary += "Description: %s\n" % (control.Find("Description"))
 532     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 533     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 534     utils.send_mail(bxa_mail)
 535
 536 ################################################################################
 537
 538 def add_overrides (new):
 539     changes = Upload.pkg.changes
 540     files = Upload.pkg.files
 541
 542     projectB.query("BEGIN WORK")
 543     for suite in changes["suite"].keys():
 544         suite_id = database.get_suite_id(suite)
 545         for pkg in new.keys():
 546             component_id = database.get_component_id(new[pkg]["component"])
 547             type_id = database.get_override_type_id(new[pkg]["type"])
 548             priority_id = new[pkg]["priority id"]
 549             section_id = new[pkg]["section id"]
 550             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 551             for f in new[pkg]["files"]:
 552                 if files[f].has_key("new"):
 553                     del files[f]["new"]
 554             del new[pkg]
 555
 556     projectB.query("COMMIT WORK")
 557
 558     if Cnf.FindB("Dinstall::BXANotify"):
 559         do_bxa_notification()
 560
 561 ################################################################################
 562
 563 def prod_maintainer ():
 564     # Here we prepare an editor and get them ready to prod...
 565     (fd, temp_filename) = utils.temp_filename()
 566     editor = os.environ.get("EDITOR","vi")
 567     answer = 'E'
 568     while answer == 'E':
 569         os.system("%s %s" % (editor, temp_filename))
 570         f = os.fdopen(fd)
 571         prod_message = "".join(f.readlines())
 572         f.close()
 573         print "Prod message:"
 574         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 575         prompt = "[P]rod, Edit, Abandon, Quit ?"
 576         answer = "XXX"
 577         while prompt.find(answer) == -1:
 578             answer = utils.our_raw_input(prompt)
 579             m = re_default_answer.search(prompt)
 580             if answer == "":
 581                 answer = m.group(1)
 582             answer = answer[:1].upper()
 583         os.unlink(temp_filename)
 584         if answer == 'A':
 585             return
 586         elif answer == 'Q':
 587             end()
 588             sys.exit(0)
 589     # Otherwise, do the proding...
 590     user_email_address = utils.whoami() + " <%s>" % (
 591         Cnf["Dinstall::MyAdminAddress"])
 592
 593     Subst = Upload.Subst
 594
 595     Subst["__FROM_ADDRESS__"] = user_email_address
 596     Subst["__PROD_MESSAGE__"] = prod_message
 597     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 598
 599     prod_mail_message = utils.TemplateSubst(
 600         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 601
 602     # Send the prod mail if appropriate
 603     if not Cnf["Dinstall::Options::No-Mail"]:
 604         utils.send_mail(prod_mail_message)
 605
 606     print "Sent proding message"
 607
 608 ################################################################################
 609
 610 def do_new():
 611     print "NEW\n"
 612     files = Upload.pkg.files
 613     changes = Upload.pkg.changes
 614
 615     # Make a copy of distribution we can happily trample on
 616     changes["suite"] = copy.copy(changes["distribution"])
 617
 618     # Fix up the list of target suites
 619     for suite in changes["suite"].keys():
 620         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 621         if override:
 622             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 623               database.get_suite_id(override) == -1)
 624             if olderr or newerr:
 625                 (oinv, newinv) = ("", "")
 626                 if olderr: oinv = "invalid "
 627                 if newerr: ninv = "invalid "
 628                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 629                         oinv, suite, ninv, override)
 630             del changes["suite"][suite]
 631             changes["suite"][override] = 1
 632     # Validate suites
 633     for suite in changes["suite"].keys():
 634         suite_id = database.get_suite_id(suite)
 635         if suite_id == -1:
 636             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 637
 638     # The main NEW processing loop
 639     done = 0
 640     while not done:
 641         # Find out what's new
 642         new = queue.determine_new(changes, files, projectB)
 643
 644         if not new:
 645             break
 646
 647         answer = "XXX"
 648         if Options["No-Action"] or Options["Automatic"]:
 649             answer = 'S'
 650
 651         (broken, note) = print_new(new, 0)
 652         prompt = ""
 653
 654         if not broken and not note:
 655             prompt = "Add overrides, "
 656         if broken:
 657             print "W: [!] marked entries must be fixed before package can be processed."
 658         if note:
 659             print "W: note must be removed before package can be processed."
 660             prompt += "Remove note, "
 661
 662         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 663
 664         while prompt.find(answer) == -1:
 665             answer = utils.our_raw_input(prompt)
 666             m = re_default_answer.search(prompt)
 667             if answer == "":
 668                 answer = m.group(1)
 669             answer = answer[:1].upper()
 670
 671         if answer == 'A':
 672             done = add_overrides (new)
 673         elif answer == 'C':
 674             check_pkg()
 675         elif answer == 'E':
 676             new = edit_overrides (new)
 677         elif answer == 'M':
 678             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 679             if not aborted:
 680                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 681                 done = 1
 682         elif answer == 'N':
 683             edit_note(changes.get("process-new note", ""))
 684         elif answer == 'P':
 685             prod_maintainer()
 686         elif answer == 'R':
 687             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 688             if confirm == "y":
 689                 del changes["process-new note"]
 690         elif answer == 'S':
 691             done = 1
 692         elif answer == 'Q':
 693             end()
 694             sys.exit(0)
 695
 696 ################################################################################
 697 ################################################################################
 698 ################################################################################
 699
 700 def usage (exit_code=0):
 701     print """Usage: dak process-new [OPTION]... [CHANGES]...
 702   -a, --automatic           automatic run
 703   -h, --help                show this help and exit.
 704   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 705   -m, --manual-reject=MSG   manual reject with `msg'
 706   -n, --no-action           don't do anything
 707   -V, --version             display the version number and exit"""
 708     sys.exit(exit_code)
 709
 710 ################################################################################
 711
 712 def init():
 713     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 714
 715     Cnf = utils.get_conf()
 716
 717     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 718                  ('h',"help","Process-New::Options::Help"),
 719                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 720                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 721                  ('n',"no-action","Process-New::Options::No-Action")]
 722
 723     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 724         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 725             Cnf["Process-New::Options::%s" % (i)] = ""
 726
 727     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 728     Options = Cnf.SubTree("Process-New::Options")
 729
 730     if Options["Help"]:
 731         usage()
 732
 733     Upload = queue.Upload(Cnf)
 734
 735     if not Options["No-Action"]:
 736         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 737
 738     projectB = Upload.projectB
 739
 740     Sections = Section_Completer()
 741     Priorities = Priority_Completer()
 742     readline.parse_and_bind("tab: complete")
 743
 744     return changes_files
 745
 746 ################################################################################
 747
 748 def do_byhand():
 749     done = 0
 750     while not done:
 751         files = Upload.pkg.files
 752         will_install = 1
 753         byhand = []
 754
 755         for f in files.keys():
 756             if files[f]["type"] == "byhand":
 757                 if os.path.exists(f):
 758                     print "W: %s still present; please process byhand components and try again." % (f)
 759                     will_install = 0
 760                 else:
 761                     byhand.append(f)
 762
 763         answer = "XXXX"
 764         if Options["No-Action"]:
 765             answer = "S"
 766         if will_install:
 767             if Options["Automatic"] and not Options["No-Action"]:
 768                 answer = 'A'
 769             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 770         else:
 771             prompt = "Manual reject, [S]kip, Quit ?"
 772
 773         while prompt.find(answer) == -1:
 774             answer = utils.our_raw_input(prompt)
 775             m = re_default_answer.search(prompt)
 776             if answer == "":
 777                 answer = m.group(1)
 778             answer = answer[:1].upper()
 779
 780         if answer == 'A':
 781             done = 1
 782             for f in byhand:
 783                 del files[f]
 784         elif answer == 'M':
 785             Upload.do_reject(1, Options["Manual-Reject"])
 786             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 787             done = 1
 788         elif answer == 'S':
 789             done = 1
 790         elif answer == 'Q':
 791             end()
 792             sys.exit(0)
 793
 794 ################################################################################
 795
 796 def get_accept_lock():
 797     retry = 0
 798     while retry < 10:
 799         try:
 800             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 801             retry = 10
 802         except OSError, e:
 803             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 804                 retry += 1
 805                 if (retry >= 10):
 806                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 807                 else:
 808                     print("Unable to get accepted lock (try %d of 10)" % retry)
 809                 time.sleep(60)
 810             else:
 811                 raise
 812
 813 def move_to_dir (dest, perms=0660, changesperms=0664):
 814     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 815     file_keys = Upload.pkg.files.keys()
 816     for f in file_keys:
 817         utils.move (f, dest, perms=perms)
 818
 819 def is_source_in_queue_dir(qdir):
 820     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 821                 and x.endswith(".changes") ]
 822     for entry in entries:
 823         # read the .dak
 824         u = queue.Upload(Cnf)
 825         u.pkg.changes_file = os.path.join(qdir, entry)
 826         u.update_vars()
 827         if not u.pkg.changes["architecture"].has_key("source"):
 828             # another binary upload, ignore
 829             continue
 830         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 831             # another version, ignore
 832             continue
 833         # found it!
 834         return True
 835     return False
 836
 837 def move_to_holding(suite, queue_dir):
 838     print "Moving to %s holding area." % (suite.upper(),)
 839     if Options["No-Action"]:
 840         return
 841     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 842     Upload.dump_vars(queue_dir)
 843     move_to_dir(queue_dir)
 844     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 845
 846 def _accept():
 847     if Options["No-Action"]:
 848         return
 849     (summary, short_summary) = Upload.build_summaries()
 850     Upload.accept(summary, short_summary)
 851     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 852
 853 def do_accept_stableupdate(suite, q):
 854     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 855     if not Upload.pkg.changes["architecture"].has_key("source"):
 856         # It is not a sourceful upload.  So its source may be either in p-u
 857         # holding, in new, in accepted or already installed.
 858         if is_source_in_queue_dir(queue_dir):
 859             # It's in p-u holding, so move it there.
 860             print "Binary-only upload, source in %s." % (q,)
 861             move_to_holding(suite, queue_dir)
 862         elif Upload.source_exists(Upload.pkg.changes["source"],
 863                 Upload.pkg.changes["version"]):
 864             # dak tells us that there is source available.  At time of
 865             # writing this means that it is installed, so put it into
 866             # accepted.
 867             print "Binary-only upload, source installed."
 868             _accept()
 869         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 870             # The source is in accepted, the binary cleared NEW: accept it.
 871             print "Binary-only upload, source in accepted."
 872             _accept()
 873         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 874             # It's in NEW.  We expect the source to land in p-u holding
 875             # pretty soon.
 876             print "Binary-only upload, source in new."
 877             move_to_holding(suite, queue_dir)
 878         else:
 879             # No case applicable.  Bail out.  Return will cause the upload
 880             # to be skipped.
 881             print "ERROR"
 882             print "Stable update failed.  Source not found."
 883             return
 884     else:
 885         # We are handling a sourceful upload.  Move to accepted if currently
 886         # in p-u holding and to p-u holding otherwise.
 887         if is_source_in_queue_dir(queue_dir):
 888             print "Sourceful upload in %s, accepting." % (q,)
 889             _accept()
 890         else:
 891             move_to_holding(suite, queue_dir)
 892
 893 def do_accept():
 894     print "ACCEPT"
 895     if not Options["No-Action"]:
 896         get_accept_lock()
 897         (summary, short_summary) = Upload.build_summaries()
 898     try:
 899         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 900             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 901             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 902             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 903             # Check for override disparities
 904             Upload.Subst["__SUMMARY__"] = summary
 905         else:
 906             # Stable updates need to be copied to proposed-updates holding
 907             # area instead of accepted.  Sourceful uploads need to go
 908             # to it directly, binaries only if the source has not yet been
 909             # accepted into p-u.
 910             for suite, q in [("proposed-updates", "ProposedUpdates"),
 911                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 912                 if not Upload.pkg.changes["distribution"].has_key(suite):
 913                     continue
 914                 return do_accept_stableupdate(suite, q)
 915             # Just a normal upload, accept it...
 916             _accept()
 917     finally:
 918         if not Options["No-Action"]:
 919             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 920
 921 def check_status(files):
 922     new = byhand = 0
 923     for f in files.keys():
 924         if files[f]["type"] == "byhand":
 925             byhand = 1
 926         elif files[f].has_key("new"):
 927             new = 1
 928     return (new, byhand)
 929
 930 def do_pkg(changes_file):
 931     Upload.pkg.changes_file = changes_file
 932     Upload.init_vars()
 933     Upload.update_vars()
 934     Upload.update_subst()
 935     files = Upload.pkg.files
 936
 937     if not recheck():
 938         return
 939
 940     (new, byhand) = check_status(files)
 941     if new or byhand:
 942         if new:
 943             do_new()
 944         if byhand:
 945             do_byhand()
 946         (new, byhand) = check_status(files)
 947
 948     if not new and not byhand:
 949         do_accept()
 950
 951 ################################################################################
 952
 953 def end():
 954     accept_count = Upload.accept_count
 955     accept_bytes = Upload.accept_bytes
 956
 957     if accept_count:
 958         sets = "set"
 959         if accept_count > 1:
 960             sets = "sets"
 961         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 962         Logger.log(["total",accept_count,accept_bytes])
 963
 964     if not Options["No-Action"]:
 965         Logger.close()
 966
 967 ################################################################################
 968
 969 def do_comments(dir, opref, npref, line, fn):
 970     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 971         lines = open("%s/%s" % (dir, comm)).readlines()
 972         if len(lines) == 0 or lines[0] != line + "\n": continue
 973         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 974                                 and x.endswith(".changes") ]
 975         changes_files = sort_changes(changes_files)
 976         for f in changes_files:
 977             f = utils.validate_changes_file_arg(f, 0)
 978             if not f: continue
 979             print "\n" + f
 980             fn(f, "".join(lines[1:]))
 981
 982         if opref != npref and not Options["No-Action"]:
 983             newcomm = npref + comm[len(opref):]
 984             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 985
 986 ################################################################################
 987
 988 def comment_accept(changes_file, comments):
 989     Upload.pkg.changes_file = changes_file
 990     Upload.init_vars()
 991     Upload.update_vars()
 992     Upload.update_subst()
 993     files = Upload.pkg.files
 994
 995     if not recheck():
 996         return # dak wants to REJECT, crap
 997
 998     (new, byhand) = check_status(files)
 999     if not new and not byhand:
1000         do_accept()
1001
1002 ################################################################################
1003
1004 def comment_reject(changes_file, comments):
1005     Upload.pkg.changes_file = changes_file
1006     Upload.init_vars()
1007     Upload.update_vars()
1008     Upload.update_subst()
1009
1010     if not recheck():
1011         pass # dak has its own reasons to reject as well, which is fine
1012
1013     reject(comments)
1014     print "REJECT\n" + reject_message,
1015     if not Options["No-Action"]:
1016         Upload.do_reject(0, reject_message)
1017         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1018
1019 ################################################################################
1020
1021 def main():
1022     changes_files = init()
1023     if len(changes_files) > 50:
1024         sys.stderr.write("Sorting changes...\n")
1025     changes_files = sort_changes(changes_files)
1026
1027     # Kill me now? **FIXME**
1028     Cnf["Dinstall::Options::No-Mail"] = ""
1029     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1030     if Cnf.has_key("Dinstall::Bcc"):
1031         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1032     else:
1033         Upload.Subst["__BCC__"] = bcc
1034
1035     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1036     if commentsdir:
1037         if changes_files != []:
1038             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1039             sys.exit(1)
1040         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1041         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1042     else:
1043         for changes_file in changes_files:
1044             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1045             if not changes_file:
1046                 continue
1047             print "\n" + changes_file
1048             do_pkg (changes_file)
1049
1050     end()
1051
1052 ################################################################################
1053
1054 if __name__ == '__main__':
1055     main()