dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 # Handles NEW and BYHAND packages
   5 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 # 23:12|<aj> I will not hush!
  24 # 23:12|<elmo> :>
  25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  26 # 23:13|<aj> I shall not be silenced!
  27 # 23:13|<aj> The world shall know!
  28 # 23:13|<aj> The world *must* know!
  29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  30 # 23:13|<aj> yay powerpuff girls!!
  31 # 23:13|<aj> buttercup's my favourite, who's yours?
  32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  33 # 23:14|<aj> *AREN'T YOU*?!
  34 # 23:15|<aj> I will not be treated like this.
  35 # 23:15|<aj> I shall have my revenge.
  36 # 23:15|<aj> I SHALL!!!
  37
  38 ################################################################################
  39
  40 import copy, errno, os, readline, stat, sys, time
  41 import apt_pkg, apt_inst
  42 import examine_package
  43 from daklib import database
  44 from daklib import logging
  45 from daklib import queue
  46 from daklib import utils
  47 from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
  48
  49 # Globals
  50 Cnf = None
  51 Options = None
  52 Upload = None
  53 projectB = None
  54 Logger = None
  55
  56 Priorities = None
  57 Sections = None
  58
  59 reject_message = ""
  60
  61 ################################################################################
  62 ################################################################################
  63 ################################################################################
  64
  65 def reject (str, prefix="Rejected: "):
  66     global reject_message
  67     if str:
  68         reject_message += prefix + str + "\n"
  69
  70 def recheck():
  71     global reject_message
  72     files = Upload.pkg.files
  73     reject_message = ""
  74
  75     for f in files.keys():
  76         # The .orig.tar.gz can disappear out from under us is it's a
  77         # duplicate of one in the archive.
  78         if not files.has_key(f):
  79             continue
  80         # Check that the source still exists
  81         if files[f]["type"] == "deb":
  82             source_version = files[f]["source version"]
  83             source_package = files[f]["source package"]
  84             if not Upload.pkg.changes["architecture"].has_key("source") \
  85                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  86                 source_epochless_version = re_no_epoch.sub('', source_version)
  87                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  88                 found = 0
  89                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  90                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  91                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  92                             found = 1
  93                 if not found:
  94                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  95
  96         # Version and file overwrite checks
  97         if files[f]["type"] == "deb":
  98             reject(Upload.check_binary_against_db(f), "")
  99         elif files[f]["type"] == "dsc":
 100             reject(Upload.check_source_against_db(f), "")
 101             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 102             reject(reject_msg, "")
 103
 104     if reject_message.find("Rejected") != -1:
 105         answer = "XXX"
 106         if Options["No-Action"] or Options["Automatic"]:
 107             answer = 'S'
 108
 109         print "REJECT\n" + reject_message,
 110         prompt = "[R]eject, Skip, Quit ?"
 111
 112         while prompt.find(answer) == -1:
 113             answer = utils.our_raw_input(prompt)
 114             m = re_default_answer.match(prompt)
 115             if answer == "":
 116                 answer = m.group(1)
 117             answer = answer[:1].upper()
 118
 119         if answer == 'R':
 120             Upload.do_reject(0, reject_message)
 121             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 122             return 0
 123         elif answer == 'S':
 124             return 0
 125         elif answer == 'Q':
 126             end()
 127             sys.exit(0)
 128
 129     return 1
 130
 131 ################################################################################
 132
 133 def indiv_sg_compare (a, b):
 134     """Sort by source name, source, version, 'have source', and
 135        finally by filename."""
 136     # Sort by source version
 137     q = apt_pkg.VersionCompare(a["version"], b["version"])
 138     if q:
 139         return -q
 140
 141     # Sort by 'have source'
 142     a_has_source = a["architecture"].get("source")
 143     b_has_source = b["architecture"].get("source")
 144     if a_has_source and not b_has_source:
 145         return -1
 146     elif b_has_source and not a_has_source:
 147         return 1
 148
 149     return cmp(a["filename"], b["filename"])
 150
 151 ############################################################
 152
 153 def sg_compare (a, b):
 154     a = a[1]
 155     b = b[1]
 156     """Sort by have note, source already in database and time of oldest upload."""
 157     # Sort by have note
 158     a_note_state = a["note_state"]
 159     b_note_state = b["note_state"]
 160     if a_note_state < b_note_state:
 161         return -1
 162     elif a_note_state > b_note_state:
 163         return 1
 164     # Sort by source already in database (descending)
 165     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 166     if source_in_database:
 167         return -source_in_database
 168
 169     # Sort by time of oldest upload
 170     return cmp(a["oldest"], b["oldest"])
 171
 172 def sort_changes(changes_files):
 173     """Sort into source groups, then sort each source group by version,
 174     have source, filename.  Finally, sort the source groups by have
 175     note, time of oldest upload of each source upload."""
 176     if len(changes_files) == 1:
 177         return changes_files
 178
 179     sorted_list = []
 180     cache = {}
 181     # Read in all the .changes files
 182     for filename in changes_files:
 183         try:
 184             Upload.pkg.changes_file = filename
 185             Upload.init_vars()
 186             Upload.update_vars()
 187             cache[filename] = copy.copy(Upload.pkg.changes)
 188             cache[filename]["filename"] = filename
 189         except:
 190             sorted_list.append(filename)
 191             break
 192     # Divide the .changes into per-source groups
 193     per_source = {}
 194     for filename in cache.keys():
 195         source = cache[filename]["source"]
 196         if not per_source.has_key(source):
 197             per_source[source] = {}
 198             per_source[source]["list"] = []
 199         per_source[source]["list"].append(cache[filename])
 200     # Determine oldest time and have note status for each source group
 201     for source in per_source.keys():
 202         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 203         ql = q.getresult()
 204         per_source[source]["source_in_database"] = len(ql)>0
 205         source_list = per_source[source]["list"]
 206         first = source_list[0]
 207         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 208         have_note = 0
 209         for d in per_source[source]["list"]:
 210             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 211             if mtime < oldest:
 212                 oldest = mtime
 213             have_note += (d.has_key("process-new note"))
 214         per_source[source]["oldest"] = oldest
 215         if not have_note:
 216             per_source[source]["note_state"] = 0; # none
 217         elif have_note < len(source_list):
 218             per_source[source]["note_state"] = 1; # some
 219         else:
 220             per_source[source]["note_state"] = 2; # all
 221         per_source[source]["list"].sort(indiv_sg_compare)
 222     per_source_items = per_source.items()
 223     per_source_items.sort(sg_compare)
 224     for i in per_source_items:
 225         for j in i[1]["list"]:
 226             sorted_list.append(j["filename"])
 227     return sorted_list
 228
 229 ################################################################################
 230
 231 class Section_Completer:
 232     def __init__ (self):
 233         self.sections = []
 234         self.matches = []
 235         q = projectB.query("SELECT section FROM section")
 236         for i in q.getresult():
 237             self.sections.append(i[0])
 238
 239     def complete(self, text, state):
 240         if state == 0:
 241             self.matches = []
 242             n = len(text)
 243             for word in self.sections:
 244                 if word[:n] == text:
 245                     self.matches.append(word)
 246         try:
 247             return self.matches[state]
 248         except IndexError:
 249             return None
 250
 251 ############################################################
 252
 253 class Priority_Completer:
 254     def __init__ (self):
 255         self.priorities = []
 256         self.matches = []
 257         q = projectB.query("SELECT priority FROM priority")
 258         for i in q.getresult():
 259             self.priorities.append(i[0])
 260
 261     def complete(self, text, state):
 262         if state == 0:
 263             self.matches = []
 264             n = len(text)
 265             for word in self.priorities:
 266                 if word[:n] == text:
 267                     self.matches.append(word)
 268         try:
 269             return self.matches[state]
 270         except IndexError:
 271             return None
 272
 273 ################################################################################
 274
 275 def print_new (new, indexed, file=sys.stdout):
 276     queue.check_valid(new)
 277     broken = 0
 278     index = 0
 279     for pkg in new.keys():
 280         index += 1
 281         section = new[pkg]["section"]
 282         priority = new[pkg]["priority"]
 283         if new[pkg]["section id"] == -1:
 284             section += "[!]"
 285             broken = 1
 286         if new[pkg]["priority id"] == -1:
 287             priority += "[!]"
 288             broken = 1
 289         if indexed:
 290             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 291         else:
 292             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 293         line = line.strip()+'\n'
 294         file.write(line)
 295     note = Upload.pkg.changes.get("process-new note")
 296     if note:
 297         print "*"*75
 298         print note
 299         print "*"*75
 300     return broken, note
 301
 302 ################################################################################
 303
 304 def index_range (index):
 305     if index == 1:
 306         return "1"
 307     else:
 308         return "1-%s" % (index)
 309
 310 ################################################################################
 311 ################################################################################
 312
 313 def edit_new (new):
 314     # Write the current data to a temporary file
 315     (fd, temp_filename) = utils.temp_filename()
 316     temp_file = os.fdopen(fd, 'w')
 317     print_new (new, 0, temp_file)
 318     temp_file.close()
 319     # Spawn an editor on that file
 320     editor = os.environ.get("EDITOR","vi")
 321     result = os.system("%s %s" % (editor, temp_filename))
 322     if result != 0:
 323         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 324     # Read the edited data back in
 325     temp_file = utils.open_file(temp_filename)
 326     lines = temp_file.readlines()
 327     temp_file.close()
 328     os.unlink(temp_filename)
 329     # Parse the new data
 330     for line in lines:
 331         line = line.strip()
 332         if line == "":
 333             continue
 334         s = line.split()
 335         # Pad the list if necessary
 336         s[len(s):3] = [None] * (3-len(s))
 337         (pkg, priority, section) = s[:3]
 338         if not new.has_key(pkg):
 339             utils.warn("Ignoring unknown package '%s'" % (pkg))
 340         else:
 341             # Strip off any invalid markers, print_new will readd them.
 342             if section.endswith("[!]"):
 343                 section = section[:-3]
 344             if priority.endswith("[!]"):
 345                 priority = priority[:-3]
 346             for f in new[pkg]["files"]:
 347                 Upload.pkg.files[f]["section"] = section
 348                 Upload.pkg.files[f]["priority"] = priority
 349             new[pkg]["section"] = section
 350             new[pkg]["priority"] = priority
 351
 352 ################################################################################
 353
 354 def edit_index (new, index):
 355     priority = new[index]["priority"]
 356     section = new[index]["section"]
 357     ftype = new[index]["type"]
 358     done = 0
 359     while not done:
 360         print "\t".join([index, priority, section])
 361
 362         answer = "XXX"
 363         if ftype != "dsc":
 364             prompt = "[B]oth, Priority, Section, Done ? "
 365         else:
 366             prompt = "[S]ection, Done ? "
 367         edit_priority = edit_section = 0
 368
 369         while prompt.find(answer) == -1:
 370             answer = utils.our_raw_input(prompt)
 371             m = re_default_answer.match(prompt)
 372             if answer == "":
 373                 answer = m.group(1)
 374             answer = answer[:1].upper()
 375
 376         if answer == 'P':
 377             edit_priority = 1
 378         elif answer == 'S':
 379             edit_section = 1
 380         elif answer == 'B':
 381             edit_priority = edit_section = 1
 382         elif answer == 'D':
 383             done = 1
 384
 385         # Edit the priority
 386         if edit_priority:
 387             readline.set_completer(Priorities.complete)
 388             got_priority = 0
 389             while not got_priority:
 390                 new_priority = utils.our_raw_input("New priority: ").strip()
 391                 if new_priority not in Priorities.priorities:
 392                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 393                 else:
 394                     got_priority = 1
 395                     priority = new_priority
 396
 397         # Edit the section
 398         if edit_section:
 399             readline.set_completer(Sections.complete)
 400             got_section = 0
 401             while not got_section:
 402                 new_section = utils.our_raw_input("New section: ").strip()
 403                 if new_section not in Sections.sections:
 404                     print "E: '%s' is not a valid section, try again." % (new_section)
 405                 else:
 406                     got_section = 1
 407                     section = new_section
 408
 409         # Reset the readline completer
 410         readline.set_completer(None)
 411
 412     for f in new[index]["files"]:
 413         Upload.pkg.files[f]["section"] = section
 414         Upload.pkg.files[f]["priority"] = priority
 415     new[index]["priority"] = priority
 416     new[index]["section"] = section
 417     return new
 418
 419 ################################################################################
 420
 421 def edit_overrides (new):
 422     print
 423     done = 0
 424     while not done:
 425         print_new (new, 1)
 426         new_index = {}
 427         index = 0
 428         for i in new.keys():
 429             index += 1
 430             new_index[index] = i
 431
 432         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 433
 434         got_answer = 0
 435         while not got_answer:
 436             answer = utils.our_raw_input(prompt)
 437             if not answer.isdigit():
 438                 answer = answer[:1].upper()
 439             if answer == "E" or answer == "D":
 440                 got_answer = 1
 441             elif re_isanum.match (answer):
 442                 answer = int(answer)
 443                 if (answer < 1) or (answer > index):
 444                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 445                 else:
 446                     got_answer = 1
 447
 448         if answer == 'E':
 449             edit_new(new)
 450         elif answer == 'D':
 451             done = 1
 452         else:
 453             edit_index (new, new_index[answer])
 454
 455     return new
 456
 457 ################################################################################
 458
 459 def edit_note(note):
 460     # Write the current data to a temporary file
 461     (fd, temp_filename) = utils.temp_filename()
 462     temp_file = os.fdopen(fd, 'w')
 463     temp_file.write(note)
 464     temp_file.close()
 465     editor = os.environ.get("EDITOR","vi")
 466     answer = 'E'
 467     while answer == 'E':
 468         os.system("%s %s" % (editor, temp_filename))
 469         temp_file = utils.open_file(temp_filename)
 470         note = temp_file.read().rstrip()
 471         temp_file.close()
 472         print "Note:"
 473         print utils.prefix_multi_line_string(note,"  ")
 474         prompt = "[D]one, Edit, Abandon, Quit ?"
 475         answer = "XXX"
 476         while prompt.find(answer) == -1:
 477             answer = utils.our_raw_input(prompt)
 478             m = re_default_answer.search(prompt)
 479             if answer == "":
 480                 answer = m.group(1)
 481             answer = answer[:1].upper()
 482     os.unlink(temp_filename)
 483     if answer == 'A':
 484         return
 485     elif answer == 'Q':
 486         end()
 487         sys.exit(0)
 488     Upload.pkg.changes["process-new note"] = note
 489     Upload.dump_vars(Cnf["Dir::Queue::New"])
 490
 491 ################################################################################
 492
 493 def check_pkg ():
 494     try:
 495         less_fd = os.popen("less -R -", 'w', 0)
 496         stdout_fd = sys.stdout
 497         try:
 498             sys.stdout = less_fd
 499             changes = utils.parse_changes (Upload.pkg.changes_file)
 500             examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file)
 501             files = Upload.pkg.files
 502             for f in files.keys():
 503                 if files[f].has_key("new"):
 504                     ftype = files[f]["type"]
 505                     if ftype == "deb":
 506                         examine_package.check_deb(changes['distribution'], f)
 507                     elif ftype == "dsc":
 508                         examine_package.check_dsc(changes['distribution'], f)
 509         finally:
 510             sys.stdout = stdout_fd
 511     except IOError, e:
 512         if e.errno == errno.EPIPE:
 513             utils.warn("[examine_package] Caught EPIPE; skipping.")
 514             pass
 515         else:
 516             raise
 517     except KeyboardInterrupt:
 518         utils.warn("[examine_package] Caught C-c; skipping.")
 519         pass
 520
 521 ################################################################################
 522
 523 ## FIXME: horribly Debian specific
 524
 525 def do_bxa_notification():
 526     files = Upload.pkg.files
 527     summary = ""
 528     for f in files.keys():
 529         if files[f]["type"] == "deb":
 530             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 531             summary += "\n"
 532             summary += "Package: %s\n" % (control.Find("Package"))
 533             summary += "Description: %s\n" % (control.Find("Description"))
 534     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 535     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 536     utils.send_mail(bxa_mail)
 537
 538 ################################################################################
 539
 540 def add_overrides (new):
 541     changes = Upload.pkg.changes
 542     files = Upload.pkg.files
 543
 544     projectB.query("BEGIN WORK")
 545     for suite in changes["suite"].keys():
 546         suite_id = database.get_suite_id(suite)
 547         for pkg in new.keys():
 548             component_id = database.get_component_id(new[pkg]["component"])
 549             type_id = database.get_override_type_id(new[pkg]["type"])
 550             priority_id = new[pkg]["priority id"]
 551             section_id = new[pkg]["section id"]
 552             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 553             for f in new[pkg]["files"]:
 554                 if files[f].has_key("new"):
 555                     del files[f]["new"]
 556             del new[pkg]
 557
 558     projectB.query("COMMIT WORK")
 559
 560     if Cnf.FindB("Dinstall::BXANotify"):
 561         do_bxa_notification()
 562
 563 ################################################################################
 564
 565 def prod_maintainer ():
 566     # Here we prepare an editor and get them ready to prod...
 567     (fd, temp_filename) = utils.temp_filename()
 568     editor = os.environ.get("EDITOR","vi")
 569     answer = 'E'
 570     while answer == 'E':
 571         os.system("%s %s" % (editor, temp_filename))
 572         f = os.fdopen(fd)
 573         prod_message = "".join(f.readlines())
 574         f.close()
 575         print "Prod message:"
 576         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 577         prompt = "[P]rod, Edit, Abandon, Quit ?"
 578         answer = "XXX"
 579         while prompt.find(answer) == -1:
 580             answer = utils.our_raw_input(prompt)
 581             m = re_default_answer.search(prompt)
 582             if answer == "":
 583                 answer = m.group(1)
 584             answer = answer[:1].upper()
 585         os.unlink(temp_filename)
 586         if answer == 'A':
 587             return
 588         elif answer == 'Q':
 589             end()
 590             sys.exit(0)
 591     # Otherwise, do the proding...
 592     user_email_address = utils.whoami() + " <%s>" % (
 593         Cnf["Dinstall::MyAdminAddress"])
 594
 595     Subst = Upload.Subst
 596
 597     Subst["__FROM_ADDRESS__"] = user_email_address
 598     Subst["__PROD_MESSAGE__"] = prod_message
 599     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 600
 601     prod_mail_message = utils.TemplateSubst(
 602         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 603
 604     # Send the prod mail if appropriate
 605     if not Cnf["Dinstall::Options::No-Mail"]:
 606         utils.send_mail(prod_mail_message)
 607
 608     print "Sent proding message"
 609
 610 ################################################################################
 611
 612 def do_new():
 613     print "NEW\n"
 614     files = Upload.pkg.files
 615     changes = Upload.pkg.changes
 616
 617     # Make a copy of distribution we can happily trample on
 618     changes["suite"] = copy.copy(changes["distribution"])
 619
 620     # Fix up the list of target suites
 621     for suite in changes["suite"].keys():
 622         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 623         if override:
 624             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 625               database.get_suite_id(override) == -1)
 626             if olderr or newerr:
 627                 (oinv, newinv) = ("", "")
 628                 if olderr: oinv = "invalid "
 629                 if newerr: ninv = "invalid "
 630                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 631                         oinv, suite, ninv, override)
 632             del changes["suite"][suite]
 633             changes["suite"][override] = 1
 634     # Validate suites
 635     for suite in changes["suite"].keys():
 636         suite_id = database.get_suite_id(suite)
 637         if suite_id == -1:
 638             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 639
 640     # The main NEW processing loop
 641     done = 0
 642     while not done:
 643         # Find out what's new
 644         new = queue.determine_new(changes, files, projectB)
 645
 646         if not new:
 647             break
 648
 649         answer = "XXX"
 650         if Options["No-Action"] or Options["Automatic"]:
 651             answer = 'S'
 652
 653         (broken, note) = print_new(new, 0)
 654         prompt = ""
 655
 656         if not broken and not note:
 657             prompt = "Add overrides, "
 658         if broken:
 659             print "W: [!] marked entries must be fixed before package can be processed."
 660         if note:
 661             print "W: note must be removed before package can be processed."
 662             prompt += "Remove note, "
 663
 664         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 665
 666         while prompt.find(answer) == -1:
 667             answer = utils.our_raw_input(prompt)
 668             m = re_default_answer.search(prompt)
 669             if answer == "":
 670                 answer = m.group(1)
 671             answer = answer[:1].upper()
 672
 673         if answer == 'A':
 674             done = add_overrides (new)
 675         elif answer == 'C':
 676             check_pkg()
 677         elif answer == 'E':
 678             new = edit_overrides (new)
 679         elif answer == 'M':
 680             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 681             if not aborted:
 682                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 683                 done = 1
 684         elif answer == 'N':
 685             edit_note(changes.get("process-new note", ""))
 686         elif answer == 'P':
 687             prod_maintainer()
 688         elif answer == 'R':
 689             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 690             if confirm == "y":
 691                 del changes["process-new note"]
 692         elif answer == 'S':
 693             done = 1
 694         elif answer == 'Q':
 695             end()
 696             sys.exit(0)
 697
 698 ################################################################################
 699 ################################################################################
 700 ################################################################################
 701
 702 def usage (exit_code=0):
 703     print """Usage: dak process-new [OPTION]... [CHANGES]...
 704   -a, --automatic           automatic run
 705   -h, --help                show this help and exit.
 706   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 707   -m, --manual-reject=MSG   manual reject with `msg'
 708   -n, --no-action           don't do anything
 709   -V, --version             display the version number and exit"""
 710     sys.exit(exit_code)
 711
 712 ################################################################################
 713
 714 def init():
 715     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 716
 717     Cnf = utils.get_conf()
 718
 719     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 720                  ('h',"help","Process-New::Options::Help"),
 721                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 722                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 723                  ('n',"no-action","Process-New::Options::No-Action")]
 724
 725     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 726         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 727             Cnf["Process-New::Options::%s" % (i)] = ""
 728
 729     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 730     Options = Cnf.SubTree("Process-New::Options")
 731
 732     if Options["Help"]:
 733         usage()
 734
 735     Upload = queue.Upload(Cnf)
 736
 737     if not Options["No-Action"]:
 738         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 739
 740     projectB = Upload.projectB
 741
 742     Sections = Section_Completer()
 743     Priorities = Priority_Completer()
 744     readline.parse_and_bind("tab: complete")
 745
 746     return changes_files
 747
 748 ################################################################################
 749
 750 def do_byhand():
 751     done = 0
 752     while not done:
 753         files = Upload.pkg.files
 754         will_install = 1
 755         byhand = []
 756
 757         for f in files.keys():
 758             if files[f]["type"] == "byhand":
 759                 if os.path.exists(f):
 760                     print "W: %s still present; please process byhand components and try again." % (f)
 761                     will_install = 0
 762                 else:
 763                     byhand.append(f)
 764
 765         answer = "XXXX"
 766         if Options["No-Action"]:
 767             answer = "S"
 768         if will_install:
 769             if Options["Automatic"] and not Options["No-Action"]:
 770                 answer = 'A'
 771             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 772         else:
 773             prompt = "Manual reject, [S]kip, Quit ?"
 774
 775         while prompt.find(answer) == -1:
 776             answer = utils.our_raw_input(prompt)
 777             m = re_default_answer.search(prompt)
 778             if answer == "":
 779                 answer = m.group(1)
 780             answer = answer[:1].upper()
 781
 782         if answer == 'A':
 783             done = 1
 784             for f in byhand:
 785                 del files[f]
 786         elif answer == 'M':
 787             Upload.do_reject(1, Options["Manual-Reject"])
 788             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 789             done = 1
 790         elif answer == 'S':
 791             done = 1
 792         elif answer == 'Q':
 793             end()
 794             sys.exit(0)
 795
 796 ################################################################################
 797
 798 def get_accept_lock():
 799     retry = 0
 800     while retry < 10:
 801         try:
 802             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 803             retry = 10
 804         except OSError, e:
 805             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 806                 retry += 1
 807                 if (retry >= 10):
 808                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 809                 else:
 810                     print("Unable to get accepted lock (try %d of 10)" % retry)
 811                 time.sleep(60)
 812             else:
 813                 raise
 814
 815 def move_to_dir (dest, perms=0660, changesperms=0664):
 816     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 817     file_keys = Upload.pkg.files.keys()
 818     for f in file_keys:
 819         utils.move (f, dest, perms=perms)
 820
 821 def is_source_in_queue_dir(qdir):
 822     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 823                 and x.endswith(".changes") ]
 824     for entry in entries:
 825         # read the .dak
 826         u = queue.Upload(Cnf)
 827         u.pkg.changes_file = os.path.join(qdir, entry)
 828         u.update_vars()
 829         if not u.pkg.changes["architecture"].has_key("source"):
 830             # another binary upload, ignore
 831             continue
 832         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 833             # another version, ignore
 834             continue
 835         # found it!
 836         return True
 837     return False
 838
 839 def move_to_holding(suite, queue_dir):
 840     print "Moving to %s holding area." % (suite.upper(),)
 841     if Options["No-Action"]:
 842         return
 843     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 844     Upload.dump_vars(queue_dir)
 845     move_to_dir(queue_dir)
 846     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 847
 848 def _accept():
 849     if Options["No-Action"]:
 850         return
 851     (summary, short_summary) = Upload.build_summaries()
 852     Upload.accept(summary, short_summary)
 853     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 854
 855 def do_accept_stableupdate(suite, q):
 856     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 857     if not Upload.pkg.changes["architecture"].has_key("source"):
 858         # It is not a sourceful upload.  So its source may be either in p-u
 859         # holding, in new, in accepted or already installed.
 860         if is_source_in_queue_dir(queue_dir):
 861             # It's in p-u holding, so move it there.
 862             print "Binary-only upload, source in %s." % (q,)
 863             move_to_holding(suite, queue_dir)
 864         elif Upload.source_exists(Upload.pkg.changes["source"],
 865                 Upload.pkg.changes["version"]):
 866             # dak tells us that there is source available.  At time of
 867             # writing this means that it is installed, so put it into
 868             # accepted.
 869             print "Binary-only upload, source installed."
 870             _accept()
 871         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 872             # The source is in accepted, the binary cleared NEW: accept it.
 873             print "Binary-only upload, source in accepted."
 874             _accept()
 875         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 876             # It's in NEW.  We expect the source to land in p-u holding
 877             # pretty soon.
 878             print "Binary-only upload, source in new."
 879             move_to_holding(suite, queue_dir)
 880         else:
 881             # No case applicable.  Bail out.  Return will cause the upload
 882             # to be skipped.
 883             print "ERROR"
 884             print "Stable update failed.  Source not found."
 885             return
 886     else:
 887         # We are handling a sourceful upload.  Move to accepted if currently
 888         # in p-u holding and to p-u holding otherwise.
 889         if is_source_in_queue_dir(queue_dir):
 890             print "Sourceful upload in %s, accepting." % (q,)
 891             _accept()
 892         else:
 893             move_to_holding(suite, queue_dir)
 894
 895 def do_accept():
 896     print "ACCEPT"
 897     if not Options["No-Action"]:
 898         get_accept_lock()
 899         (summary, short_summary) = Upload.build_summaries()
 900     try:
 901         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 902             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 903             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 904             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 905             # Check for override disparities
 906             Upload.Subst["__SUMMARY__"] = summary
 907         else:
 908             # Stable updates need to be copied to proposed-updates holding
 909             # area instead of accepted.  Sourceful uploads need to go
 910             # to it directly, binaries only if the source has not yet been
 911             # accepted into p-u.
 912             for suite, q in [("proposed-updates", "ProposedUpdates"),
 913                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 914                 if not Upload.pkg.changes["distribution"].has_key(suite):
 915                     continue
 916                 return do_accept_stableupdate(suite, q)
 917             # Just a normal upload, accept it...
 918             _accept()
 919     finally:
 920         if not Options["No-Action"]:
 921             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 922
 923 def check_status(files):
 924     new = byhand = 0
 925     for f in files.keys():
 926         if files[f]["type"] == "byhand":
 927             byhand = 1
 928         elif files[f].has_key("new"):
 929             new = 1
 930     return (new, byhand)
 931
 932 def do_pkg(changes_file):
 933     Upload.pkg.changes_file = changes_file
 934     Upload.init_vars()
 935     Upload.update_vars()
 936     Upload.update_subst()
 937     files = Upload.pkg.files
 938
 939     if not recheck():
 940         return
 941
 942     (new, byhand) = check_status(files)
 943     if new or byhand:
 944         if new:
 945             do_new()
 946         if byhand:
 947             do_byhand()
 948         (new, byhand) = check_status(files)
 949
 950     if not new and not byhand:
 951         do_accept()
 952
 953 ################################################################################
 954
 955 def end():
 956     accept_count = Upload.accept_count
 957     accept_bytes = Upload.accept_bytes
 958
 959     if accept_count:
 960         sets = "set"
 961         if accept_count > 1:
 962             sets = "sets"
 963         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 964         Logger.log(["total",accept_count,accept_bytes])
 965
 966     if not Options["No-Action"]:
 967         Logger.close()
 968
 969 ################################################################################
 970
 971 def do_comments(dir, opref, npref, line, fn):
 972     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 973         lines = open("%s/%s" % (dir, comm)).readlines()
 974         if len(lines) == 0 or lines[0] != line + "\n": continue
 975         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 976                                 and x.endswith(".changes") ]
 977         changes_files = sort_changes(changes_files)
 978         for f in changes_files:
 979             f = utils.validate_changes_file_arg(f, 0)
 980             if not f: continue
 981             print "\n" + f
 982             fn(f, "".join(lines[1:]))
 983
 984         if opref != npref and not Options["No-Action"]:
 985             newcomm = npref + comm[len(opref):]
 986             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 987
 988 ################################################################################
 989
 990 def comment_accept(changes_file, comments):
 991     Upload.pkg.changes_file = changes_file
 992     Upload.init_vars()
 993     Upload.update_vars()
 994     Upload.update_subst()
 995     files = Upload.pkg.files
 996
 997     if not recheck():
 998         return # dak wants to REJECT, crap
 999
1000     (new, byhand) = check_status(files)
1001     if not new and not byhand:
1002         do_accept()
1003
1004 ################################################################################
1005
1006 def comment_reject(changes_file, comments):
1007     Upload.pkg.changes_file = changes_file
1008     Upload.init_vars()
1009     Upload.update_vars()
1010     Upload.update_subst()
1011
1012     if not recheck():
1013         pass # dak has its own reasons to reject as well, which is fine
1014
1015     reject(comments)
1016     print "REJECT\n" + reject_message,
1017     if not Options["No-Action"]:
1018         Upload.do_reject(0, reject_message)
1019         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1020
1021 ################################################################################
1022
1023 def main():
1024     changes_files = init()
1025     if len(changes_files) > 50:
1026         sys.stderr.write("Sorting changes...\n")
1027     changes_files = sort_changes(changes_files)
1028
1029     # Kill me now? **FIXME**
1030     Cnf["Dinstall::Options::No-Mail"] = ""
1031     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1032     if Cnf.has_key("Dinstall::Bcc"):
1033         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1034     else:
1035         Upload.Subst["__BCC__"] = bcc
1036
1037     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1038     if commentsdir:
1039         if changes_files != []:
1040             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1041             sys.exit(1)
1042         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1043         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1044     else:
1045         for changes_file in changes_files:
1046             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1047             if not changes_file:
1048                 continue
1049             print "\n" + changes_file
1050             do_pkg (changes_file)
1051
1052     end()
1053
1054 ################################################################################
1055
1056 if __name__ == '__main__':
1057     main()