dak/process_new.py

   1 #!/usr/bin/env python
   2 # vim:set et ts=4 sw=4:
   3
   4 # Handles NEW and BYHAND packages
   5 # Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 # 23:12|<aj> I will not hush!
  24 # 23:12|<elmo> :>
  25 # 23:12|<aj> Where there is injustice in the world, I shall be there!
  26 # 23:13|<aj> I shall not be silenced!
  27 # 23:13|<aj> The world shall know!
  28 # 23:13|<aj> The world *must* know!
  29 # 23:13|<elmo> oh dear, he's gone back to powerpuff girls... ;-)
  30 # 23:13|<aj> yay powerpuff girls!!
  31 # 23:13|<aj> buttercup's my favourite, who's yours?
  32 # 23:14|<aj> you're backing away from the keyboard right now aren't you?
  33 # 23:14|<aj> *AREN'T YOU*?!
  34 # 23:15|<aj> I will not be treated like this.
  35 # 23:15|<aj> I shall have my revenge.
  36 # 23:15|<aj> I SHALL!!!
  37
  38 ################################################################################
  39
  40 import copy, errno, os, readline, stat, sys, time
  41 import apt_pkg, apt_inst
  42 import examine_package
  43 from daklib import database
  44 from daklib import logging
  45 from daklib import queue
  46 from daklib import utils
  47
  48 # Globals
  49 Cnf = None
  50 Options = None
  51 Upload = None
  52 projectB = None
  53 Logger = None
  54
  55 Priorities = None
  56 Sections = None
  57
  58 reject_message = ""
  59
  60 ################################################################################
  61 ################################################################################
  62 ################################################################################
  63
  64 def reject (str, prefix="Rejected: "):
  65     global reject_message
  66     if str:
  67         reject_message += prefix + str + "\n"
  68
  69 def recheck():
  70     global reject_message
  71     files = Upload.pkg.files
  72     reject_message = ""
  73
  74     for f in files.keys():
  75         # The .orig.tar.gz can disappear out from under us is it's a
  76         # duplicate of one in the archive.
  77         if not files.has_key(f):
  78             continue
  79         # Check that the source still exists
  80         if files[f]["type"] == "deb":
  81             source_version = files[f]["source version"]
  82             source_package = files[f]["source package"]
  83             if not Upload.pkg.changes["architecture"].has_key("source") \
  84                and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
  85                 source_epochless_version = utils.re_no_epoch.sub('', source_version)
  86                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
  87                 found = 0
  88                 for q in ["Accepted", "Embargoed", "Unembargoed"]:
  89                     if Cnf.has_key("Dir::Queue::%s" % (q)):
  90                         if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
  91                             found = 1
  92                 if not found:
  93                     reject("no source found for %s %s (%s)." % (source_package, source_version, f))
  94
  95         # Version and file overwrite checks
  96         if files[f]["type"] == "deb":
  97             reject(Upload.check_binary_against_db(f), "")
  98         elif files[f]["type"] == "dsc":
  99             reject(Upload.check_source_against_db(f), "")
 100             (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f)
 101             reject(reject_msg, "")
 102
 103     if reject_message.find("Rejected") != -1:
 104         answer = "XXX"
 105         if Options["No-Action"] or Options["Automatic"]:
 106             answer = 'S'
 107
 108         print "REJECT\n" + reject_message,
 109         prompt = "[R]eject, Skip, Quit ?"
 110
 111         while prompt.find(answer) == -1:
 112             answer = utils.our_raw_input(prompt)
 113             m = queue.re_default_answer.match(prompt)
 114             if answer == "":
 115                 answer = m.group(1)
 116             answer = answer[:1].upper()
 117
 118         if answer == 'R':
 119             Upload.do_reject(0, reject_message)
 120             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 121             return 0
 122         elif answer == 'S':
 123             return 0
 124         elif answer == 'Q':
 125             end()
 126             sys.exit(0)
 127
 128     return 1
 129
 130 ################################################################################
 131
 132 def indiv_sg_compare (a, b):
 133     """Sort by source name, source, version, 'have source', and
 134        finally by filename."""
 135     # Sort by source version
 136     q = apt_pkg.VersionCompare(a["version"], b["version"])
 137     if q:
 138         return -q
 139
 140     # Sort by 'have source'
 141     a_has_source = a["architecture"].get("source")
 142     b_has_source = b["architecture"].get("source")
 143     if a_has_source and not b_has_source:
 144         return -1
 145     elif b_has_source and not a_has_source:
 146         return 1
 147
 148     return cmp(a["filename"], b["filename"])
 149
 150 ############################################################
 151
 152 def sg_compare (a, b):
 153     a = a[1]
 154     b = b[1]
 155     """Sort by have note, source already in database and time of oldest upload."""
 156     # Sort by have note
 157     a_note_state = a["note_state"]
 158     b_note_state = b["note_state"]
 159     if a_note_state < b_note_state:
 160         return -1
 161     elif a_note_state > b_note_state:
 162         return 1
 163     # Sort by source already in database (descending)
 164     source_in_database = cmp(a["source_in_database"], b["source_in_database"])
 165     if source_in_database:
 166         return -source_in_database
 167
 168     # Sort by time of oldest upload
 169     return cmp(a["oldest"], b["oldest"])
 170
 171 def sort_changes(changes_files):
 172     """Sort into source groups, then sort each source group by version,
 173     have source, filename.  Finally, sort the source groups by have
 174     note, time of oldest upload of each source upload."""
 175     if len(changes_files) == 1:
 176         return changes_files
 177
 178     sorted_list = []
 179     cache = {}
 180     # Read in all the .changes files
 181     for filename in changes_files:
 182         try:
 183             Upload.pkg.changes_file = filename
 184             Upload.init_vars()
 185             Upload.update_vars()
 186             cache[filename] = copy.copy(Upload.pkg.changes)
 187             cache[filename]["filename"] = filename
 188         except:
 189             sorted_list.append(filename)
 190             break
 191     # Divide the .changes into per-source groups
 192     per_source = {}
 193     for filename in cache.keys():
 194         source = cache[filename]["source"]
 195         if not per_source.has_key(source):
 196             per_source[source] = {}
 197             per_source[source]["list"] = []
 198         per_source[source]["list"].append(cache[filename])
 199     # Determine oldest time and have note status for each source group
 200     for source in per_source.keys():
 201         q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source)
 202         ql = q.getresult()
 203         per_source[source]["source_in_database"] = len(ql)>0
 204         source_list = per_source[source]["list"]
 205         first = source_list[0]
 206         oldest = os.stat(first["filename"])[stat.ST_MTIME]
 207         have_note = 0
 208         for d in per_source[source]["list"]:
 209             mtime = os.stat(d["filename"])[stat.ST_MTIME]
 210             if mtime < oldest:
 211                 oldest = mtime
 212             have_note += (d.has_key("process-new note"))
 213         per_source[source]["oldest"] = oldest
 214         if not have_note:
 215             per_source[source]["note_state"] = 0; # none
 216         elif have_note < len(source_list):
 217             per_source[source]["note_state"] = 1; # some
 218         else:
 219             per_source[source]["note_state"] = 2; # all
 220         per_source[source]["list"].sort(indiv_sg_compare)
 221     per_source_items = per_source.items()
 222     per_source_items.sort(sg_compare)
 223     for i in per_source_items:
 224         for j in i[1]["list"]:
 225             sorted_list.append(j["filename"])
 226     return sorted_list
 227
 228 ################################################################################
 229
 230 class Section_Completer:
 231     def __init__ (self):
 232         self.sections = []
 233         q = projectB.query("SELECT section FROM section")
 234         for i in q.getresult():
 235             self.sections.append(i[0])
 236
 237     def complete(self, text, state):
 238         if state == 0:
 239             self.matches = []
 240             n = len(text)
 241             for word in self.sections:
 242                 if word[:n] == text:
 243                     self.matches.append(word)
 244         try:
 245             return self.matches[state]
 246         except IndexError:
 247             return None
 248
 249 ############################################################
 250
 251 class Priority_Completer:
 252     def __init__ (self):
 253         self.priorities = []
 254         q = projectB.query("SELECT priority FROM priority")
 255         for i in q.getresult():
 256             self.priorities.append(i[0])
 257
 258     def complete(self, text, state):
 259         if state == 0:
 260             self.matches = []
 261             n = len(text)
 262             for word in self.priorities:
 263                 if word[:n] == text:
 264                     self.matches.append(word)
 265         try:
 266             return self.matches[state]
 267         except IndexError:
 268             return None
 269
 270 ################################################################################
 271
 272 def print_new (new, indexed, file=sys.stdout):
 273     queue.check_valid(new)
 274     broken = 0
 275     index = 0
 276     for pkg in new.keys():
 277         index += 1
 278         section = new[pkg]["section"]
 279         priority = new[pkg]["priority"]
 280         if new[pkg]["section id"] == -1:
 281             section += "[!]"
 282             broken = 1
 283         if new[pkg]["priority id"] == -1:
 284             priority += "[!]"
 285             broken = 1
 286         if indexed:
 287             line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section)
 288         else:
 289             line = "%-20s %-20s %-20s" % (pkg, priority, section)
 290         line = line.strip()+'\n'
 291         file.write(line)
 292     note = Upload.pkg.changes.get("process-new note")
 293     if note:
 294         print "*"*75
 295         print note
 296         print "*"*75
 297     return broken, note
 298
 299 ################################################################################
 300
 301 def index_range (index):
 302     if index == 1:
 303         return "1"
 304     else:
 305         return "1-%s" % (index)
 306
 307 ################################################################################
 308 ################################################################################
 309
 310 def edit_new (new):
 311     # Write the current data to a temporary file
 312     temp_filename = utils.temp_filename()
 313     temp_file = utils.open_file(temp_filename, 'w')
 314     print_new (new, 0, temp_file)
 315     temp_file.close()
 316     # Spawn an editor on that file
 317     editor = os.environ.get("EDITOR","vi")
 318     result = os.system("%s %s" % (editor, temp_filename))
 319     if result != 0:
 320         utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result)
 321     # Read the edited data back in
 322     temp_file = utils.open_file(temp_filename)
 323     lines = temp_file.readlines()
 324     temp_file.close()
 325     os.unlink(temp_filename)
 326     # Parse the new data
 327     for line in lines:
 328         line = line.strip()
 329         if line == "":
 330             continue
 331         s = line.split()
 332         # Pad the list if necessary
 333         s[len(s):3] = [None] * (3-len(s))
 334         (pkg, priority, section) = s[:3]
 335         if not new.has_key(pkg):
 336             utils.warn("Ignoring unknown package '%s'" % (pkg))
 337         else:
 338             # Strip off any invalid markers, print_new will readd them.
 339             if section.endswith("[!]"):
 340                 section = section[:-3]
 341             if priority.endswith("[!]"):
 342                 priority = priority[:-3]
 343             for f in new[pkg]["files"]:
 344                 Upload.pkg.files[f]["section"] = section
 345                 Upload.pkg.files[f]["priority"] = priority
 346             new[pkg]["section"] = section
 347             new[pkg]["priority"] = priority
 348
 349 ################################################################################
 350
 351 def edit_index (new, index):
 352     priority = new[index]["priority"]
 353     section = new[index]["section"]
 354     ftype = new[index]["type"]
 355     done = 0
 356     while not done:
 357         print "\t".join([index, priority, section])
 358
 359         answer = "XXX"
 360         if ftype != "dsc":
 361             prompt = "[B]oth, Priority, Section, Done ? "
 362         else:
 363             prompt = "[S]ection, Done ? "
 364         edit_priority = edit_section = 0
 365
 366         while prompt.find(answer) == -1:
 367             answer = utils.our_raw_input(prompt)
 368             m = queue.re_default_answer.match(prompt)
 369             if answer == "":
 370                 answer = m.group(1)
 371             answer = answer[:1].upper()
 372
 373         if answer == 'P':
 374             edit_priority = 1
 375         elif answer == 'S':
 376             edit_section = 1
 377         elif answer == 'B':
 378             edit_priority = edit_section = 1
 379         elif answer == 'D':
 380             done = 1
 381
 382         # Edit the priority
 383         if edit_priority:
 384             readline.set_completer(Priorities.complete)
 385             got_priority = 0
 386             while not got_priority:
 387                 new_priority = utils.our_raw_input("New priority: ").strip()
 388                 if new_priority not in Priorities.priorities:
 389                     print "E: '%s' is not a valid priority, try again." % (new_priority)
 390                 else:
 391                     got_priority = 1
 392                     priority = new_priority
 393
 394         # Edit the section
 395         if edit_section:
 396             readline.set_completer(Sections.complete)
 397             got_section = 0
 398             while not got_section:
 399                 new_section = utils.our_raw_input("New section: ").strip()
 400                 if new_section not in Sections.sections:
 401                     print "E: '%s' is not a valid section, try again." % (new_section)
 402                 else:
 403                     got_section = 1
 404                     section = new_section
 405
 406         # Reset the readline completer
 407         readline.set_completer(None)
 408
 409     for f in new[index]["files"]:
 410         Upload.pkg.files[f]["section"] = section
 411         Upload.pkg.files[f]["priority"] = priority
 412     new[index]["priority"] = priority
 413     new[index]["section"] = section
 414     return new
 415
 416 ################################################################################
 417
 418 def edit_overrides (new):
 419     print
 420     done = 0
 421     while not done:
 422         print_new (new, 1)
 423         new_index = {}
 424         index = 0
 425         for i in new.keys():
 426             index += 1
 427             new_index[index] = i
 428
 429         prompt = "(%s) edit override <n>, Editor, Done ? " % (index_range(index))
 430
 431         got_answer = 0
 432         while not got_answer:
 433             answer = utils.our_raw_input(prompt)
 434             if not answer.isdigit():
 435                 answer = answer[:1].upper()
 436             if answer == "E" or answer == "D":
 437                 got_answer = 1
 438             elif queue.re_isanum.match (answer):
 439                 answer = int(answer)
 440                 if (answer < 1) or (answer > index):
 441                     print "%s is not a valid index (%s).  Please retry." % (answer, index_range(index))
 442                 else:
 443                     got_answer = 1
 444
 445         if answer == 'E':
 446             edit_new(new)
 447         elif answer == 'D':
 448             done = 1
 449         else:
 450             edit_index (new, new_index[answer])
 451
 452     return new
 453
 454 ################################################################################
 455
 456 def edit_note(note):
 457     # Write the current data to a temporary file
 458     temp_filename = utils.temp_filename()
 459     temp_file = utils.open_file(temp_filename, 'w')
 460     temp_file.write(note)
 461     temp_file.close()
 462     editor = os.environ.get("EDITOR","vi")
 463     answer = 'E'
 464     while answer == 'E':
 465         os.system("%s %s" % (editor, temp_filename))
 466         temp_file = utils.open_file(temp_filename)
 467         note = temp_file.read().rstrip()
 468         temp_file.close()
 469         print "Note:"
 470         print utils.prefix_multi_line_string(note,"  ")
 471         prompt = "[D]one, Edit, Abandon, Quit ?"
 472         answer = "XXX"
 473         while prompt.find(answer) == -1:
 474             answer = utils.our_raw_input(prompt)
 475             m = queue.re_default_answer.search(prompt)
 476             if answer == "":
 477                 answer = m.group(1)
 478             answer = answer[:1].upper()
 479     os.unlink(temp_filename)
 480     if answer == 'A':
 481         return
 482     elif answer == 'Q':
 483         end()
 484         sys.exit(0)
 485     Upload.pkg.changes["process-new note"] = note
 486     Upload.dump_vars(Cnf["Dir::Queue::New"])
 487
 488 ################################################################################
 489
 490 def check_pkg ():
 491     try:
 492         less_fd = os.popen("less -R -", 'w', 0)
 493         stdout_fd = sys.stdout
 494         try:
 495             sys.stdout = less_fd
 496             examine_package.display_changes(Upload.pkg.changes_file)
 497             files = Upload.pkg.files
 498             for f in files.keys():
 499                 if files[f].has_key("new"):
 500                     ftype = files[f]["type"]
 501                     if ftype == "deb":
 502                         examine_package.check_deb(f)
 503                     elif ftype == "dsc":
 504                         examine_package.check_dsc(f)
 505         finally:
 506             sys.stdout = stdout_fd
 507     except IOError, e:
 508         if e.errno == errno.EPIPE:
 509             utils.warn("[examine_package] Caught EPIPE; skipping.")
 510             pass
 511         else:
 512             raise
 513     except KeyboardInterrupt:
 514         utils.warn("[examine_package] Caught C-c; skipping.")
 515         pass
 516
 517 ################################################################################
 518
 519 ## FIXME: horribly Debian specific
 520
 521 def do_bxa_notification():
 522     files = Upload.pkg.files
 523     summary = ""
 524     for f in files.keys():
 525         if files[f]["type"] == "deb":
 526             control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f)))
 527             summary += "\n"
 528             summary += "Package: %s\n" % (control.Find("Package"))
 529             summary += "Description: %s\n" % (control.Find("Description"))
 530     Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary
 531     bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification")
 532     utils.send_mail(bxa_mail)
 533
 534 ################################################################################
 535
 536 def add_overrides (new):
 537     changes = Upload.pkg.changes
 538     files = Upload.pkg.files
 539
 540     projectB.query("BEGIN WORK")
 541     for suite in changes["suite"].keys():
 542         suite_id = database.get_suite_id(suite)
 543         for pkg in new.keys():
 544             component_id = database.get_component_id(new[pkg]["component"])
 545             type_id = database.get_override_type_id(new[pkg]["type"])
 546             priority_id = new[pkg]["priority id"]
 547             section_id = new[pkg]["section id"]
 548             projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id))
 549             for f in new[pkg]["files"]:
 550                 if files[f].has_key("new"):
 551                     del files[f]["new"]
 552             del new[pkg]
 553
 554     projectB.query("COMMIT WORK")
 555
 556     if Cnf.FindB("Dinstall::BXANotify"):
 557         do_bxa_notification()
 558
 559 ################################################################################
 560
 561 def prod_maintainer ():
 562     # Here we prepare an editor and get them ready to prod...
 563     temp_filename = utils.temp_filename()
 564     editor = os.environ.get("EDITOR","vi")
 565     answer = 'E'
 566     while answer == 'E':
 567         os.system("%s %s" % (editor, temp_filename))
 568         f = utils.open_file(temp_filename)
 569         prod_message = "".join(f.readlines())
 570         f.close()
 571         print "Prod message:"
 572         print utils.prefix_multi_line_string(prod_message,"  ",include_blank_lines=1)
 573         prompt = "[P]rod, Edit, Abandon, Quit ?"
 574         answer = "XXX"
 575         while prompt.find(answer) == -1:
 576             answer = utils.our_raw_input(prompt)
 577             m = queue.re_default_answer.search(prompt)
 578             if answer == "":
 579                 answer = m.group(1)
 580             answer = answer[:1].upper()
 581         os.unlink(temp_filename)
 582         if answer == 'A':
 583             return
 584         elif answer == 'Q':
 585             end()
 586             sys.exit(0)
 587     # Otherwise, do the proding...
 588     user_email_address = utils.whoami() + " <%s>" % (
 589         Cnf["Dinstall::MyAdminAddress"])
 590
 591     Subst = Upload.Subst
 592
 593     Subst["__FROM_ADDRESS__"] = user_email_address
 594     Subst["__PROD_MESSAGE__"] = prod_message
 595     Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]
 596
 597     prod_mail_message = utils.TemplateSubst(
 598         Subst,Cnf["Dir::Templates"]+"/process-new.prod")
 599
 600     # Send the prod mail if appropriate
 601     if not Cnf["Dinstall::Options::No-Mail"]:
 602         utils.send_mail(prod_mail_message)
 603
 604     print "Sent proding message"
 605
 606 ################################################################################
 607
 608 def do_new():
 609     print "NEW\n"
 610     files = Upload.pkg.files
 611     changes = Upload.pkg.changes
 612
 613     # Make a copy of distribution we can happily trample on
 614     changes["suite"] = copy.copy(changes["distribution"])
 615
 616     # Fix up the list of target suites
 617     for suite in changes["suite"].keys():
 618         override = Cnf.Find("Suite::%s::OverrideSuite" % (suite))
 619         if override:
 620             (olderr, newerr) = (database.get_suite_id(suite) == -1,
 621               database.get_suite_id(override) == -1)
 622             if olderr or newerr:
 623                 (oinv, newinv) = ("", "")
 624                 if olderr: oinv = "invalid "
 625                 if newerr: ninv = "invalid "
 626                 print "warning: overriding %ssuite %s to %ssuite %s" % (
 627                         oinv, suite, ninv, override)
 628             del changes["suite"][suite]
 629             changes["suite"][override] = 1
 630     # Validate suites
 631     for suite in changes["suite"].keys():
 632         suite_id = database.get_suite_id(suite)
 633         if suite_id == -1:
 634             utils.fubar("%s has invalid suite '%s' (possibly overriden).  say wha?" % (changes, suite))
 635
 636     # The main NEW processing loop
 637     done = 0
 638     while not done:
 639         # Find out what's new
 640         new = queue.determine_new(changes, files, projectB)
 641
 642         if not new:
 643             break
 644
 645         answer = "XXX"
 646         if Options["No-Action"] or Options["Automatic"]:
 647             answer = 'S'
 648
 649         (broken, note) = print_new(new, 0)
 650         prompt = ""
 651
 652         if not broken and not note:
 653             prompt = "Add overrides, "
 654         if broken:
 655             print "W: [!] marked entries must be fixed before package can be processed."
 656         if note:
 657             print "W: note must be removed before package can be processed."
 658             prompt += "Remove note, "
 659
 660         prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"
 661
 662         while prompt.find(answer) == -1:
 663             answer = utils.our_raw_input(prompt)
 664             m = queue.re_default_answer.search(prompt)
 665             if answer == "":
 666                 answer = m.group(1)
 667             answer = answer[:1].upper()
 668
 669         if answer == 'A':
 670             done = add_overrides (new)
 671         elif answer == 'C':
 672             check_pkg()
 673         elif answer == 'E':
 674             new = edit_overrides (new)
 675         elif answer == 'M':
 676             aborted = Upload.do_reject(1, Options["Manual-Reject"])
 677             if not aborted:
 678                 os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 679                 done = 1
 680         elif answer == 'N':
 681             edit_note(changes.get("process-new note", ""))
 682         elif answer == 'P':
 683             prod_maintainer()
 684         elif answer == 'R':
 685             confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
 686             if confirm == "y":
 687                 del changes["process-new note"]
 688         elif answer == 'S':
 689             done = 1
 690         elif answer == 'Q':
 691             end()
 692             sys.exit(0)
 693
 694 ################################################################################
 695 ################################################################################
 696 ################################################################################
 697
 698 def usage (exit_code=0):
 699     print """Usage: dak process-new [OPTION]... [CHANGES]...
 700   -a, --automatic           automatic run
 701   -h, --help                show this help and exit.
 702   -C, --comments-dir=DIR    use DIR as comments-dir, for [o-]p-u-new
 703   -m, --manual-reject=MSG   manual reject with `msg'
 704   -n, --no-action           don't do anything
 705   -V, --version             display the version number and exit"""
 706     sys.exit(exit_code)
 707
 708 ################################################################################
 709
 710 def init():
 711     global Cnf, Options, Logger, Upload, projectB, Sections, Priorities
 712
 713     Cnf = utils.get_conf()
 714
 715     Arguments = [('a',"automatic","Process-New::Options::Automatic"),
 716                  ('h',"help","Process-New::Options::Help"),
 717                  ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
 718                  ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
 719                  ('n',"no-action","Process-New::Options::No-Action")]
 720
 721     for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
 722         if not Cnf.has_key("Process-New::Options::%s" % (i)):
 723             Cnf["Process-New::Options::%s" % (i)] = ""
 724
 725     changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
 726     Options = Cnf.SubTree("Process-New::Options")
 727
 728     if Options["Help"]:
 729         usage()
 730
 731     Upload = queue.Upload(Cnf)
 732
 733     if not Options["No-Action"]:
 734         Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
 735
 736     projectB = Upload.projectB
 737
 738     Sections = Section_Completer()
 739     Priorities = Priority_Completer()
 740     readline.parse_and_bind("tab: complete")
 741
 742     return changes_files
 743
 744 ################################################################################
 745
 746 def do_byhand():
 747     done = 0
 748     while not done:
 749         files = Upload.pkg.files
 750         will_install = 1
 751         byhand = []
 752
 753         for f in files.keys():
 754             if files[f]["type"] == "byhand":
 755                 if os.path.exists(f):
 756                     print "W: %s still present; please process byhand components and try again." % (f)
 757                     will_install = 0
 758                 else:
 759                     byhand.append(f)
 760
 761         answer = "XXXX"
 762         if Options["No-Action"]:
 763             answer = "S"
 764         if will_install:
 765             if Options["Automatic"] and not Options["No-Action"]:
 766                 answer = 'A'
 767             prompt = "[A]ccept, Manual reject, Skip, Quit ?"
 768         else:
 769             prompt = "Manual reject, [S]kip, Quit ?"
 770
 771         while prompt.find(answer) == -1:
 772             answer = utils.our_raw_input(prompt)
 773             m = queue.re_default_answer.search(prompt)
 774             if answer == "":
 775                 answer = m.group(1)
 776             answer = answer[:1].upper()
 777
 778         if answer == 'A':
 779             done = 1
 780             for f in byhand:
 781                 del files[f]
 782         elif answer == 'M':
 783             Upload.do_reject(1, Options["Manual-Reject"])
 784             os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 785             done = 1
 786         elif answer == 'S':
 787             done = 1
 788         elif answer == 'Q':
 789             end()
 790             sys.exit(0)
 791
 792 ################################################################################
 793
 794 def get_accept_lock():
 795     retry = 0
 796     while retry < 10:
 797         try:
 798             os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
 799             retry = 10
 800         except OSError, e:
 801             if e.errno == errno.EACCES or e.errno == errno.EEXIST:
 802                 retry += 1
 803                 if (retry >= 10):
 804                     utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
 805                 else:
 806                     print("Unable to get accepted lock (try %d of 10)" % retry)
 807                 time.sleep(60)
 808             else:
 809                 raise
 810
 811 def move_to_dir (dest, perms=0660, changesperms=0664):
 812     utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
 813     file_keys = Upload.pkg.files.keys()
 814     for f in file_keys:
 815         utils.move (f, dest, perms=perms)
 816
 817 def is_source_in_queue_dir(qdir):
 818     entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"])
 819                 and x.endswith(".changes") ]
 820     for entry in entries:
 821         # read the .dak
 822         u = queue.Upload(Cnf)
 823         u.pkg.changes_file = os.path.join(qdir, entry)
 824         u.update_vars()
 825         if not u.pkg.changes["architecture"].has_key("source"):
 826             # another binary upload, ignore
 827             continue
 828         if Upload.pkg.changes["version"] != u.pkg.changes["version"]:
 829             # another version, ignore
 830             continue
 831         # found it!
 832         return True
 833     return False
 834
 835 def move_to_holding(suite, queue_dir):
 836     print "Moving to %s holding area." % (suite.upper(),)
 837     if Options["No-Action"]:
 838         return
 839     Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file])
 840     Upload.dump_vars(queue_dir)
 841     move_to_dir(queue_dir)
 842     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 843
 844 def _accept():
 845     if Options["No-Action"]:
 846         return
 847     (summary, short_summary) = Upload.build_summaries()
 848     Upload.accept(summary, short_summary)
 849     os.unlink(Upload.pkg.changes_file[:-8]+".dak")
 850
 851 def do_accept_stableupdate(suite, q):
 852     queue_dir = Cnf["Dir::Queue::%s" % (q,)]
 853     if not Upload.pkg.changes["architecture"].has_key("source"):
 854         # It is not a sourceful upload.  So its source may be either in p-u
 855         # holding, in new, in accepted or already installed.
 856         if is_source_in_queue_dir(queue_dir):
 857             # It's in p-u holding, so move it there.
 858             print "Binary-only upload, source in %s." % (q,)
 859             move_to_holding(suite, queue_dir)
 860         elif Upload.source_exists(Upload.pkg.changes["source"],
 861                 Upload.pkg.changes["version"]):
 862             # dak tells us that there is source available.  At time of
 863             # writing this means that it is installed, so put it into
 864             # accepted.
 865             print "Binary-only upload, source installed."
 866             _accept()
 867         elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]):
 868             # The source is in accepted, the binary cleared NEW: accept it.
 869             print "Binary-only upload, source in accepted."
 870             _accept()
 871         elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]):
 872             # It's in NEW.  We expect the source to land in p-u holding
 873             # pretty soon.
 874             print "Binary-only upload, source in new."
 875             move_to_holding(suite, queue_dir)
 876         else:
 877             # No case applicable.  Bail out.  Return will cause the upload
 878             # to be skipped.
 879             print "ERROR"
 880             print "Stable update failed.  Source not found."
 881             return
 882     else:
 883         # We are handling a sourceful upload.  Move to accepted if currently
 884         # in p-u holding and to p-u holding otherwise.
 885         if is_source_in_queue_dir(queue_dir):
 886             print "Sourceful upload in %s, accepting." % (q,)
 887             _accept()
 888         else:
 889             move_to_holding(suite, queue_dir)
 890
 891 def do_accept():
 892     print "ACCEPT"
 893     if not Options["No-Action"]:
 894         get_accept_lock()
 895         (summary, short_summary) = Upload.build_summaries()
 896     try:
 897         if Cnf.FindB("Dinstall::SecurityQueueHandling"):
 898             Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
 899             move_to_dir(Cnf["Dir::Queue::Embargoed"])
 900             Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
 901             # Check for override disparities
 902             Upload.Subst["__SUMMARY__"] = summary
 903         else:
 904             # Stable updates need to be copied to proposed-updates holding
 905             # area instead of accepted.  Sourceful uploads need to go
 906             # to it directly, binaries only if the source has not yet been
 907             # accepted into p-u.
 908             for suite, q in [("proposed-updates", "ProposedUpdates"),
 909                     ("oldstable-proposed-updates", "OldProposedUpdates")]:
 910                 if not Upload.pkg.changes["distribution"].has_key(suite):
 911                     continue
 912                 return do_accept_stableupdate(suite, q)
 913             # Just a normal upload, accept it...
 914             _accept()
 915     finally:
 916         if not Options["No-Action"]:
 917             os.unlink(Cnf["Process-New::AcceptedLockFile"])
 918
 919 def check_status(files):
 920     new = byhand = 0
 921     for f in files.keys():
 922         if files[f]["type"] == "byhand":
 923             byhand = 1
 924         elif files[f].has_key("new"):
 925             new = 1
 926     return (new, byhand)
 927
 928 def do_pkg(changes_file):
 929     Upload.pkg.changes_file = changes_file
 930     Upload.init_vars()
 931     Upload.update_vars()
 932     Upload.update_subst()
 933     files = Upload.pkg.files
 934
 935     if not recheck():
 936         return
 937
 938     (new, byhand) = check_status(files)
 939     if new or byhand:
 940         if new:
 941             do_new()
 942         if byhand:
 943             do_byhand()
 944         (new, byhand) = check_status(files)
 945
 946     if not new and not byhand:
 947         do_accept()
 948
 949 ################################################################################
 950
 951 def end():
 952     accept_count = Upload.accept_count
 953     accept_bytes = Upload.accept_bytes
 954
 955     if accept_count:
 956         sets = "set"
 957         if accept_count > 1:
 958             sets = "sets"
 959         sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
 960         Logger.log(["total",accept_count,accept_bytes])
 961
 962     if not Options["No-Action"]:
 963         Logger.close()
 964
 965 ################################################################################
 966
 967 def do_comments(dir, opref, npref, line, fn):
 968     for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
 969         lines = open("%s/%s" % (dir, comm)).readlines()
 970         if len(lines) == 0 or lines[0] != line + "\n": continue
 971         changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
 972                                 and x.endswith(".changes") ]
 973         changes_files = sort_changes(changes_files)
 974         for f in changes_files:
 975             f = utils.validate_changes_file_arg(f, 0)
 976             if not f: continue
 977             print "\n" + f
 978             fn(f, "".join(lines[1:]))
 979
 980         if opref != npref and not Options["No-Action"]:
 981             newcomm = npref + comm[len(opref):]
 982             os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
 983
 984 ################################################################################
 985
 986 def comment_accept(changes_file, comments):
 987     Upload.pkg.changes_file = changes_file
 988     Upload.init_vars()
 989     Upload.update_vars()
 990     Upload.update_subst()
 991     files = Upload.pkg.files
 992
 993     if not recheck():
 994         return # dak wants to REJECT, crap
 995
 996     (new, byhand) = check_status(files)
 997     if not new and not byhand:
 998         do_accept()
 999
1000 ################################################################################
1001
1002 def comment_reject(changes_file, comments):
1003     Upload.pkg.changes_file = changes_file
1004     Upload.init_vars()
1005     Upload.update_vars()
1006     Upload.update_subst()
1007
1008     if not recheck():
1009         pass # dak has its own reasons to reject as well, which is fine
1010
1011     reject(comments)
1012     print "REJECT\n" + reject_message,
1013     if not Options["No-Action"]:
1014         Upload.do_reject(0, reject_message)
1015         os.unlink(Upload.pkg.changes_file[:-8]+".dak")
1016
1017 ################################################################################
1018
1019 def main():
1020     changes_files = init()
1021     if len(changes_files) > 50:
1022         sys.stderr.write("Sorting changes...\n")
1023     changes_files = sort_changes(changes_files)
1024
1025     # Kill me now? **FIXME**
1026     Cnf["Dinstall::Options::No-Mail"] = ""
1027     bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $"
1028     if Cnf.has_key("Dinstall::Bcc"):
1029         Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"])
1030     else:
1031         Upload.Subst["__BCC__"] = bcc
1032
1033     commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
1034     if commentsdir:
1035         if changes_files != []:
1036             sys.stderr.write("Can't specify any changes files if working with comments-dir")
1037             sys.exit(1)
1038         do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
1039         do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
1040     else:
1041         for changes_file in changes_files:
1042             changes_file = utils.validate_changes_file_arg(changes_file, 0)
1043             if not changes_file:
1044                 continue
1045             print "\n" + changes_file
1046             do_pkg (changes_file)
1047
1048     end()
1049
1050 ################################################################################
1051
1052 if __name__ == '__main__':
1053     main()