katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001, 2002  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.24 2002-06-08 00:19:55 troup Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28 from string import lower;
  29
  30 ###############################################################################
  31
  32 re_isanum = re.compile (r"^\d+$");
  33 re_default_answer = re.compile(r"\[(.*)\]");
  34 re_fdnic = re.compile("\n\n");
  35 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
  36 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
  37
  38 ###############################################################################
  39
  40 # Convenience wrapper to carry around all the package information in
  41
  42 class Pkg:
  43     def __init__(self, **kwds):
  44         self.__dict__.update(kwds);
  45
  46     def update(self, **kwds):
  47         self.__dict__.update(kwds);
  48
  49 ###############################################################################
  50
  51 class nmu_p:
  52     # Read in the group maintainer override file
  53     def __init__ (self, Cnf):
  54         self.group_maint = {};
  55         self.Cnf = Cnf;
  56         if Cnf.get("Dinstall::GroupOverrideFilename"):
  57             filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"];
  58             file = utils.open_file(filename);
  59             for line in file.readlines():
  60                 line = lower(string.strip(utils.re_comments.sub('', line)));
  61                 if line != "":
  62                     self.group_maint[line] = 1;
  63             file.close();
  64
  65     def is_an_nmu (self, pkg):
  66         Cnf = self.Cnf;
  67         changes = pkg.changes;
  68         dsc = pkg.dsc;
  69
  70         (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (lower(dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"])));
  71         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  72         if dsc_name == lower(changes["maintainername"]) and \
  73            (changes["changedby822"] == "" or lower(changes["changedbyname"]) == dsc_name):
  74             return 0;
  75
  76         if dsc.has_key("uploaders"):
  77             uploaders = string.split(lower(dsc["uploaders"]), ",");
  78             uploadernames = {};
  79             for i in uploaders:
  80                 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
  81                 uploadernames[name] = "";
  82             if uploadernames.has_key(lower(changes["changedbyname"])):
  83                 return 0;
  84
  85         # Some group maintained packages (e.g. Debian QA) are never NMU's
  86         if self.group_maint.has_key(lower(changes["maintaineremail"])):
  87             return 0;
  88
  89         return 1;
  90
  91 ###############################################################################
  92
  93 class Katie:
  94
  95     def __init__(self, Cnf):
  96         self.Cnf = Cnf;
  97         self.values = {};
  98         # Read in the group-maint override file
  99         self.nmu = nmu_p(Cnf);
 100         self.accept_count = 0;
 101         self.accept_bytes = 0L;
 102         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 103                        legacy_source_untouchable = {});
 104
 105         # Initialize the substitution template mapping global
 106         Subst = self.Subst = {};
 107         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 108         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 109         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 110         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 111
 112         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 113         db_access.init(Cnf, self.projectB);
 114
 115     ###########################################################################
 116
 117     def init_vars (self):
 118         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 119             exec "self.pkg.%s.clear();" % (i);
 120         self.pkg.orig_tar_id = None;
 121         self.pkg.orig_tar_location = "";
 122
 123     ###########################################################################
 124
 125     def update_vars (self):
 126         dump_filename = self.pkg.changes_file[:-8]+".katie";
 127         dump_file = utils.open_file(dump_filename);
 128         p = cPickle.Unpickler(dump_file);
 129         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 130             exec "self.pkg.%s.update(p.load());" % (i);
 131         for i in [ "orig_tar_id", "orig_tar_location" ]:
 132             exec "self.pkg.%s = p.load();" % (i);
 133         dump_file.close();
 134
 135     ###########################################################################
 136
 137     # This could just dump the dictionaries as is, but I'd like to avoid
 138     # this so there's some idea of what katie & lisa use from jennifer
 139
 140     def dump_vars(self, dest_dir):
 141         for i in [ "changes", "dsc", "files", "dsc_files",
 142                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 143             exec "%s = self.pkg.%s;" % (i,i);
 144         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 145         dump_file = utils.open_file(dump_filename, 'w');
 146         try:
 147             os.chmod(dump_filename, 0660);
 148         except OSError, e:
 149             if errno.errorcode[e.errno] == 'EPERM':
 150                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]);
 151                 if perms & stat.S_IROTH:
 152                     utils.fubar("%s is world readable and chmod failed." % (dump_filename));
 153             else:
 154                 raise;
 155
 156         p = cPickle.Pickler(dump_file, 1);
 157         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 158             exec "%s = {}" % i;
 159         ## files
 160         for file in files.keys():
 161             d_files[file] = {};
 162             for i in [ "package", "version", "architecture", "type", "size",
 163                        "md5sum", "component", "location id", "source package",
 164                        "source version", "maintainer", "dbtype", "files id",
 165                        "new", "section", "priority", "othercomponents",
 166                        "pool name", "original component" ]:
 167                 if files[file].has_key(i):
 168                     d_files[file][i] = files[file][i];
 169         ## changes
 170         # Mandatory changes fields
 171         for i in [ "distribution", "source", "architecture", "version", "maintainer",
 172                    "urgency", "fingerprint", "changedby822", "changedbyname",
 173                    "maintainername", "maintaineremail", "closes" ]:
 174             d_changes[i] = changes[i];
 175         # Optional changes fields
 176         # FIXME: changes should be mandatory
 177         for i in [ "changed-by", "maintainer822", "filecontents", "format",
 178                    "changes", "lisa note" ]:
 179             if changes.has_key(i):
 180                 d_changes[i] = changes[i];
 181         ## dsc
 182         for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
 183             if dsc.has_key(i):
 184                 d_dsc[i] = dsc[i];
 185         ## dsc_files
 186         for file in dsc_files.keys():
 187             d_dsc_files[file] = {};
 188             # Mandatory dsc_files fields
 189             for i in [ "size", "md5sum" ]:
 190                 d_dsc_files[file][i] = dsc_files[file][i];
 191             # Optional dsc_files fields
 192             for i in [ "files id" ]:
 193                 if dsc_files[file].has_key(i):
 194                     d_dsc_files[file][i] = dsc_files[file][i];
 195
 196         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 197                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 198             p.dump(i);
 199         dump_file.close();
 200
 201     ###########################################################################
 202
 203     # Set up the per-package template substitution mappings
 204
 205     def update_subst (self, reject_message = ""):
 206         Subst = self.Subst;
 207         changes = self.pkg.changes;
 208         # If jennifer crashed out in the right place, architecture may still be a string.
 209         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 210             changes["architecture"] = { "Unknown" : "" };
 211         # and maintainer822 may not exist.
 212         if not changes.has_key("maintainer822"):
 213             changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
 214
 215         Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
 216         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 217         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 218
 219         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 220         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 221             Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
 222             Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
 223             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 224         else:
 225             Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
 226             Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
 227             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 228         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 229             Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 230
 231         # Apply any global override of the Maintainer field
 232         if self.Cnf.get("Dinstall::OverrideMaintainer"):
 233             Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 234             Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 235
 236         Subst["__REJECT_MESSAGE__"] = reject_message;
 237         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 238         Subst["__VERSION__"] = changes.get("version", "Unknown");
 239
 240     ###########################################################################
 241
 242     def build_summaries(self):
 243         changes = self.pkg.changes;
 244         files = self.pkg.files;
 245
 246         byhand = summary = new = "";
 247
 248         # changes["distribution"] may not exist in corner cases
 249         # (e.g. unreadable changes files)
 250         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 251             changes["distribution"] = {};
 252
 253         file_keys = files.keys();
 254         file_keys.sort();
 255         for file in file_keys:
 256             if files[file].has_key("byhand"):
 257                 byhand = 1
 258                 summary = summary + file + " byhand\n"
 259             elif files[file].has_key("new"):
 260                 new = 1
 261                 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 262                 if files[file].has_key("othercomponents"):
 263                     summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 264                 if files[file]["type"] == "deb":
 265                     summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 266             else:
 267                 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
 268                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 269                 summary = summary + file + "\n  to " + destination + "\n"
 270
 271         short_summary = summary;
 272
 273         # This is for direport's benefit...
 274         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 275
 276         if byhand or new:
 277             summary = summary + "Changes: " + f;
 278
 279         summary = summary + self.announce(short_summary, 0)
 280
 281         return (summary, short_summary);
 282
 283     ###########################################################################
 284
 285     def close_bugs (self, summary, action):
 286         changes = self.pkg.changes;
 287         Subst = self.Subst;
 288         Cnf = self.Cnf;
 289
 290         bugs = changes["closes"].keys();
 291
 292         if not bugs:
 293             return summary;
 294
 295         bugs.sort();
 296         if not self.nmu.is_an_nmu(self.pkg):
 297             summary = summary + "Closing bugs: ";
 298             for bug in bugs:
 299                 summary = summary + "%s " % (bug);
 300                 if action:
 301                     Subst["__BUG_NUMBER__"] = bug;
 302                     if changes["distribution"].has_key("stable"):
 303                         Subst["__STABLE_WARNING__"] = """
 304 Note that this package is not part of the released stable Debian
 305 distribution.  It may have dependencies on other unreleased software,
 306 or other instabilities.  Please take care if you wish to install it.
 307 The update will eventually make its way into the next released Debian
 308 distribution.""";
 309                     else:
 310                         Subst["__STABLE_WARNING__"] = "";
 311                     mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
 312                     utils.send_mail (mail_message, "");
 313             if action:
 314                 self.Logger.log(["closing bugs"]+bugs);
 315         else:                     # NMU
 316             summary = summary + "Setting bugs to severity fixed: ";
 317             control_message = "";
 318             for bug in bugs:
 319                 summary = summary + "%s " % (bug);
 320                 control_message = control_message + "tag %s + fixed\n" % (bug);
 321             if action and control_message != "":
 322                 Subst["__CONTROL_MESSAGE__"] = control_message;
 323                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed");
 324                 utils.send_mail (mail_message, "");
 325             if action:
 326                 self.Logger.log(["setting bugs to fixed"]+bugs);
 327         summary = summary + "\n";
 328         return summary;
 329
 330     ###########################################################################
 331
 332     def announce (self, short_summary, action):
 333         Subst = self.Subst;
 334         Cnf = self.Cnf;
 335         changes = self.pkg.changes;
 336
 337         # Only do announcements for source uploads with a recent dpkg-dev installed
 338         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 339             return "";
 340
 341         lists_done = {};
 342         summary = "";
 343         Subst["__SHORT_SUMMARY__"] = short_summary;
 344
 345         for dist in changes["distribution"].keys():
 346             list = Cnf.Find("Suite::%s::Announce" % (dist));
 347             if list == "" or lists_done.has_key(list):
 348                 continue;
 349             lists_done[list] = 1;
 350             summary = summary + "Announcing to %s\n" % (list);
 351
 352             if action:
 353                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 354                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 355                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]);
 356                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce");
 357                 utils.send_mail (mail_message, "");
 358
 359         if Cnf.get("Dinstall::CloseBugs"):
 360             summary = self.close_bugs(summary, action);
 361
 362         return summary;
 363
 364     ###########################################################################
 365
 366     def accept (self, summary, short_summary):
 367         Cnf = self.Cnf;
 368         Subst = self.Subst;
 369         files = self.pkg.files;
 370
 371         print "Accepting."
 372         self.Logger.log(["Accepting changes",self.pkg.changes_file]);
 373
 374         self.dump_vars(Cnf["Dir::Queue::Accepted"]);
 375
 376         # Move all the files into the accepted directory
 377         utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
 378         file_keys = files.keys();
 379         for file in file_keys:
 380             utils.move(file, Cnf["Dir::Queue::Accepted"]);
 381             self.accept_bytes = self.accept_bytes + float(files[file]["size"])
 382         self.accept_count = self.accept_count + 1;
 383
 384         # Send accept mail, announce to lists, close bugs and check for
 385         # override disparities
 386         if not Cnf["Dinstall::Options::No-Mail"]:
 387             Subst["__SUITE__"] = "";
 388             Subst["__SUMMARY__"] = summary;
 389             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted");
 390             utils.send_mail(mail_message, "")
 391             self.announce(short_summary, 1)
 392
 393         # Special support to enable clean auto-building of accepted packages
 394         self.projectB.query("BEGIN WORK");
 395         for suite in self.pkg.changes["distribution"].keys():
 396             if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
 397                 continue;
 398             suite_id = db_access.get_suite_id(suite);
 399             dest_dir = Cnf["Dir::AcceptedAutoBuild"];
 400             if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 401                 dest_dir = os.path.join(dest_dir, suite);
 402             for file in file_keys:
 403                 src = os.path.join(Cnf["Dir::Queue::Accepted"], file);
 404                 dest = os.path.join(dest_dir, file);
 405                 if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 406                     # Copy it since the original won't be readable by www-data
 407                     utils.copy(src, dest);
 408                 else:
 409                     # Create a symlink to it
 410                     os.symlink(src, dest);
 411                 # Add it to the list of packages for later processing by apt-ftparchive
 412                 self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 413             # If the .orig.tar.gz is in the pool, create a symlink to
 414             # it (if one doesn't already exist)
 415             if self.pkg.orig_tar_id:
 416                 # Determine the .orig.tar.gz file name
 417                 for dsc_file in self.pkg.dsc_files.keys():
 418                     if dsc_file[-12:] == ".orig.tar.gz":
 419                         filename = dsc_file;
 420                 dest = os.path.join(dest_dir, filename);
 421                 # If it doesn't exist, create a symlink
 422                 if not os.path.exists(dest):
 423                     # Find the .orig.tar.gz in the pool
 424                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 425                     ql = q.getresult();
 426                     if not ql:
 427                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 428                     src = os.path.join(ql[0][0], ql[0][1]);
 429                     os.symlink(src, dest);
 430                     # Add it to the list of packages for later processing by apt-ftparchive
 431                     self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 432                 # if it does, update things to ensure it's not removed prematurely
 433                 else:
 434                     self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id));
 435
 436         self.projectB.query("COMMIT WORK");
 437
 438     ###########################################################################
 439
 440     def check_override (self):
 441         Subst = self.Subst;
 442         changes = self.pkg.changes;
 443         files = self.pkg.files;
 444         Cnf = self.Cnf;
 445
 446         # Abandon the check if:
 447         #  a) it's a non-sourceful upload
 448         #  b) override disparity checks have been disabled
 449         #  c) we're not sending mail
 450         if not changes["architecture"].has_key("source") or \
 451            not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 452            Cnf["Dinstall::Options::No-Mail"]:
 453             return;
 454
 455         summary = "";
 456         for file in files.keys():
 457             if not files[file].has_key("new") and files[file]["type"] == "deb":
 458                 section = files[file]["section"];
 459                 override_section = files[file]["override section"];
 460                 if lower(section) != lower(override_section) and section != "-":
 461                     # Ignore this; it's a common mistake and not worth whining about
 462                     if lower(section) == "non-us/main" and lower(override_section) == "non-us":
 463                         continue;
 464                     summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
 465                 priority = files[file]["priority"];
 466                 override_priority = files[file]["override priority"];
 467                 if priority != override_priority and priority != "-":
 468                     summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
 469
 470         if summary == "":
 471             return;
 472
 473         Subst["__SUMMARY__"] = summary;
 474         mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity");
 475         utils.send_mail (mail_message, "");
 476
 477     ###########################################################################
 478
 479     def force_move (self, files):
 480         """Forcefully move files from the current directory to the reject
 481            directory.  If any file already exists it will be moved to the
 482            morgue to make way for the new file."""
 483
 484         Cnf = self.Cnf
 485
 486         for file in files:
 487             # Skip any files which don't exist or which we don't have permission to copy.
 488             if os.access(file,os.R_OK) == 0:
 489                 continue;
 490             dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file);
 491             try:
 492                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 493             except OSError, e:
 494                 # File exists?  Let's try and move it to the morgue
 495                 if errno.errorcode[e.errno] == 'EEXIST':
 496                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file);
 497                     try:
 498                         morgue_file = utils.find_next_free(morgue_file);
 499                     except utils.tried_too_hard_exc:
 500                         # Something's either gone badly Pete Tong, or
 501                         # someone is trying to exploit us.
 502                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 503                         return;
 504                     utils.move(dest_file, morgue_file, perms=0660);
 505                     try:
 506                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 507                     except OSError, e:
 508                         # Likewise
 509                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 510                         return;
 511                 else:
 512                     raise;
 513             # If we got here, we own the destination file, so we can
 514             # safely overwrite it.
 515             utils.move(file, dest_file, 1, perms=0660);
 516
 517     ###########################################################################
 518
 519     def do_reject (self, manual = 0, reject_message = ""):
 520         # If we weren't given a manual rejection message, spawn an
 521         # editor so the user can add one in...
 522         if manual and not reject_message:
 523             temp_filename = tempfile.mktemp();
 524             fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
 525             os.close(fd);
 526             editor = os.environ.get("EDITOR","vi")
 527             answer = 'E';
 528             while answer == 'E':
 529                 os.system("%s %s" % (editor, temp_filename))
 530                 file = utils.open_file(temp_filename);
 531                 reject_message = string.join(file.readlines());
 532                 file.close();
 533                 print "Reject message:";
 534                 print utils.prefix_multi_line_string(reject_message,"  ");
 535                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 536                 answer = "XXX";
 537                 while string.find(prompt, answer) == -1:
 538                     answer = utils.our_raw_input(prompt);
 539                     m = re_default_answer.search(prompt);
 540                     if answer == "":
 541                         answer = m.group(1);
 542                     answer = string.upper(answer[:1]);
 543             os.unlink(temp_filename);
 544             if answer == 'A':
 545                 return 1;
 546             elif answer == 'Q':
 547                 sys.exit(0);
 548
 549         print "Rejecting.\n"
 550
 551         Cnf = self.Cnf;
 552         Subst = self.Subst;
 553         pkg = self.pkg;
 554
 555         reason_filename = pkg.changes_file[:-8] + ".reason";
 556         reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
 557
 558         # Move all the files into the reject directory
 559         reject_files = pkg.files.keys() + [pkg.changes_file];
 560         self.force_move(reject_files);
 561
 562         # If we fail here someone is probably trying to exploit the race
 563         # so let's just raise an exception ...
 564         if os.path.exists(reject_filename):
 565             os.unlink(reject_filename);
 566         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 567
 568         if not manual:
 569             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 570             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 571             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 572             os.write(fd, reject_message);
 573             os.close(fd);
 574             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 575         else:
 576             # Build up the rejection email
 577             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 578
 579             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 580             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 581             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 582             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 583
 584             # Write the rejection email out as the <foo>.reason file
 585             os.write(fd, reject_mail_message);
 586             os.close(fd);
 587
 588         # Send the rejection mail if appropriate
 589         if not Cnf["Dinstall::Options::No-Mail"]:
 590             utils.send_mail (reject_mail_message, "");
 591
 592         self.Logger.log(["rejected", pkg.changes_file]);
 593         return 0;
 594
 595     ################################################################################
 596
 597     # Ensure that source exists somewhere in the archive for the binary
 598     # upload being processed.
 599     #
 600     # (1) exact match                      => 1.0-3
 601     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 602     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 603
 604     def source_exists (self, package, source_version):
 605         q = self.projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
 606
 607         # Reduce the query results to a list of version numbers
 608         ql = map(lambda x: x[0], q.getresult());
 609
 610         # Try (1)
 611         if ql.count(source_version):
 612             return 1;
 613
 614         # Try (2)
 615         orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
 616         if ql.count(orig_source_version):
 617             return 1;
 618
 619         # Try (3)
 620         orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
 621         if ql.count(orig_source_version):
 622             return 1;
 623
 624         # No source found...
 625         return 0;
 626
 627     ################################################################################
 628
 629     def in_override_p (self, package, component, suite, binary_type, file):
 630         files = self.pkg.files;
 631
 632         if binary_type == "": # must be source
 633             type = "dsc";
 634         else:
 635             type = binary_type;
 636
 637         # Override suite name; used for example with proposed-updates
 638         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 639             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 640
 641         # Avoid <undef> on unknown distributions
 642         suite_id = db_access.get_suite_id(suite);
 643         if suite_id == -1:
 644             return None;
 645         component_id = db_access.get_component_id(component);
 646         type_id = db_access.get_override_type_id(type);
 647
 648         # FIXME: nasty non-US speficic hack
 649         if lower(component[:7]) == "non-us/":
 650             component = component[7:];
 651
 652         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 653                            % (package, suite_id, component_id, type_id));
 654         result = q.getresult();
 655         # If checking for a source package fall back on the binary override type
 656         if type == "dsc" and not result:
 657             type_id = db_access.get_override_type_id("deb");
 658             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 659                                % (package, suite_id, component_id, type_id));
 660             result = q.getresult();
 661
 662         # Remember the section and priority so we can check them later if appropriate
 663         if result:
 664             files[file]["override section"] = result[0][0];
 665             files[file]["override priority"] = result[0][1];
 666
 667         return result;
 668
 669     ################################################################################
 670
 671     def reject (self, str, prefix="Rejected: "):
 672         if str:
 673             # Unlike other rejects we add new lines first to avoid trailing
 674             # new lines when this message is passed back up to a caller.
 675             if self.reject_message:
 676                 self.reject_message = self.reject_message + "\n";
 677             self.reject_message = self.reject_message + prefix + str;
 678
 679     ################################################################################
 680
 681     def cross_suite_version_check(self, query_result, file, new_version):
 682         """Ensure versions are newer than existing packages in target
 683         suites and that cross-suite version checking rules as
 684         set out in the conf file are satisfied."""
 685
 686         # Check versions for each target suite
 687         for target_suite in self.pkg.changes["distribution"].keys():
 688             must_be_newer_than = map(lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
 689             must_be_older_than = map(lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
 690             # Enforce "must be newer than target suite" even if conffile omits it
 691             if target_suite not in must_be_newer_than:
 692                 must_be_newer_than.append(target_suite);
 693             for entry in query_result:
 694                 existent_version = entry[0];
 695                 suite = entry[1];
 696                 if suite in must_be_newer_than and \
 697                    apt_pkg.VersionCompare(new_version, existent_version) != 1:
 698                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 699                 if suite in must_be_older_than and \
 700                    apt_pkg.VersionCompare(new_version, existent_version) != -1:
 701                     self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 702
 703     ################################################################################
 704
 705     def check_binary_against_db(self, file):
 706         self.reject_message = "";
 707         files = self.pkg.files;
 708
 709         # Ensure version is sane
 710         q = self.projectB.query("""
 711 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
 712                                      architecture a
 713  WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
 714    AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
 715                                 % (files[file]["package"],
 716                                    files[file]["architecture"]));
 717         self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
 718
 719         # Check for any existing copies of the file
 720         q = self.projectB.query("""
 721 SELECT b.id FROM binaries b, architecture a
 722  WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
 723    AND a.id = b.architecture"""
 724                                 % (files[file]["package"],
 725                                    files[file]["version"],
 726                                    files[file]["architecture"]))
 727         if q.getresult():
 728             self.reject("can not overwrite existing copy of '%s' already in the archive." % (file));
 729
 730         return self.reject_message;
 731
 732     ################################################################################
 733
 734     def check_source_against_db(self, file):
 735         self.reject_message = "";
 736         dsc = self.pkg.dsc;
 737
 738         # Ensure version is sane
 739         q = self.projectB.query("""
 740 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
 741  WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
 742         self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
 743
 744         return self.reject_message;
 745
 746     ################################################################################
 747
 748     def check_dsc_against_db(self, file):
 749         self.reject_message = "";
 750         files = self.pkg.files;
 751         dsc_files = self.pkg.dsc_files;
 752         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 753         orig_tar_gz = None;
 754
 755         # Try and find all files mentioned in the .dsc.  This has
 756         # to work harder to cope with the multiple possible
 757         # locations of an .orig.tar.gz.
 758         for dsc_file in dsc_files.keys():
 759             found = None;
 760             if files.has_key(dsc_file):
 761                 actual_md5 = files[dsc_file]["md5sum"];
 762                 actual_size = int(files[dsc_file]["size"]);
 763                 found = "%s in incoming" % (dsc_file)
 764                 # Check the file does not already exist in the archive
 765                 q = self.projectB.query("SELECT f.size, f.md5sum FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
 766
 767                 # "It has not broken them.  It has fixed a
 768                 # brokenness.  Your crappy hack exploited a bug in
 769                 # the old dinstall.
 770                 #
 771                 # "(Come on!  I thought it was always obvious that
 772                 # one just doesn't release different files with
 773                 # the same name and version.)"
 774                 #                        -- ajk@ on d-devel@l.d.o
 775
 776                 ql = q.getresult();
 777                 if ql:
 778                     # Ignore exact matches for .orig.tar.gz
 779                     match = 0;
 780                     if dsc_file[-12:] == ".orig.tar.gz":
 781                         for i in ql:
 782                             if files.has_key(dsc_file) and \
 783                                int(files[dsc_file]["size"]) == int(i[0]) and \
 784                                files[dsc_file]["md5sum"] == i[1]:
 785                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
 786                                 del files[dsc_file];
 787                                 match = 1;
 788
 789                     if not match:
 790                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 791             elif dsc_file[-12:] == ".orig.tar.gz":
 792                 # Check in the pool
 793                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
 794                 ql = q.getresult();
 795
 796                 if ql:
 797                     # Unfortunately, we make get more than one
 798                     # match here if, for example, the package was
 799                     # in potato but had a -sa upload in woody.  So
 800                     # we need to choose the right one.
 801
 802                     x = ql[0]; # default to something sane in case we don't match any or have only one
 803
 804                     if len(ql) > 1:
 805                         for i in ql:
 806                             old_file = i[0] + i[1];
 807                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 808                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 809                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 810                                 x = i;
 811                             else:
 812                                 legacy_source_untouchable[i[3]] = "";
 813
 814                     old_file = x[0] + x[1];
 815                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 816                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 817                     found = old_file;
 818                     suite_type = x[2];
 819                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 820                     # See install() in katie...
 821                     self.pkg.orig_tar_id = x[3];
 822                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 823                         self.pkg.orig_tar_location = "legacy";
 824                     else:
 825                         self.pkg.orig_tar_location = x[4];
 826                 else:
 827                     # Not there? Check the queue directories...
 828
 829                     in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file);
 830                     # See process_it() in jennifer for explanation of this
 831                     if os.path.exists(in_unchecked):
 832                         return (self.reject_message, in_unchecked);
 833                     else:
 834                         for dir in [ "Accepted", "New", "Byhand" ]:
 835                             in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file);
 836                             if os.path.exists(in_otherdir):
 837                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 838                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 839                                 found = in_otherdir;
 840
 841                     if not found:
 842                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 843                         continue;
 844             else:
 845                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 846                 continue;
 847             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 848                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 849             if actual_size != int(dsc_files[dsc_file]["size"]):
 850                 self.reject("size for %s doesn't match %s." % (found, file));
 851
 852         return (self.reject_message, orig_tar_gz);
 853
 854     def do_query(self, q):
 855         sys.stderr.write("query: \"%s\" ... " % (q));
 856         before = time.time();
 857         r = self.projectB.query(q);
 858         time_diff = time.time()-before;
 859         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
 860         return r;