katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.17 2002-04-29 22:00:44 troup Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28 from string import lower;
  29
  30 ###############################################################################
  31
  32 re_isanum = re.compile (r"^\d+$");
  33 re_default_answer = re.compile(r"\[(.*)\]");
  34 re_fdnic = re.compile("\n\n");
  35 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
  36 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
  37
  38 ###############################################################################
  39
  40 # Convenience wrapper to carry around all the package information in
  41
  42 class Pkg:
  43     def __init__(self, **kwds):
  44         self.__dict__.update(kwds);
  45
  46     def update(self, **kwds):
  47         self.__dict__.update(kwds);
  48
  49 ###############################################################################
  50
  51 class nmu_p:
  52     # Read in the group maintainer override file
  53     def __init__ (self, Cnf):
  54         self.group_maint = {};
  55         self.Cnf = Cnf;
  56         if Cnf.get("Dinstall::GroupOverrideFilename"):
  57             filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
  58             file = utils.open_file(filename);
  59             for line in file.readlines():
  60                 line = lower(string.strip(utils.re_comments.sub('', line)));
  61                 if line != "":
  62                     self.group_maint[line] = 1;
  63             file.close();
  64
  65     def is_an_nmu (self, pkg):
  66         Cnf = self.Cnf;
  67         changes = pkg.changes;
  68         dsc = pkg.dsc;
  69
  70         (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (lower(dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"])));
  71         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  72         if dsc_name == lower(changes["maintainername"]) and \
  73            (changes["changedby822"] == "" or lower(changes["changedbyname"]) == dsc_name):
  74             return 0;
  75
  76         if dsc.has_key("uploaders"):
  77             uploaders = string.split(lower(dsc["uploaders"]), ",");
  78             uploadernames = {};
  79             for i in uploaders:
  80                 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
  81                 uploadernames[name] = "";
  82             if uploadernames.has_key(lower(changes["changedbyname"])):
  83                 return 0;
  84
  85         # Some group maintained packages (e.g. Debian QA) are never NMU's
  86         if self.group_maint.has_key(lower(changes["maintaineremail"])):
  87             return 0;
  88
  89         return 1;
  90
  91 ###############################################################################
  92
  93 class Katie:
  94
  95     def __init__(self, Cnf):
  96         self.Cnf = Cnf;
  97         self.values = {};
  98         # Read in the group-maint override file
  99         self.nmu = nmu_p(Cnf);
 100         self.accept_count = 0;
 101         self.accept_bytes = 0L;
 102         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 103                        legacy_source_untouchable = {});
 104
 105         # Initialize the substitution template mapping global
 106         Subst = self.Subst = {};
 107         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 108         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 109         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 110         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 111
 112         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 113         db_access.init(Cnf, self.projectB);
 114
 115     ###########################################################################
 116
 117     def init_vars (self):
 118         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 119             exec "self.pkg.%s.clear();" % (i);
 120         self.pkg.orig_tar_id = None;
 121         self.pkg.orig_tar_location = "";
 122
 123     ###########################################################################
 124
 125     def update_vars (self):
 126         dump_filename = self.pkg.changes_file[:-8]+".katie";
 127         dump_file = utils.open_file(dump_filename);
 128         p = cPickle.Unpickler(dump_file);
 129         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 130             exec "self.pkg.%s.update(p.load());" % (i);
 131         for i in [ "orig_tar_id", "orig_tar_location" ]:
 132             exec "self.pkg.%s = p.load();" % (i);
 133         dump_file.close();
 134
 135     ###########################################################################
 136
 137     # This could just dump the dictionaries as is, but I'd like to avoid
 138     # this so there's some idea of what katie & lisa use from jennifer
 139
 140     def dump_vars(self, dest_dir):
 141         for i in [ "changes", "dsc", "files", "dsc_files",
 142                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 143             exec "%s = self.pkg.%s;" % (i,i);
 144         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 145         dump_file = utils.open_file(dump_filename, 'w');
 146         p = cPickle.Pickler(dump_file, 1);
 147         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 148             exec "%s = {}" % i;
 149         ## files
 150         for file in files.keys():
 151             d_files[file] = {};
 152             for i in [ "package", "version", "architecture", "type", "size",
 153                        "md5sum", "component", "location id", "source package",
 154                        "source version", "maintainer", "dbtype", "files id",
 155                        "new", "section", "priority", "oldfiles", "othercomponents" ]:
 156                 if files[file].has_key(i):
 157                     d_files[file][i] = files[file][i];
 158         ## changes
 159         # Mandatory changes fields
 160         for i in [ "distribution", "source", "architecture", "version", "maintainer",
 161                    "urgency", "fingerprint", "changedby822", "changedbyname",
 162                    "maintainername", "maintaineremail", "closes" ]:
 163             d_changes[i] = changes[i];
 164         # Optional changes fields
 165         # FIXME: changes should be mandatory
 166         for i in [ "changed-by", "maintainer822", "filecontents", "format", "changes" ]:
 167             if changes.has_key(i):
 168                 d_changes[i] = changes[i];
 169         ## dsc
 170         for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
 171             if dsc.has_key(i):
 172                 d_dsc[i] = dsc[i];
 173         ## dsc_files
 174         for file in dsc_files.keys():
 175             d_dsc_files[file] = {};
 176             # Mandatory dsc_files fields
 177             for i in [ "size", "md5sum" ]:
 178                 d_dsc_files[file][i] = dsc_files[file][i];
 179             # Optional dsc_files fields
 180             for i in [ "files id" ]:
 181                 if dsc_files[file].has_key(i):
 182                     d_dsc_files[file][i] = dsc_files[file][i];
 183
 184         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 185                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 186             p.dump(i);
 187         dump_file.close();
 188
 189     ###########################################################################
 190
 191     # Set up the per-package template substitution mappings
 192
 193     def update_subst (self, reject_message = ""):
 194         Subst = self.Subst;
 195         changes = self.pkg.changes;
 196         # If jennifer crashed out in the right place, architecture may still be a string.
 197         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 198             changes["architecture"] = { "Unknown" : "" };
 199         # and maintainer822 may not exist.
 200         if not changes.has_key("maintainer822"):
 201             changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
 202
 203         Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
 204         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 205         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 206
 207         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 208         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 209             Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
 210             Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
 211             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 212         else:
 213             Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
 214             Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
 215             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 216         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 217             Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 218
 219         Subst["__REJECT_MESSAGE__"] = reject_message;
 220         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 221         Subst["__VERSION__"] = changes.get("version", "Unknown");
 222
 223     ###########################################################################
 224
 225     def build_summaries(self):
 226         changes = self.pkg.changes;
 227         files = self.pkg.files;
 228
 229         byhand = summary = new = "";
 230
 231         # changes["distribution"] may not exist in corner cases
 232         # (e.g. unreadable changes files)
 233         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 234             changes["distribution"] = {};
 235
 236         file_keys = files.keys();
 237         file_keys.sort();
 238         for file in file_keys:
 239             if files[file].has_key("byhand"):
 240                 byhand = 1
 241                 summary = summary + file + " byhand\n"
 242             elif files[file].has_key("new"):
 243                 new = 1
 244                 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 245                 if files[file].has_key("othercomponents"):
 246                     summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 247                 if files[file]["type"] == "deb":
 248                     summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 249             else:
 250                 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
 251                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 252                 summary = summary + file + "\n  to " + destination + "\n"
 253
 254         short_summary = summary;
 255
 256         # This is for direport's benefit...
 257         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 258
 259         if byhand or new:
 260             summary = summary + "Changes: " + f;
 261
 262         summary = summary + self.announce(short_summary, 0)
 263
 264         return (summary, short_summary);
 265
 266     ###########################################################################
 267
 268     def announce (self, short_summary, action):
 269         Subst = self.Subst;
 270         Cnf = self.Cnf;
 271         changes = self.pkg.changes;
 272         dsc = self.pkg.dsc;
 273
 274         # Only do announcements for source uploads with a recent dpkg-dev installed
 275         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 276             return ""
 277
 278         lists_done = {}
 279         summary = ""
 280         Subst["__SHORT_SUMMARY__"] = short_summary;
 281
 282         for dist in changes["distribution"].keys():
 283             list = Cnf.Find("Suite::%s::Announce" % (dist))
 284             if list == "" or lists_done.has_key(list):
 285                 continue
 286             lists_done[list] = 1
 287             summary = summary + "Announcing to %s\n" % (list)
 288
 289             if action:
 290                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 291                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 292                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"])
 293                 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.announce","r").read());
 294                 utils.send_mail (mail_message, "")
 295
 296         bugs = changes["closes"].keys()
 297         bugs.sort()
 298         if not self.nmu.is_an_nmu(self.pkg):
 299             summary = summary + "Closing bugs: "
 300             for bug in bugs:
 301                 summary = summary + "%s " % (bug)
 302                 if action:
 303                     Subst["__BUG_NUMBER__"] = bug;
 304                     if changes["distribution"].has_key("stable"):
 305                         Subst["__STABLE_WARNING__"] = """
 306     Note that this package is not part of the released stable Debian
 307     distribution.  It may have dependencies on other unreleased software,
 308     or other instabilities.  Please take care if you wish to install it.
 309     The update will eventually make its way into the next released Debian
 310     distribution."""
 311                     else:
 312                         Subst["__STABLE_WARNING__"] = "";
 313                     mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.bug-close","r").read());
 314                     utils.send_mail (mail_message, "")
 315             if action:
 316                 self.Logger.log(["closing bugs"]+bugs);
 317         else:                     # NMU
 318             summary = summary + "Setting bugs to severity fixed: "
 319             control_message = ""
 320             for bug in bugs:
 321                 summary = summary + "%s " % (bug)
 322                 control_message = control_message + "tag %s + fixed\n" % (bug)
 323             if action and control_message != "":
 324                 Subst["__CONTROL_MESSAGE__"] = control_message;
 325                 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.bug-nmu-fixed","r").read());
 326                 utils.send_mail (mail_message, "")
 327             if action:
 328                 self.Logger.log(["setting bugs to fixed"]+bugs);
 329         summary = summary + "\n"
 330
 331         return summary
 332
 333     ###########################################################################
 334
 335     def accept (self, summary, short_summary):
 336         Cnf = self.Cnf;
 337         Subst = self.Subst;
 338         files = self.pkg.files;
 339
 340         print "Accepting."
 341         self.Logger.log(["Accepting changes",self.pkg.changes_file]);
 342
 343         self.dump_vars(Cnf["Dir::QueueAcceptedDir"]);
 344
 345         # Move all the files into the accepted directory
 346         utils.move(self.pkg.changes_file, Cnf["Dir::QueueAcceptedDir"]);
 347         file_keys = files.keys();
 348         for file in file_keys:
 349             utils.move(file, Cnf["Dir::QueueAcceptedDir"]);
 350             self.accept_bytes = self.accept_bytes + float(files[file]["size"])
 351         self.accept_count = self.accept_count + 1;
 352
 353         # Send accept mail, announce to lists, close bugs and check for
 354         # override disparities
 355         if not Cnf["Dinstall::Options::No-Mail"]:
 356             Subst["__SUITE__"] = "";
 357             Subst["__SUMMARY__"] = summary;
 358             mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.accepted","r").read());
 359             utils.send_mail(mail_message, "")
 360             self.announce(short_summary, 1)
 361
 362         # Special support to enable clean auto-building of accepted packages
 363         if Cnf.get("Dinstall::SpecialAcceptedAutoBuild") and \
 364            self.pkg.changes["distribution"].has_key("unstable"):
 365             self.projectB.query("BEGIN WORK");
 366             for file in file_keys:
 367                 src = os.path.join(Cnf["Dir::QueueAcceptedDir"], file);
 368                 dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"], file);
 369                 # Create a symlink to it
 370                 os.symlink(src, dest);
 371                 # Add it to the list of packages for later processing by apt-ftparchive
 372                 self.projectB.query("INSERT INTO unstable_accepted (filename, in_accepted) VALUES ('%s', 't')" % (dest));
 373             # If the .orig.tar.gz is in the pool, create a symlink to
 374             # it (if one doesn't already exist)
 375             if self.pkg.orig_tar_id:
 376                 # Determine the .orig.tar.gz file name
 377                 for dsc_file in self.pkg.dsc_files.keys():
 378                     if dsc_file[-12:] == ".orig.tar.gz":
 379                         filename = dsc_file;
 380                 dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"],filename);
 381                 # If it doesn't exist, create a symlink
 382                 if not os.path.exists(dest):
 383                     # Find the .orig.tar.gz in the pool
 384                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 385                     ql = q.getresult();
 386                     if not ql:
 387                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 388                     src = os.path.join(ql[0][0], ql[0][1]);
 389                     os.symlink(src, dest);
 390                     # Add it to the list of packages for later processing by apt-ftparchive
 391                     self.projectB.query("INSERT INTO unstable_accepted (filename, in_accepted) VALUES ('%s', 't')" % (dest));
 392
 393             self.projectB.query("COMMIT WORK");
 394
 395     ###########################################################################
 396
 397     def check_override (self):
 398         Subst = self.Subst;
 399         changes = self.pkg.changes;
 400         files = self.pkg.files;
 401
 402         # Only check section & priority on sourceful uploads
 403         if not changes["architecture"].has_key("source"):
 404             return;
 405
 406         summary = "";
 407         for file in files.keys():
 408             if not files[file].has_key("new") and files[file]["type"] == "deb":
 409                 section = files[file]["section"];
 410                 override_section = files[file]["override section"];
 411                 if lower(section) != lower(override_section) and section != "-":
 412                     # Ignore this; it's a common mistake and not worth whining about
 413                     if lower(section) == "non-us/main" and lower(override_section) == "non-us":
 414                         continue;
 415                     summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
 416                 priority = files[file]["priority"];
 417                 override_priority = files[file]["override priority"];
 418                 if priority != override_priority and priority != "-":
 419                     summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
 420
 421         if summary == "":
 422             return;
 423
 424         Subst["__SUMMARY__"] = summary;
 425         mail_message = utils.TemplateSubst(Subst,utils.open_file(self.Cnf["Dir::TemplatesDir"]+"/jennifer.override-disparity").read());
 426         utils.send_mail (mail_message, "");
 427
 428     ###########################################################################
 429
 430     def force_move (self, files):
 431         """Forcefully move files from the current directory to the reject
 432            directory.  If any file already exists it will be moved to the
 433            morgue to make way for the new file."""
 434
 435         Cnf = self.Cnf
 436
 437         for file in files:
 438             # Skip any files which don't exist or which we don't have permission to copy.
 439             if os.access(file,os.R_OK) == 0:
 440                 continue;
 441             dest_file = os.path.join(Cnf["Dir::QueueRejectDir"], file);
 442             try:
 443                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 444             except OSError, e:
 445                 # File exists?  Let's try and move it to the morgue
 446                 if errno.errorcode[e.errno] == 'EEXIST':
 447                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueRejectDir"],file);
 448                     try:
 449                         morgue_file = utils.find_next_free(morgue_file);
 450                     except utils.tried_too_hard_exc:
 451                         # Something's either gone badly Pete Tong, or
 452                         # someone is trying to exploit us.
 453                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 454                         return;
 455                     utils.move(dest_file, morgue_file, perms=0660);
 456                     try:
 457                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 458                     except OSError, e:
 459                         # Likewise
 460                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 461                         return;
 462                 else:
 463                     raise;
 464             # If we got here, we own the destination file, so we can
 465             # safely overwrite it.
 466             utils.move(file, dest_file, 1, perms=0660);
 467
 468     ###########################################################################
 469
 470     def do_reject (self, manual = 0, reject_message = ""):
 471         # If we weren't given a manual rejection message, spawn an
 472         # editor so the user can add one in...
 473         if manual and not reject_message:
 474             temp_filename = tempfile.mktemp();
 475             fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
 476             os.close(fd);
 477             editor = os.environ.get("EDITOR","vi")
 478             answer = 'E';
 479             while answer == 'E':
 480                 os.system("%s %s" % (editor, temp_filename))
 481                 file = utils.open_file(temp_filename);
 482                 reject_message = string.join(file.readlines());
 483                 file.close();
 484                 print "Reject message:";
 485                 print utils.prefix_multi_line_string(reject_message,"  ");
 486                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 487                 answer = "XXX";
 488                 while string.find(prompt, answer) == -1:
 489                     answer = utils.our_raw_input(prompt);
 490                     m = re_default_answer.search(prompt);
 491                     if answer == "":
 492                         answer = m.group(1);
 493                     answer = string.upper(answer[:1]);
 494             os.unlink(temp_filename);
 495             if answer == 'A':
 496                 return 1;
 497             elif answer == 'Q':
 498                 sys.exit(0);
 499
 500         print "Rejecting.\n"
 501
 502         Cnf = self.Cnf;
 503         Subst = self.Subst;
 504         pkg = self.pkg;
 505
 506         reason_filename = pkg.changes_file[:-8] + ".reason";
 507         reject_filename = Cnf["Dir::QueueRejectDir"] + '/' + reason_filename;
 508
 509         # Move all the files into the reject directory
 510         reject_files = pkg.files.keys() + [pkg.changes_file];
 511         self.force_move(reject_files);
 512
 513         # If we fail here someone is probably trying to exploit the race
 514         # so let's just raise an exception ...
 515         if os.path.exists(reject_filename):
 516             os.unlink(reject_filename);
 517         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 518
 519         if not manual:
 520             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 521             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 522             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 523             os.write(fd, reject_message);
 524             os.close(fd);
 525             reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.rejected").read());
 526         else:
 527             # Build up the rejection email
 528             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 529
 530             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 531             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 532             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 533             reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.rejected").read());
 534
 535             # Write the rejection email out as the <foo>.reason file
 536             os.write(fd, reject_mail_message);
 537             os.close(fd);
 538
 539         # Send the rejection mail if appropriate
 540         if not Cnf["Dinstall::Options::No-Mail"]:
 541             utils.send_mail (reject_mail_message, "");
 542
 543         self.Logger.log(["rejected", pkg.changes_file]);
 544         return 0;
 545
 546     ################################################################################
 547
 548     # Ensure that source exists somewhere in the archive for the binary
 549     # upload being processed.
 550     #
 551     # (1) exact match                      => 1.0-3
 552     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 553     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 554
 555     def source_exists (self, package, source_version):
 556         q = self.projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
 557
 558         # Reduce the query results to a list of version numbers
 559         ql = map(lambda x: x[0], q.getresult());
 560
 561         # Try (1)
 562         if ql.count(source_version):
 563             return 1;
 564
 565         # Try (2)
 566         orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
 567         if ql.count(orig_source_version):
 568             return 1;
 569
 570         # Try (3)
 571         orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
 572         if ql.count(orig_source_version):
 573             return 1;
 574
 575         # No source found...
 576         return 0;
 577
 578     ################################################################################
 579
 580     def in_override_p (self, package, component, suite, binary_type, file):
 581         files = self.pkg.files;
 582
 583         if binary_type == "": # must be source
 584             type = "dsc";
 585         else:
 586             type = binary_type;
 587
 588         # Override suite name; used for example with proposed-updates
 589         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 590             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 591
 592         # Avoid <undef> on unknown distributions
 593         suite_id = db_access.get_suite_id(suite);
 594         if suite_id == -1:
 595             return None;
 596         component_id = db_access.get_component_id(component);
 597         type_id = db_access.get_override_type_id(type);
 598
 599         # FIXME: nasty non-US speficic hack
 600         if lower(component[:7]) == "non-us/":
 601             component = component[7:];
 602
 603         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 604                            % (package, suite_id, component_id, type_id));
 605         result = q.getresult();
 606         # If checking for a source package fall back on the binary override type
 607         if type == "dsc" and not result:
 608             type_id = db_access.get_override_type_id("deb");
 609             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 610                                % (package, suite_id, component_id, type_id));
 611             result = q.getresult();
 612
 613         # Remember the section and priority so we can check them later if appropriate
 614         if result != []:
 615             files[file]["override section"] = result[0][0];
 616             files[file]["override priority"] = result[0][1];
 617
 618         return result;
 619
 620     ################################################################################
 621
 622     def reject (self, str, prefix="Rejected: "):
 623         if str:
 624             # Unlike other rejects we add new lines first to avoid trailing
 625             # new lines when this message is passed back up to a caller.
 626             if self.reject_message:
 627                 self.reject_message = self.reject_message + "\n";
 628             self.reject_message = self.reject_message + prefix + str;
 629
 630     def check_binaries_against_db(self, file, suite):
 631         self.reject_message = "";
 632         files = self.pkg.files;
 633
 634         # Find any old binary packages
 635         q = self.projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name  FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
 636                            % (files[file]["package"], suite, files[file]["architecture"]))
 637         for oldfile in q.dictresult():
 638             files[file]["oldfiles"][suite] = oldfile;
 639             # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
 640             if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
 641                 self.reject("%s: old version (%s) >= new version (%s)." % (file, oldfile["version"], files[file]["version"]));
 642         # Check for any existing copies of the file
 643         q = self.projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
 644         if q.getresult() != []:
 645             self.reject("can not overwrite existing copy of '%s' already in the archive." % (file));
 646
 647         return self.reject_message;
 648
 649     ################################################################################
 650
 651     def check_source_against_db(self, file):
 652         """Ensure source is newer than existing source in target suites."""
 653         self.reject_message = "";
 654         changes = self.pkg.changes;
 655         dsc = self.pkg.dsc;
 656
 657         package = dsc.get("source");
 658         new_version = dsc.get("version");
 659         for suite in changes["distribution"].keys():
 660             q = self.projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
 661                                % (package, suite));
 662             ql = map(lambda x: x[0], q.getresult());
 663             for old_version in ql:
 664                 if apt_pkg.VersionCompare(new_version, old_version) != 1:
 665                     self.reject("%s: Old version `%s' >= new version `%s'." % (file, old_version, new_version));
 666         return self.reject_message;
 667
 668     ################################################################################
 669
 670     def check_dsc_against_db(self, file):
 671         self.reject_message = "";
 672         files = self.pkg.files;
 673         dsc_files = self.pkg.dsc_files;
 674         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 675         orig_tar_gz = None;
 676
 677         # Try and find all files mentioned in the .dsc.  This has
 678         # to work harder to cope with the multiple possible
 679         # locations of an .orig.tar.gz.
 680         for dsc_file in dsc_files.keys():
 681             found = None;
 682             if files.has_key(dsc_file):
 683                 actual_md5 = files[dsc_file]["md5sum"];
 684                 actual_size = int(files[dsc_file]["size"]);
 685                 found = "%s in incoming" % (dsc_file)
 686                 # Check the file does not already exist in the archive
 687                 q = self.projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
 688
 689                 # "It has not broken them.  It has fixed a
 690                 # brokenness.  Your crappy hack exploited a bug in
 691                 # the old dinstall.
 692                 #
 693                 # "(Come on!  I thought it was always obvious that
 694                 # one just doesn't release different files with
 695                 # the same name and version.)"
 696                 #                        -- ajk@ on d-devel@l.d.o
 697
 698                 if q.getresult() != []:
 699                     self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 700             elif dsc_file[-12:] == ".orig.tar.gz":
 701                 # Check in the pool
 702                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
 703                 ql = q.getresult();
 704
 705                 if ql != []:
 706                     # Unfortunately, we make get more than one
 707                     # match here if, for example, the package was
 708                     # in potato but had a -sa upload in woody.  So
 709                     # we need to choose the right one.
 710
 711                     x = ql[0]; # default to something sane in case we don't match any or have only one
 712
 713                     if len(ql) > 1:
 714                         for i in ql:
 715                             old_file = i[0] + i[1];
 716                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 717                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 718                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 719                                 x = i;
 720                             else:
 721                                 legacy_source_untouchable[i[3]] = "";
 722
 723                     old_file = x[0] + x[1];
 724                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 725                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 726                     found = old_file;
 727                     suite_type = x[2];
 728                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 729                     # See install() in katie...
 730                     self.pkg.orig_tar_id = x[3];
 731                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 732                         self.pkg.orig_tar_location = "legacy";
 733                     else:
 734                         self.pkg.orig_tar_location = x[4];
 735                 else:
 736                     # Not there? Check the queue directories...
 737
 738                     in_unchecked = os.path.join(self.Cnf["Dir::QueueUncheckedDir"],dsc_file);
 739                     # See process_it() in jennifer for explanation of this
 740                     if os.path.exists(in_unchecked):
 741                         return (self.reject_message, in_unchecked);
 742                     else:
 743                         for dir in [ "Accepted", "New", "Byhand" ]:
 744                             in_otherdir = os.path.join(self.Cnf["Dir::Queue%sDir" % (dir)],dsc_file);
 745                             if os.path.exists(in_otherdir):
 746                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 747                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 748                                 found = in_otherdir;
 749
 750                     if not found:
 751                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 752                         continue;
 753             else:
 754                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 755                 continue;
 756             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 757                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 758             if actual_size != int(dsc_files[dsc_file]["size"]):
 759                 self.reject("size for %s doesn't match %s." % (found, file));
 760
 761         return (self.reject_message, orig_tar_gz);
 762
 763     def do_query(self, q):
 764         sys.stderr.write("query: \"%s\" ... " % (q));
 765         before = time.time();
 766         r = self.projectB.query(q);
 767         time_diff = time.time()-before;
 768         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
 769         return r;