katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001, 2002, 2003, 2004  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.52 2005-01-14 14:07:17 ajt Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28
  29 ###############################################################################
  30
  31 re_isanum = re.compile (r"^\d+$");
  32 re_default_answer = re.compile(r"\[(.*)\]");
  33 re_fdnic = re.compile(r"\n\n");
  34 re_bin_only_nmu_of_mu = re.compile(r"\.\d+\.\d+$");
  35 re_bin_only_nmu_of_nmu = re.compile(r"\.\d+$");
  36
  37 ###############################################################################
  38
  39 # Convenience wrapper to carry around all the package information in
  40
  41 class Pkg:
  42     def __init__(self, **kwds):
  43         self.__dict__.update(kwds);
  44
  45     def update(self, **kwds):
  46         self.__dict__.update(kwds);
  47
  48 ###############################################################################
  49
  50 class nmu_p:
  51     # Read in the group maintainer override file
  52     def __init__ (self, Cnf):
  53         self.group_maint = {};
  54         self.Cnf = Cnf;
  55         if Cnf.get("Dinstall::GroupOverrideFilename"):
  56             filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"];
  57             file = utils.open_file(filename);
  58             for line in file.readlines():
  59                 line = utils.re_comments.sub('', line).lower().strip();
  60                 if line != "":
  61                     self.group_maint[line] = 1;
  62             file.close();
  63
  64     def is_an_nmu (self, pkg):
  65         Cnf = self.Cnf;
  66         changes = pkg.changes;
  67         dsc = pkg.dsc;
  68
  69         i = utils.fix_maintainer (dsc.get("maintainer",
  70                                           Cnf["Dinstall::MyEmailAddress"]).lower());
  71         (dsc_rfc822, dsc_rfc2047, dsc_name, dsc_email) = i;
  72         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  73         if dsc_name == changes["maintainername"].lower() and \
  74            (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name):
  75             return 0;
  76
  77         if dsc.has_key("uploaders"):
  78             uploaders = dsc["uploaders"].lower().split(",");
  79             uploadernames = {};
  80             for i in uploaders:
  81                 (rfc822, rfc2047, name, email) = utils.fix_maintainer (i.strip());
  82                 uploadernames[name] = "";
  83             if uploadernames.has_key(changes["changedbyname"].lower()):
  84                 return 0;
  85
  86         # Some group maintained packages (e.g. Debian QA) are never NMU's
  87         if self.group_maint.has_key(changes["maintaineremail"].lower()):
  88             return 0;
  89
  90         return 1;
  91
  92 ###############################################################################
  93
  94 class Katie:
  95
  96     def __init__(self, Cnf):
  97         self.Cnf = Cnf;
  98         # Read in the group-maint override file
  99         self.nmu = nmu_p(Cnf);
 100         self.accept_count = 0;
 101         self.accept_bytes = 0L;
 102         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 103                        legacy_source_untouchable = {});
 104
 105         # Initialize the substitution template mapping global
 106         Subst = self.Subst = {};
 107         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 108         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 109         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 110         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 111
 112         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 113         db_access.init(Cnf, self.projectB);
 114
 115     ###########################################################################
 116
 117     def init_vars (self):
 118         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 119             exec "self.pkg.%s.clear();" % (i);
 120         self.pkg.orig_tar_id = None;
 121         self.pkg.orig_tar_location = "";
 122         self.pkg.orig_tar_gz = None;
 123
 124     ###########################################################################
 125
 126     def update_vars (self):
 127         dump_filename = self.pkg.changes_file[:-8]+".katie";
 128         dump_file = utils.open_file(dump_filename);
 129         p = cPickle.Unpickler(dump_file);
 130         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 131             exec "self.pkg.%s.update(p.load());" % (i);
 132         for i in [ "orig_tar_id", "orig_tar_location" ]:
 133             exec "self.pkg.%s = p.load();" % (i);
 134         dump_file.close();
 135
 136     ###########################################################################
 137
 138     # This could just dump the dictionaries as is, but I'd like to avoid
 139     # this so there's some idea of what katie & lisa use from jennifer
 140
 141     def dump_vars(self, dest_dir):
 142         for i in [ "changes", "dsc", "files", "dsc_files",
 143                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 144             exec "%s = self.pkg.%s;" % (i,i);
 145         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 146         dump_file = utils.open_file(dump_filename, 'w');
 147         try:
 148             os.chmod(dump_filename, 0660);
 149         except OSError, e:
 150             if errno.errorcode[e.errno] == 'EPERM':
 151                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]);
 152                 if perms & stat.S_IROTH:
 153                     utils.fubar("%s is world readable and chmod failed." % (dump_filename));
 154             else:
 155                 raise;
 156
 157         p = cPickle.Pickler(dump_file, 1);
 158         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 159             exec "%s = {}" % i;
 160         ## files
 161         for file in files.keys():
 162             d_files[file] = {};
 163             for i in [ "package", "version", "architecture", "type", "size",
 164                        "md5sum", "component", "location id", "source package",
 165                        "source version", "maintainer", "dbtype", "files id",
 166                        "new", "section", "priority", "othercomponents",
 167                        "pool name", "original component" ]:
 168                 if files[file].has_key(i):
 169                     d_files[file][i] = files[file][i];
 170         ## changes
 171         # Mandatory changes fields
 172         for i in [ "distribution", "source", "architecture", "version",
 173                    "maintainer", "urgency", "fingerprint", "changedby822",
 174                    "changedby2047", "changedbyname", "maintainer822",
 175                    "maintainer2047", "maintainername", "maintaineremail",
 176                    "closes", "changes" ]:
 177             d_changes[i] = changes[i];
 178         # Optional changes fields
 179         for i in [ "changed-by", "filecontents", "format", "lisa note", "distribution-version" ]:
 180             if changes.has_key(i):
 181                 d_changes[i] = changes[i];
 182         ## dsc
 183         for i in [ "source", "version", "maintainer", "fingerprint",
 184                    "uploaders", "bts changelog" ]:
 185             if dsc.has_key(i):
 186                 d_dsc[i] = dsc[i];
 187         ## dsc_files
 188         for file in dsc_files.keys():
 189             d_dsc_files[file] = {};
 190             # Mandatory dsc_files fields
 191             for i in [ "size", "md5sum" ]:
 192                 d_dsc_files[file][i] = dsc_files[file][i];
 193             # Optional dsc_files fields
 194             for i in [ "files id" ]:
 195                 if dsc_files[file].has_key(i):
 196                     d_dsc_files[file][i] = dsc_files[file][i];
 197
 198         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 199                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 200             p.dump(i);
 201         dump_file.close();
 202
 203     ###########################################################################
 204
 205     # Set up the per-package template substitution mappings
 206
 207     def update_subst (self, reject_message = ""):
 208         Subst = self.Subst;
 209         changes = self.pkg.changes;
 210         # If jennifer crashed out in the right place, architecture may still be a string.
 211         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 212             changes["architecture"] = { "Unknown" : "" };
 213         # and maintainer2047 may not exist.
 214         if not changes.has_key("maintainer2047"):
 215             changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"];
 216
 217         Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys());
 218         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 219         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 220
 221         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 222         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 223             Subst["__MAINTAINER_FROM__"] = changes["changedby2047"];
 224             Subst["__MAINTAINER_TO__"] = "%s, %s" % (changes["changedby2047"],
 225                                                      changes["maintainer2047"]);
 226             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 227         else:
 228             Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"];
 229             Subst["__MAINTAINER_TO__"] = changes["maintainer2047"];
 230             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 231         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 232             Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 233
 234         # Apply any global override of the Maintainer field
 235         if self.Cnf.get("Dinstall::OverrideMaintainer"):
 236             Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 237             Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 238
 239         Subst["__REJECT_MESSAGE__"] = reject_message;
 240         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 241         Subst["__VERSION__"] = changes.get("version", "Unknown");
 242
 243     ###########################################################################
 244
 245     def build_summaries(self):
 246         changes = self.pkg.changes;
 247         files = self.pkg.files;
 248
 249         byhand = summary = new = "";
 250
 251         # changes["distribution"] may not exist in corner cases
 252         # (e.g. unreadable changes files)
 253         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 254             changes["distribution"] = {};
 255
 256         file_keys = files.keys();
 257         file_keys.sort();
 258         for file in file_keys:
 259             if files[file].has_key("byhand"):
 260                 byhand = 1
 261                 summary += file + " byhand\n"
 262             elif files[file].has_key("new"):
 263                 new = 1
 264                 summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 265                 if files[file].has_key("othercomponents"):
 266                     summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 267                 if files[file]["type"] == "deb":
 268                     summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 269             else:
 270                 files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"])
 271                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 272                 summary += file + "\n  to " + destination + "\n"
 273
 274         short_summary = summary;
 275
 276         # This is for direport's benefit...
 277         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 278
 279         if byhand or new:
 280             summary += "Changes: " + f;
 281
 282         summary += self.announce(short_summary, 0)
 283
 284         return (summary, short_summary);
 285
 286     ###########################################################################
 287
 288     def close_bugs (self, summary, action):
 289         changes = self.pkg.changes;
 290         Subst = self.Subst;
 291         Cnf = self.Cnf;
 292
 293         bugs = changes["closes"].keys();
 294
 295         if not bugs:
 296             return summary;
 297
 298         bugs.sort();
 299         if not self.nmu.is_an_nmu(self.pkg):
 300             if changes["distribution"].has_key("experimental"):
 301                 # tag bugs as fixed-in-experimental for uploads to experimental
 302                 summary += "Setting bugs to severity fixed: ";
 303                 control_message = "";
 304                 for bug in bugs:
 305                     summary += "%s " % (bug);
 306                     control_message += "tag %s + fixed-in-experimental\n" % (bug);
 307                 if action and control_message != "":
 308                     Subst["__CONTROL_MESSAGE__"] = control_message;
 309                     mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-experimental-fixed");
 310                     utils.send_mail (mail_message);
 311                 if action:
 312                     self.Logger.log(["setting bugs to fixed"]+bugs);
 313
 314
 315             else:
 316                 summary += "Closing bugs: ";
 317                 for bug in bugs:
 318                     summary += "%s " % (bug);
 319                     if action:
 320                         Subst["__BUG_NUMBER__"] = bug;
 321                         if changes["distribution"].has_key("stable"):
 322                             Subst["__STABLE_WARNING__"] = """
 323 Note that this package is not part of the released stable Debian
 324 distribution.  It may have dependencies on other unreleased software,
 325 or other instabilities.  Please take care if you wish to install it.
 326 The update will eventually make its way into the next released Debian
 327 distribution.""";
 328                         else:
 329                             Subst["__STABLE_WARNING__"] = "";
 330                             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
 331                             utils.send_mail (mail_message);
 332                 if action:
 333                     self.Logger.log(["closing bugs"]+bugs);
 334
 335         else:                     # NMU
 336             summary += "Setting bugs to severity fixed: ";
 337             control_message = "";
 338             for bug in bugs:
 339                 summary += "%s " % (bug);
 340                 control_message += "tag %s + fixed\n" % (bug);
 341             if action and control_message != "":
 342                 Subst["__CONTROL_MESSAGE__"] = control_message;
 343                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed");
 344                 utils.send_mail (mail_message);
 345             if action:
 346                 self.Logger.log(["setting bugs to fixed"]+bugs);
 347         summary += "\n";
 348         return summary;
 349
 350     ###########################################################################
 351
 352     def announce (self, short_summary, action):
 353         Subst = self.Subst;
 354         Cnf = self.Cnf;
 355         changes = self.pkg.changes;
 356
 357         # Only do announcements for source uploads with a recent dpkg-dev installed
 358         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 359             return "";
 360
 361         lists_done = {};
 362         summary = "";
 363         Subst["__SHORT_SUMMARY__"] = short_summary;
 364
 365         for dist in changes["distribution"].keys():
 366             list = Cnf.Find("Suite::%s::Announce" % (dist));
 367             if list == "" or lists_done.has_key(list):
 368                 continue;
 369             lists_done[list] = 1;
 370             summary += "Announcing to %s\n" % (list);
 371
 372             if action:
 373                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 374                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 375                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]);
 376                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce");
 377                 utils.send_mail (mail_message);
 378
 379         if Cnf.FindB("Dinstall::CloseBugs"):
 380             summary = self.close_bugs(summary, action);
 381
 382         return summary;
 383
 384     ###########################################################################
 385
 386     def accept (self, summary, short_summary):
 387         Cnf = self.Cnf;
 388         Subst = self.Subst;
 389         files = self.pkg.files;
 390         changes = self.pkg.changes;
 391         changes_file = self.pkg.changes_file;
 392         dsc = self.pkg.dsc;
 393
 394         print "Accepting."
 395         self.Logger.log(["Accepting changes",changes_file]);
 396
 397         self.dump_vars(Cnf["Dir::Queue::Accepted"]);
 398
 399         # Move all the files into the accepted directory
 400         utils.move(changes_file, Cnf["Dir::Queue::Accepted"]);
 401         file_keys = files.keys();
 402         for file in file_keys:
 403             utils.move(file, Cnf["Dir::Queue::Accepted"]);
 404             self.accept_bytes += float(files[file]["size"])
 405         self.accept_count += 1;
 406
 407         # Send accept mail, announce to lists, close bugs and check for
 408         # override disparities
 409         if not Cnf["Dinstall::Options::No-Mail"]:
 410             Subst["__SUITE__"] = "";
 411             Subst["__SUMMARY__"] = summary;
 412             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted");
 413             utils.send_mail(mail_message)
 414             self.announce(short_summary, 1)
 415
 416
 417         ## Helper stuff for DebBugs Version Tracking
 418         if Cnf.Find("Dir::Queue::BTSVersionTrack"):
 419             # ??? once queue/* is cleared on *.d.o and/or reprocessed
 420             # the conditionalization on dsc["bts changelog"] should be
 421             # dropped.
 422
 423             # Write out the version history from the changelog
 424             if changes["architecture"].has_key("source") and \
 425                dsc.has_key("bts changelog"):
 426
 427                 temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
 428                                                     dotprefix=1, perms=0644);
 429                 version_history = utils.open_file(temp_filename, 'w');
 430                 version_history.write(dsc["bts changelog"]);
 431                 version_history.close();
 432                 filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
 433                                       changes_file[:-8]+".versions");
 434                 os.rename(temp_filename, filename);
 435
 436             # Write out the binary -> source mapping.
 437             temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
 438                                                 dotprefix=1, perms=0644);
 439             debinfo = utils.open_file(temp_filename, 'w');
 440             for file in file_keys:
 441                 f = files[file];
 442                 if f["type"] == "deb":
 443                     line = " ".join([f["package"], f["version"],
 444                                      f["architecture"], f["source package"],
 445                                      f["source version"]]);
 446                     debinfo.write(line+"\n");
 447             debinfo.close();
 448             filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
 449                                   changes_file[:-8]+".debinfo");
 450             os.rename(temp_filename, filename);
 451
 452         ## Special support to enable clean auto-building of accepted packages
 453         self.projectB.query("BEGIN WORK");
 454         for suite in changes["distribution"].keys():
 455             if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
 456                 continue;
 457             suite_id = db_access.get_suite_id(suite);
 458             dest_dir = Cnf["Dir::AcceptedAutoBuild"];
 459             if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 460                 dest_dir = os.path.join(dest_dir, suite);
 461             for file in file_keys:
 462                 src = os.path.join(Cnf["Dir::Queue::Accepted"], file);
 463                 dest = os.path.join(dest_dir, file);
 464                 if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 465                     # Copy it since the original won't be readable by www-data
 466                     utils.copy(src, dest);
 467                 else:
 468                     # Create a symlink to it
 469                     os.symlink(src, dest);
 470                 # Add it to the list of packages for later processing by apt-ftparchive
 471                 self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 472             # If the .orig.tar.gz is in the pool, create a symlink to
 473             # it (if one doesn't already exist)
 474             if self.pkg.orig_tar_id:
 475                 # Determine the .orig.tar.gz file name
 476                 for dsc_file in self.pkg.dsc_files.keys():
 477                     if dsc_file.endswith(".orig.tar.gz"):
 478                         filename = dsc_file;
 479                 dest = os.path.join(dest_dir, filename);
 480                 # If it doesn't exist, create a symlink
 481                 if not os.path.exists(dest):
 482                     # Find the .orig.tar.gz in the pool
 483                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 484                     ql = q.getresult();
 485                     if not ql:
 486                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 487                     src = os.path.join(ql[0][0], ql[0][1]);
 488                     os.symlink(src, dest);
 489                     # Add it to the list of packages for later processing by apt-ftparchive
 490                     self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 491                 # if it does, update things to ensure it's not removed prematurely
 492                 else:
 493                     self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id));
 494
 495         self.projectB.query("COMMIT WORK");
 496
 497     ###########################################################################
 498
 499     def check_override (self):
 500         Subst = self.Subst;
 501         changes = self.pkg.changes;
 502         files = self.pkg.files;
 503         Cnf = self.Cnf;
 504
 505         # Abandon the check if:
 506         #  a) it's a non-sourceful upload
 507         #  b) override disparity checks have been disabled
 508         #  c) we're not sending mail
 509         if not changes["architecture"].has_key("source") or \
 510            not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 511            Cnf["Dinstall::Options::No-Mail"]:
 512             return;
 513
 514         summary = "";
 515         file_keys = files.keys();
 516         file_keys.sort();
 517         for file in file_keys:
 518             if not files[file].has_key("new") and files[file]["type"] == "deb":
 519                 section = files[file]["section"];
 520                 override_section = files[file]["override section"];
 521                 if section.lower() != override_section.lower() and section != "-":
 522                     # Ignore this; it's a common mistake and not worth whining about
 523                     if section.lower() == "non-us/main" and override_section.lower() == "non-us":
 524                         continue;
 525                     summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section);
 526                 priority = files[file]["priority"];
 527                 override_priority = files[file]["override priority"];
 528                 if priority != override_priority and priority != "-":
 529                     summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority);
 530
 531         if summary == "":
 532             return;
 533
 534         Subst["__SUMMARY__"] = summary;
 535         mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity");
 536         utils.send_mail(mail_message);
 537
 538     ###########################################################################
 539
 540     def force_reject (self, files):
 541         """Forcefully move files from the current directory to the
 542            reject directory.  If any file already exists in the reject
 543            directory it will be moved to the morgue to make way for
 544            the new file."""
 545
 546         Cnf = self.Cnf
 547
 548         for file in files:
 549             # Skip any files which don't exist or which we don't have permission to copy.
 550             if os.access(file,os.R_OK) == 0:
 551                 continue;
 552             dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file);
 553             try:
 554                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 555             except OSError, e:
 556                 # File exists?  Let's try and move it to the morgue
 557                 if errno.errorcode[e.errno] == 'EEXIST':
 558                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file);
 559                     try:
 560                         morgue_file = utils.find_next_free(morgue_file);
 561                     except utils.tried_too_hard_exc:
 562                         # Something's either gone badly Pete Tong, or
 563                         # someone is trying to exploit us.
 564                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 565                         return;
 566                     utils.move(dest_file, morgue_file, perms=0660);
 567                     try:
 568                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 569                     except OSError, e:
 570                         # Likewise
 571                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 572                         return;
 573                 else:
 574                     raise;
 575             # If we got here, we own the destination file, so we can
 576             # safely overwrite it.
 577             utils.move(file, dest_file, 1, perms=0660);
 578
 579     ###########################################################################
 580
 581     def do_reject (self, manual = 0, reject_message = ""):
 582         # If we weren't given a manual rejection message, spawn an
 583         # editor so the user can add one in...
 584         if manual and not reject_message:
 585             temp_filename = utils.temp_filename();
 586             editor = os.environ.get("EDITOR","vi")
 587             answer = 'E';
 588             while answer == 'E':
 589                 os.system("%s %s" % (editor, temp_filename))
 590                 file = utils.open_file(temp_filename);
 591                 reject_message = "".join(file.readlines());
 592                 file.close();
 593                 print "Reject message:";
 594                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1);
 595                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 596                 answer = "XXX";
 597                 while prompt.find(answer) == -1:
 598                     answer = utils.our_raw_input(prompt);
 599                     m = re_default_answer.search(prompt);
 600                     if answer == "":
 601                         answer = m.group(1);
 602                     answer = answer[:1].upper();
 603             os.unlink(temp_filename);
 604             if answer == 'A':
 605                 return 1;
 606             elif answer == 'Q':
 607                 sys.exit(0);
 608
 609         print "Rejecting.\n"
 610
 611         Cnf = self.Cnf;
 612         Subst = self.Subst;
 613         pkg = self.pkg;
 614
 615         reason_filename = pkg.changes_file[:-8] + ".reason";
 616         reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
 617
 618         # Move all the files into the reject directory
 619         reject_files = pkg.files.keys() + [pkg.changes_file];
 620         self.force_reject(reject_files);
 621
 622         # If we fail here someone is probably trying to exploit the race
 623         # so let's just raise an exception ...
 624         if os.path.exists(reason_filename):
 625             os.unlink(reason_filename);
 626         reason_file = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 627
 628         if not manual:
 629             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 630             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 631             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 632             os.write(reason_file, reject_message);
 633             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 634         else:
 635             # Build up the rejection email
 636             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 637
 638             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 639             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 640             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 641             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 642             # Write the rejection email out as the <foo>.reason file
 643             os.write(reason_file, reject_mail_message);
 644
 645         os.close(reason_file);
 646
 647         # Send the rejection mail if appropriate
 648         if not Cnf["Dinstall::Options::No-Mail"]:
 649             utils.send_mail(reject_mail_message);
 650
 651         self.Logger.log(["rejected", pkg.changes_file]);
 652         return 0;
 653
 654     ################################################################################
 655
 656     # Ensure that source exists somewhere in the archive for the binary
 657     # upload being processed.
 658     #
 659     # (1) exact match                      => 1.0-3
 660     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 661     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 662
 663     def source_exists (self, package, source_version, suites = ["any"]):
 664         okay = 1
 665         for suite in suites:
 666             if suite == "any":
 667                 que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
 668                     (package)
 669             else:
 670                 # source must exist in suite X, or in some other suite that's
 671                 # mapped to X, recursively... silent-maps are counted too,
 672                 # unreleased-maps aren't.
 673                 maps = self.Cnf.ValueList("SuiteMappings")[:]
 674                 maps.reverse()
 675                 maps = [ m.split() for m in maps ]
 676                 maps = [ (x[1], x[2]) for x in maps
 677                                 if x[0] == "map" or x[0] == "silent-map" ]
 678                 s = [suite]
 679                 for x in maps:
 680                         if x[1] in s and x[0] not in s:
 681                                 s.append(x[0])
 682
 683                 que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR "));
 684             q = self.projectB.query(que)
 685
 686             # Reduce the query results to a list of version numbers
 687             ql = map(lambda x: x[0], q.getresult());
 688
 689             # Try (1)
 690             if source_version in ql:
 691                 continue
 692
 693             # Try (2)
 694             orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version)
 695             if orig_source_version in ql:
 696                 continue
 697
 698             # Try (3)
 699             orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version)
 700             if orig_source_version in ql:
 701                 continue
 702
 703             # No source found...
 704             okay = 0
 705         return okay
 706
 707     ################################################################################
 708
 709     def in_override_p (self, package, component, suite, binary_type, file):
 710         files = self.pkg.files;
 711
 712         if binary_type == "": # must be source
 713             type = "dsc";
 714         else:
 715             type = binary_type;
 716
 717         # Override suite name; used for example with proposed-updates
 718         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 719             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 720
 721         # Avoid <undef> on unknown distributions
 722         suite_id = db_access.get_suite_id(suite);
 723         if suite_id == -1:
 724             return None;
 725         component_id = db_access.get_component_id(component);
 726         type_id = db_access.get_override_type_id(type);
 727
 728         # FIXME: nasty non-US speficic hack
 729         if component.lower().startswith("non-us/"):
 730             component = component[7:];
 731
 732         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 733                            % (package, suite_id, component_id, type_id));
 734         result = q.getresult();
 735         # If checking for a source package fall back on the binary override type
 736         if type == "dsc" and not result:
 737             deb_type_id = db_access.get_override_type_id("deb");
 738             udeb_type_id = db_access.get_override_type_id("udeb");
 739             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
 740                                % (package, suite_id, component_id, deb_type_id, udeb_type_id));
 741             result = q.getresult();
 742
 743         # Remember the section and priority so we can check them later if appropriate
 744         if result:
 745             files[file]["override section"] = result[0][0];
 746             files[file]["override priority"] = result[0][1];
 747
 748         return result;
 749
 750     ################################################################################
 751
 752     def reject (self, str, prefix="Rejected: "):
 753         if str:
 754             # Unlike other rejects we add new lines first to avoid trailing
 755             # new lines when this message is passed back up to a caller.
 756             if self.reject_message:
 757                 self.reject_message += "\n";
 758             self.reject_message += prefix + str;
 759
 760     ################################################################################
 761
 762     def get_anyversion(self, query_result, suite):
 763         anyversion=None
 764         anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
 765         for (v, s) in query_result:
 766             if s in [ string.lower(x) for x in anysuite ]:
 767                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
 768                     anyversion=v
 769         return anyversion
 770
 771     ################################################################################
 772
 773     def cross_suite_version_check(self, query_result, file, new_version):
 774         """Ensure versions are newer than existing packages in target
 775         suites and that cross-suite version checking rules as
 776         set out in the conf file are satisfied."""
 777
 778         # Check versions for each target suite
 779         for target_suite in self.pkg.changes["distribution"].keys():
 780             must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
 781             must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
 782             # Enforce "must be newer than target suite" even if conffile omits it
 783             if target_suite not in must_be_newer_than:
 784                 must_be_newer_than.append(target_suite);
 785             for entry in query_result:
 786                 existent_version = entry[0];
 787                 suite = entry[1];
 788                 if suite in must_be_newer_than and \
 789                    apt_pkg.VersionCompare(new_version, existent_version) < 1:
 790                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 791                 if suite in must_be_older_than and \
 792                    apt_pkg.VersionCompare(new_version, existent_version) > -1:
 793                     ch = self.pkg.changes
 794                     cansave = 0
 795                     if ch.get('distribution-version', {}).has_key(suite):
 796                         # we really use the other suite, ignoring the conflicting one ...
 797                         addsuite = ch["distribution-version"][suite]
 798
 799                         add_version = self.get_anyversion(query_result, addsuite)
 800                         target_version = self.get_anyversion(query_result, target_suite)
 801
 802                         if not add_version:
 803                             # not add_version can only happen if we map to a suite
 804                             # that doesn't enhance the suite we're propup'ing from.
 805                             # so "propup-ver x a b c; map a d" is a problem only if
 806                             # d doesn't enhance a.
 807                             #
 808                             # i think we could always propagate in this case, rather
 809                             # than complaining. either way, this isn't a REJECT issue
 810                             #
 811                             # And - we really should complain to the dorks who configured dak
 812                             self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ")
 813                             self.pkg.changes["distribution"][addsuite] = 1
 814                             cansave = 1
 815                         elif not target_version:
 816                             # not targets_version is true when the package is NEW
 817                             # we could just stick with the "...old version..." REJECT
 818                             # for this, I think.
 819                             self.reject("Won't propogate NEW packages.")
 820                         elif apt_pkg.VersionCompare(new_version, add_version) < 0:
 821                             # propogation would be redundant. no need to reject though.
 822                             #self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ");
 823                             self.reject("foo", "Warning: ")
 824                             cansave = 1
 825                         elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \
 826                              apt_pkg.VersionCompare(add_version, target_version) == 0:
 827                             # propogate!!
 828                             self.pkg.changes["distribution"][addsuite] = 1
 829                             cansave = 1
 830
 831                     if not cansave:
 832                         self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 833
 834     ################################################################################
 835
 836     def check_binary_against_db(self, file):
 837         self.reject_message = "";
 838         files = self.pkg.files;
 839
 840         # Ensure version is sane
 841         q = self.projectB.query("""
 842 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
 843                                      architecture a
 844  WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
 845    AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
 846                                 % (files[file]["package"],
 847                                    files[file]["architecture"]));
 848         self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
 849
 850         # Check for any existing copies of the file
 851         q = self.projectB.query("""
 852 SELECT b.id FROM binaries b, architecture a
 853  WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
 854    AND a.id = b.architecture"""
 855                                 % (files[file]["package"],
 856                                    files[file]["version"],
 857                                    files[file]["architecture"]))
 858         if q.getresult():
 859             self.reject("%s: can not overwrite existing copy already in the archive." % (file));
 860
 861         return self.reject_message;
 862
 863     ################################################################################
 864
 865     def check_source_against_db(self, file):
 866         self.reject_message = "";
 867         dsc = self.pkg.dsc;
 868
 869         # Ensure version is sane
 870         q = self.projectB.query("""
 871 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
 872  WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
 873         self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
 874
 875         return self.reject_message;
 876
 877     ################################################################################
 878
 879     # **WARNING**
 880     # NB: this function can remove entries from the 'files' index [if
 881     # the .orig.tar.gz is a duplicate of the one in the archive]; if
 882     # you're iterating over 'files' and call this function as part of
 883     # the loop, be sure to add a check to the top of the loop to
 884     # ensure you haven't just tried to derefernece the deleted entry.
 885     # **WARNING**
 886
 887     def check_dsc_against_db(self, file):
 888         self.reject_message = "";
 889         files = self.pkg.files;
 890         dsc_files = self.pkg.dsc_files;
 891         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 892         self.pkg.orig_tar_gz = None;
 893
 894         # Try and find all files mentioned in the .dsc.  This has
 895         # to work harder to cope with the multiple possible
 896         # locations of an .orig.tar.gz.
 897         for dsc_file in dsc_files.keys():
 898             found = None;
 899             if files.has_key(dsc_file):
 900                 actual_md5 = files[dsc_file]["md5sum"];
 901                 actual_size = int(files[dsc_file]["size"]);
 902                 found = "%s in incoming" % (dsc_file)
 903                 # Check the file does not already exist in the archive
 904                 q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file));
 905                 ql = q.getresult();
 906                 # Strip out anything that isn't '%s' or '/%s$'
 907                 for i in ql:
 908                     if i[3] != dsc_file and i[3][-(len(dsc_file)+1):] != '/'+dsc_file:
 909                         ql.remove(i);
 910
 911                 # "[katie] has not broken them.  [katie] has fixed a
 912                 # brokenness.  Your crappy hack exploited a bug in
 913                 # the old dinstall.
 914                 #
 915                 # "(Come on!  I thought it was always obvious that
 916                 # one just doesn't release different files with
 917                 # the same name and version.)"
 918                 #                        -- ajk@ on d-devel@l.d.o
 919
 920                 if ql:
 921                     # Ignore exact matches for .orig.tar.gz
 922                     match = 0;
 923                     if dsc_file.endswith(".orig.tar.gz"):
 924                         for i in ql:
 925                             if files.has_key(dsc_file) and \
 926                                int(files[dsc_file]["size"]) == int(i[0]) and \
 927                                files[dsc_file]["md5sum"] == i[1]:
 928                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
 929                                 del files[dsc_file];
 930                                 self.pkg.orig_tar_gz = i[2] + i[3];
 931                                 match = 1;
 932
 933                     if not match:
 934                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 935             elif dsc_file.endswith(".orig.tar.gz"):
 936                 # Check in the pool
 937                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file));
 938                 ql = q.getresult();
 939                 # Strip out anything that isn't '%s' or '/%s$'
 940                 for i in ql:
 941                     if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
 942                         ql.remove(i);
 943
 944                 if ql:
 945                     # Unfortunately, we may get more than one match here if,
 946                     # for example, the package was in potato but had an -sa
 947                     # upload in woody.  So we need to choose the right one.
 948
 949                     x = ql[0]; # default to something sane in case we don't match any or have only one
 950
 951                     if len(ql) > 1:
 952                         for i in ql:
 953                             old_file = i[0] + i[1];
 954                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 955                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 956                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 957                                 x = i;
 958                             else:
 959                                 legacy_source_untouchable[i[3]] = "";
 960
 961                     old_file = x[0] + x[1];
 962                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 963                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 964                     found = old_file;
 965                     suite_type = x[2];
 966                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 967                     # See install() in katie...
 968                     self.pkg.orig_tar_id = x[3];
 969                     self.pkg.orig_tar_gz = old_file;
 970                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 971                         self.pkg.orig_tar_location = "legacy";
 972                     else:
 973                         self.pkg.orig_tar_location = x[4];
 974                 else:
 975                     # Not there? Check the queue directories...
 976
 977                     in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file);
 978                     # See process_it() in jennifer for explanation of this
 979                     if os.path.exists(in_unchecked):
 980                         return (self.reject_message, in_unchecked);
 981                     else:
 982                         for dir in [ "Accepted", "New", "Byhand" ]:
 983                             in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file);
 984                             if os.path.exists(in_otherdir):
 985                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 986                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 987                                 found = in_otherdir;
 988                                 self.pkg.orig_tar_gz = in_otherdir;
 989
 990                     if not found:
 991                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 992                         self.pkg.orig_tar_gz = -1;
 993                         continue;
 994             else:
 995                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 996                 continue;
 997             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 998                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 999             if actual_size != int(dsc_files[dsc_file]["size"]):
1000                 self.reject("size for %s doesn't match %s." % (found, file));
1001
1002         return (self.reject_message, None);
1003
1004     def do_query(self, q):
1005         sys.stderr.write("query: \"%s\" ... " % (q));
1006         before = time.time();
1007         r = self.projectB.query(q);
1008         time_diff = time.time()-before;
1009         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
1010         return r;