katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001, 2002, 2003  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.37 2003-08-09 02:49:35 rdonald Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28
  29 ###############################################################################
  30
  31 re_isanum = re.compile (r"^\d+$");
  32 re_default_answer = re.compile(r"\[(.*)\]");
  33 re_fdnic = re.compile("\n\n");
  34 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
  35 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
  36
  37 ###############################################################################
  38
  39 # Convenience wrapper to carry around all the package information in
  40
  41 class Pkg:
  42     def __init__(self, **kwds):
  43         self.__dict__.update(kwds);
  44
  45     def update(self, **kwds):
  46         self.__dict__.update(kwds);
  47
  48 ###############################################################################
  49
  50 class nmu_p:
  51     # Read in the group maintainer override file
  52     def __init__ (self, Cnf):
  53         self.group_maint = {};
  54         self.Cnf = Cnf;
  55         if Cnf.get("Dinstall::GroupOverrideFilename"):
  56             filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"];
  57             file = utils.open_file(filename);
  58             for line in file.readlines():
  59                 line = utils.re_comments.sub('', line).lower().strip();
  60                 if line != "":
  61                     self.group_maint[line] = 1;
  62             file.close();
  63
  64     def is_an_nmu (self, pkg):
  65         Cnf = self.Cnf;
  66         changes = pkg.changes;
  67         dsc = pkg.dsc;
  68
  69         (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]).lower());
  70         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  71         if dsc_name == changes["maintainername"].lower() and \
  72            (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name):
  73             return 0;
  74
  75         if dsc.has_key("uploaders"):
  76             uploaders = dsc["uploaders"].lower().split(",");
  77             uploadernames = {};
  78             for i in uploaders:
  79                 (rfc822, name, email) = utils.fix_maintainer (i.strip());
  80                 uploadernames[name] = "";
  81             if uploadernames.has_key(changes["changedbyname"].lower()):
  82                 return 0;
  83
  84         # Some group maintained packages (e.g. Debian QA) are never NMU's
  85         if self.group_maint.has_key(changes["maintaineremail"].lower()):
  86             return 0;
  87
  88         return 1;
  89
  90 ###############################################################################
  91
  92 class Katie:
  93
  94     def __init__(self, Cnf):
  95         self.Cnf = Cnf;
  96         self.values = {};
  97         # Read in the group-maint override file
  98         self.nmu = nmu_p(Cnf);
  99         self.accept_count = 0;
 100         self.accept_bytes = 0L;
 101         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 102                        legacy_source_untouchable = {});
 103
 104         # Initialize the substitution template mapping global
 105         Subst = self.Subst = {};
 106         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 107         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 108         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 109         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 110
 111         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 112         db_access.init(Cnf, self.projectB);
 113
 114     ###########################################################################
 115
 116     def init_vars (self):
 117         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 118             exec "self.pkg.%s.clear();" % (i);
 119         self.pkg.orig_tar_id = None;
 120         self.pkg.orig_tar_location = "";
 121
 122     ###########################################################################
 123
 124     def update_vars (self):
 125         dump_filename = self.pkg.changes_file[:-8]+".katie";
 126         dump_file = utils.open_file(dump_filename);
 127         p = cPickle.Unpickler(dump_file);
 128         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 129             exec "self.pkg.%s.update(p.load());" % (i);
 130         for i in [ "orig_tar_id", "orig_tar_location" ]:
 131             exec "self.pkg.%s = p.load();" % (i);
 132         dump_file.close();
 133
 134     ###########################################################################
 135
 136     # This could just dump the dictionaries as is, but I'd like to avoid
 137     # this so there's some idea of what katie & lisa use from jennifer
 138
 139     def dump_vars(self, dest_dir):
 140         for i in [ "changes", "dsc", "files", "dsc_files",
 141                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 142             exec "%s = self.pkg.%s;" % (i,i);
 143         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 144         dump_file = utils.open_file(dump_filename, 'w');
 145         try:
 146             os.chmod(dump_filename, 0660);
 147         except OSError, e:
 148             if errno.errorcode[e.errno] == 'EPERM':
 149                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]);
 150                 if perms & stat.S_IROTH:
 151                     utils.fubar("%s is world readable and chmod failed." % (dump_filename));
 152             else:
 153                 raise;
 154
 155         p = cPickle.Pickler(dump_file, 1);
 156         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 157             exec "%s = {}" % i;
 158         ## files
 159         for file in files.keys():
 160             d_files[file] = {};
 161             for i in [ "package", "version", "architecture", "type", "size",
 162                        "md5sum", "component", "location id", "source package",
 163                        "source version", "maintainer", "dbtype", "files id",
 164                        "new", "section", "priority", "othercomponents",
 165                        "pool name", "original component" ]:
 166                 if files[file].has_key(i):
 167                     d_files[file][i] = files[file][i];
 168         ## changes
 169         # Mandatory changes fields
 170         for i in [ "distribution", "source", "architecture", "version", "maintainer",
 171                    "urgency", "fingerprint", "changedby822", "changedbyname",
 172                    "maintainername", "maintaineremail", "closes" ]:
 173             d_changes[i] = changes[i];
 174         # Optional changes fields
 175         # FIXME: changes should be mandatory
 176         for i in [ "changed-by", "maintainer822", "filecontents", "format",
 177                    "changes", "lisa note" ]:
 178             if changes.has_key(i):
 179                 d_changes[i] = changes[i];
 180         ## dsc
 181         for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
 182             if dsc.has_key(i):
 183                 d_dsc[i] = dsc[i];
 184         ## dsc_files
 185         for file in dsc_files.keys():
 186             d_dsc_files[file] = {};
 187             # Mandatory dsc_files fields
 188             for i in [ "size", "md5sum" ]:
 189                 d_dsc_files[file][i] = dsc_files[file][i];
 190             # Optional dsc_files fields
 191             for i in [ "files id" ]:
 192                 if dsc_files[file].has_key(i):
 193                     d_dsc_files[file][i] = dsc_files[file][i];
 194
 195         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 196                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 197             p.dump(i);
 198         dump_file.close();
 199
 200     ###########################################################################
 201
 202     # Set up the per-package template substitution mappings
 203
 204     def update_subst (self, reject_message = ""):
 205         Subst = self.Subst;
 206         changes = self.pkg.changes;
 207         # If jennifer crashed out in the right place, architecture may still be a string.
 208         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 209             changes["architecture"] = { "Unknown" : "" };
 210         # and maintainer822 may not exist.
 211         if not changes.has_key("maintainer822"):
 212             changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
 213
 214         Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys());
 215         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 216         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 217
 218         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 219         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 220             Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
 221             Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
 222             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 223         else:
 224             Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
 225             Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
 226             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 227         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 228             Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 229
 230         # Apply any global override of the Maintainer field
 231         if self.Cnf.get("Dinstall::OverrideMaintainer"):
 232             Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 233             Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 234
 235         Subst["__REJECT_MESSAGE__"] = reject_message;
 236         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 237         Subst["__VERSION__"] = changes.get("version", "Unknown");
 238
 239     ###########################################################################
 240
 241     def build_summaries(self):
 242         changes = self.pkg.changes;
 243         files = self.pkg.files;
 244
 245         byhand = summary = new = "";
 246
 247         # changes["distribution"] may not exist in corner cases
 248         # (e.g. unreadable changes files)
 249         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 250             changes["distribution"] = {};
 251
 252         file_keys = files.keys();
 253         file_keys.sort();
 254         for file in file_keys:
 255             if files[file].has_key("byhand"):
 256                 byhand = 1
 257                 summary += file + " byhand\n"
 258             elif files[file].has_key("new"):
 259                 new = 1
 260                 summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 261                 if files[file].has_key("othercomponents"):
 262                     summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 263                 if files[file]["type"] == "deb":
 264                     summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 265             else:
 266                 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
 267                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 268                 summary += file + "\n  to " + destination + "\n"
 269
 270         short_summary = summary;
 271
 272         # This is for direport's benefit...
 273         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 274
 275         if byhand or new:
 276             summary += "Changes: " + f;
 277
 278         summary += self.announce(short_summary, 0)
 279
 280         return (summary, short_summary);
 281
 282     ###########################################################################
 283
 284     def close_bugs (self, summary, action):
 285         changes = self.pkg.changes;
 286         Subst = self.Subst;
 287         Cnf = self.Cnf;
 288
 289         bugs = changes["closes"].keys();
 290
 291         if not bugs:
 292             return summary;
 293
 294         bugs.sort();
 295         if not self.nmu.is_an_nmu(self.pkg):
 296             if changes["distribution"].has_key("experimental"):
 297                 # tag bugs as fixed-in-experimental for uploads to experimental
 298                 summary += "Setting bugs to severity fixed: ";
 299                 control_message = "";
 300                 for bug in bugs:
 301                     summary += "%s " % (bug);
 302                     control_message += "tag %s + fixed-in-experimental\n" % (bug);
 303                 if action and control_message != "":
 304                     Subst["__CONTROL_MESSAGE__"] = control_message;
 305                     mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-experimental-fixed");
 306                     utils.send_mail (mail_message);
 307                 if action:
 308                     self.Logger.log(["setting bugs to fixed"]+bugs);
 309
 310
 311             else:
 312                 summary += "Closing bugs: ";
 313                 for bug in bugs:
 314                     summary += "%s " % (bug);
 315                     if action:
 316                         Subst["__BUG_NUMBER__"] = bug;
 317                         if changes["distribution"].has_key("stable"):
 318                             Subst["__STABLE_WARNING__"] = """
 319 Note that this package is not part of the released stable Debian
 320 distribution.  It may have dependencies on other unreleased software,
 321 or other instabilities.  Please take care if you wish to install it.
 322 The update will eventually make its way into the next released Debian
 323 distribution.""";
 324                         else:
 325                             Subst["__STABLE_WARNING__"] = "";
 326                             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
 327                             utils.send_mail (mail_message);
 328                         if action:
 329                             self.Logger.log(["closing bugs"]+bugs);
 330
 331         else:                     # NMU
 332             summary += "Setting bugs to severity fixed: ";
 333             control_message = "";
 334             for bug in bugs:
 335                 summary += "%s " % (bug);
 336                 control_message += "tag %s + fixed\n" % (bug);
 337             if action and control_message != "":
 338                 Subst["__CONTROL_MESSAGE__"] = control_message;
 339                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed");
 340                 utils.send_mail (mail_message);
 341             if action:
 342                 self.Logger.log(["setting bugs to fixed"]+bugs);
 343         summary += "\n";
 344         return summary;
 345
 346     ###########################################################################
 347
 348     def announce (self, short_summary, action):
 349         Subst = self.Subst;
 350         Cnf = self.Cnf;
 351         changes = self.pkg.changes;
 352
 353         # Only do announcements for source uploads with a recent dpkg-dev installed
 354         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 355             return "";
 356
 357         lists_done = {};
 358         summary = "";
 359         Subst["__SHORT_SUMMARY__"] = short_summary;
 360
 361         for dist in changes["distribution"].keys():
 362             list = Cnf.Find("Suite::%s::Announce" % (dist));
 363             if list == "" or lists_done.has_key(list):
 364                 continue;
 365             lists_done[list] = 1;
 366             summary += "Announcing to %s\n" % (list);
 367
 368             if action:
 369                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 370                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 371                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]);
 372                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce");
 373                 utils.send_mail (mail_message);
 374
 375         if Cnf.FindB("Dinstall::CloseBugs"):
 376             summary = self.close_bugs(summary, action);
 377
 378         return summary;
 379
 380     ###########################################################################
 381
 382     def accept (self, summary, short_summary):
 383         Cnf = self.Cnf;
 384         Subst = self.Subst;
 385         files = self.pkg.files;
 386
 387         print "Accepting."
 388         self.Logger.log(["Accepting changes",self.pkg.changes_file]);
 389
 390         self.dump_vars(Cnf["Dir::Queue::Accepted"]);
 391
 392         # Move all the files into the accepted directory
 393         utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
 394         file_keys = files.keys();
 395         for file in file_keys:
 396             utils.move(file, Cnf["Dir::Queue::Accepted"]);
 397             self.accept_bytes += float(files[file]["size"])
 398         self.accept_count += 1;
 399
 400         # Send accept mail, announce to lists, close bugs and check for
 401         # override disparities
 402         if not Cnf["Dinstall::Options::No-Mail"]:
 403             Subst["__SUITE__"] = "";
 404             Subst["__SUMMARY__"] = summary;
 405             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted");
 406             utils.send_mail(mail_message)
 407             self.announce(short_summary, 1)
 408
 409         # Special support to enable clean auto-building of accepted packages
 410         self.projectB.query("BEGIN WORK");
 411         for suite in self.pkg.changes["distribution"].keys():
 412             if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
 413                 continue;
 414             suite_id = db_access.get_suite_id(suite);
 415             dest_dir = Cnf["Dir::AcceptedAutoBuild"];
 416             if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 417                 dest_dir = os.path.join(dest_dir, suite);
 418             for file in file_keys:
 419                 src = os.path.join(Cnf["Dir::Queue::Accepted"], file);
 420                 dest = os.path.join(dest_dir, file);
 421                 if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 422                     # Copy it since the original won't be readable by www-data
 423                     utils.copy(src, dest);
 424                 else:
 425                     # Create a symlink to it
 426                     os.symlink(src, dest);
 427                 # Add it to the list of packages for later processing by apt-ftparchive
 428                 self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 429             # If the .orig.tar.gz is in the pool, create a symlink to
 430             # it (if one doesn't already exist)
 431             if self.pkg.orig_tar_id:
 432                 # Determine the .orig.tar.gz file name
 433                 for dsc_file in self.pkg.dsc_files.keys():
 434                     if dsc_file.endswith(".orig.tar.gz"):
 435                         filename = dsc_file;
 436                 dest = os.path.join(dest_dir, filename);
 437                 # If it doesn't exist, create a symlink
 438                 if not os.path.exists(dest):
 439                     # Find the .orig.tar.gz in the pool
 440                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 441                     ql = q.getresult();
 442                     if not ql:
 443                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 444                     src = os.path.join(ql[0][0], ql[0][1]);
 445                     os.symlink(src, dest);
 446                     # Add it to the list of packages for later processing by apt-ftparchive
 447                     self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 448                 # if it does, update things to ensure it's not removed prematurely
 449                 else:
 450                     self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id));
 451
 452         self.projectB.query("COMMIT WORK");
 453
 454     ###########################################################################
 455
 456     def check_override (self):
 457         Subst = self.Subst;
 458         changes = self.pkg.changes;
 459         files = self.pkg.files;
 460         Cnf = self.Cnf;
 461
 462         # Abandon the check if:
 463         #  a) it's a non-sourceful upload
 464         #  b) override disparity checks have been disabled
 465         #  c) we're not sending mail
 466         if not changes["architecture"].has_key("source") or \
 467            not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 468            Cnf["Dinstall::Options::No-Mail"]:
 469             return;
 470
 471         summary = "";
 472         for file in files.keys():
 473             if not files[file].has_key("new") and files[file]["type"] == "deb":
 474                 section = files[file]["section"];
 475                 override_section = files[file]["override section"];
 476                 if section.lower() != override_section.lower() and section != "-":
 477                     # Ignore this; it's a common mistake and not worth whining about
 478                     if section.lower() == "non-us/main" and override_section.lower() == "non-us":
 479                         continue;
 480                     summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section);
 481                 priority = files[file]["priority"];
 482                 override_priority = files[file]["override priority"];
 483                 if priority != override_priority and priority != "-":
 484                     summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority);
 485
 486         if summary == "":
 487             return;
 488
 489         Subst["__SUMMARY__"] = summary;
 490         mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity");
 491         utils.send_mail(mail_message);
 492
 493     ###########################################################################
 494
 495     def force_reject (self, files):
 496         """Forcefully move files from the current directory to the
 497            reject directory.  If any file already exists in the reject
 498            directory it will be moved to the morgue to make way for
 499            the new file."""
 500
 501         Cnf = self.Cnf
 502
 503         for file in files:
 504             # Skip any files which don't exist or which we don't have permission to copy.
 505             if os.access(file,os.R_OK) == 0:
 506                 continue;
 507             dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file);
 508             try:
 509                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 510             except OSError, e:
 511                 # File exists?  Let's try and move it to the morgue
 512                 if errno.errorcode[e.errno] == 'EEXIST':
 513                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file);
 514                     try:
 515                         morgue_file = utils.find_next_free(morgue_file);
 516                     except utils.tried_too_hard_exc:
 517                         # Something's either gone badly Pete Tong, or
 518                         # someone is trying to exploit us.
 519                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 520                         return;
 521                     utils.move(dest_file, morgue_file, perms=0660);
 522                     try:
 523                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 524                     except OSError, e:
 525                         # Likewise
 526                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 527                         return;
 528                 else:
 529                     raise;
 530             # If we got here, we own the destination file, so we can
 531             # safely overwrite it.
 532             utils.move(file, dest_file, 1, perms=0660);
 533
 534     ###########################################################################
 535
 536     def do_reject (self, manual = 0, reject_message = ""):
 537         # If we weren't given a manual rejection message, spawn an
 538         # editor so the user can add one in...
 539         if manual and not reject_message:
 540             temp_filename = tempfile.mktemp();
 541             fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
 542             os.close(fd);
 543             editor = os.environ.get("EDITOR","vi")
 544             answer = 'E';
 545             while answer == 'E':
 546                 os.system("%s %s" % (editor, temp_filename))
 547                 file = utils.open_file(temp_filename);
 548                 reject_message = "".join(file.readlines());
 549                 file.close();
 550                 print "Reject message:";
 551                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1);
 552                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 553                 answer = "XXX";
 554                 while prompt.find(answer) == -1:
 555                     answer = utils.our_raw_input(prompt);
 556                     m = re_default_answer.search(prompt);
 557                     if answer == "":
 558                         answer = m.group(1);
 559                     answer = answer[:1].upper();
 560             os.unlink(temp_filename);
 561             if answer == 'A':
 562                 return 1;
 563             elif answer == 'Q':
 564                 sys.exit(0);
 565
 566         print "Rejecting.\n"
 567
 568         Cnf = self.Cnf;
 569         Subst = self.Subst;
 570         pkg = self.pkg;
 571
 572         reason_filename = pkg.changes_file[:-8] + ".reason";
 573         reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
 574
 575         # Move all the files into the reject directory
 576         reject_files = pkg.files.keys() + [pkg.changes_file];
 577         self.force_reject(reject_files);
 578
 579         # If we fail here someone is probably trying to exploit the race
 580         # so let's just raise an exception ...
 581         if os.path.exists(reject_filename):
 582             os.unlink(reject_filename);
 583         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 584
 585         if not manual:
 586             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 587             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 588             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 589             os.write(fd, reject_message);
 590             os.close(fd);
 591             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 592         else:
 593             # Build up the rejection email
 594             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 595
 596             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 597             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 598             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 599             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 600
 601             # Write the rejection email out as the <foo>.reason file
 602             os.write(fd, reject_mail_message);
 603             os.close(fd);
 604
 605         # Send the rejection mail if appropriate
 606         if not Cnf["Dinstall::Options::No-Mail"]:
 607             utils.send_mail(reject_mail_message);
 608
 609         self.Logger.log(["rejected", pkg.changes_file]);
 610         return 0;
 611
 612     ################################################################################
 613
 614     # Ensure that source exists somewhere in the archive for the binary
 615     # upload being processed.
 616     #
 617     # (1) exact match                      => 1.0-3
 618     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 619     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 620
 621     def source_exists (self, package, source_version, suites = ["any"]):
 622         okay = 1
 623         for suite in suites:
 624             if suite == "any":
 625                 que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
 626                     (package)
 627             else:
 628                 suite_id = db_access.get_suite_id(suite);
 629                 que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE sa.suite = %d AND s.source = '%s'" % (suite_id, package)
 630             q = self.projectB.query(que)
 631
 632             # Reduce the query results to a list of version numbers
 633             ql = map(lambda x: x[0], q.getresult());
 634
 635             # Try (1)
 636             if ql.count(source_version):
 637                 continue
 638
 639             # Try (2)
 640             orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version)
 641             if ql.count(orig_source_version):
 642                 continue
 643
 644             # Try (3)
 645             orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version)
 646             if ql.count(orig_source_version):
 647                 continue
 648
 649             # No source found...
 650             okay = 0
 651         return okay
 652
 653     ################################################################################
 654
 655     def in_override_p (self, package, component, suite, binary_type, file):
 656         files = self.pkg.files;
 657
 658         if binary_type == "": # must be source
 659             type = "dsc";
 660         else:
 661             type = binary_type;
 662
 663         # Override suite name; used for example with proposed-updates
 664         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 665             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 666
 667         # Avoid <undef> on unknown distributions
 668         suite_id = db_access.get_suite_id(suite);
 669         if suite_id == -1:
 670             return None;
 671         component_id = db_access.get_component_id(component);
 672         type_id = db_access.get_override_type_id(type);
 673
 674         # FIXME: nasty non-US speficic hack
 675         if component[:7].lower() == "non-us/":
 676             component = component[7:];
 677
 678         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 679                            % (package, suite_id, component_id, type_id));
 680         result = q.getresult();
 681         # If checking for a source package fall back on the binary override type
 682         if type == "dsc" and not result:
 683             deb_type_id = db_access.get_override_type_id("deb");
 684             udeb_type_id = db_access.get_override_type_id("udeb");
 685             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
 686                                % (package, suite_id, component_id, deb_type_id, udeb_type_id));
 687             result = q.getresult();
 688
 689         # Remember the section and priority so we can check them later if appropriate
 690         if result:
 691             files[file]["override section"] = result[0][0];
 692             files[file]["override priority"] = result[0][1];
 693
 694         return result;
 695
 696     ################################################################################
 697
 698     def reject (self, str, prefix="Rejected: "):
 699         if str:
 700             # Unlike other rejects we add new lines first to avoid trailing
 701             # new lines when this message is passed back up to a caller.
 702             if self.reject_message:
 703                 self.reject_message += "\n";
 704             self.reject_message += prefix + str;
 705
 706     ################################################################################
 707
 708     def cross_suite_version_check(self, query_result, file, new_version):
 709         """Ensure versions are newer than existing packages in target
 710         suites and that cross-suite version checking rules as
 711         set out in the conf file are satisfied."""
 712
 713         # Check versions for each target suite
 714         for target_suite in self.pkg.changes["distribution"].keys():
 715             must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
 716             must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
 717             # Enforce "must be newer than target suite" even if conffile omits it
 718             if target_suite not in must_be_newer_than:
 719                 must_be_newer_than.append(target_suite);
 720             for entry in query_result:
 721                 existent_version = entry[0];
 722                 suite = entry[1];
 723                 if suite in must_be_newer_than and \
 724                    apt_pkg.VersionCompare(new_version, existent_version) != 1:
 725                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 726                 if suite in must_be_older_than and \
 727                    apt_pkg.VersionCompare(new_version, existent_version) != -1:
 728                     self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 729
 730     ################################################################################
 731
 732     def check_binary_against_db(self, file):
 733         self.reject_message = "";
 734         files = self.pkg.files;
 735
 736         # Ensure version is sane
 737         q = self.projectB.query("""
 738 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
 739                                      architecture a
 740  WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
 741    AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
 742                                 % (files[file]["package"],
 743                                    files[file]["architecture"]));
 744         self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
 745
 746         # Check for any existing copies of the file
 747         q = self.projectB.query("""
 748 SELECT b.id FROM binaries b, architecture a
 749  WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
 750    AND a.id = b.architecture"""
 751                                 % (files[file]["package"],
 752                                    files[file]["version"],
 753                                    files[file]["architecture"]))
 754         if q.getresult():
 755             self.reject("can not overwrite existing copy of '%s' already in the archive." % (file));
 756
 757         return self.reject_message;
 758
 759     ################################################################################
 760
 761     def check_source_against_db(self, file):
 762         self.reject_message = "";
 763         dsc = self.pkg.dsc;
 764
 765         # Ensure version is sane
 766         q = self.projectB.query("""
 767 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
 768  WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
 769         self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
 770
 771         return self.reject_message;
 772
 773     ################################################################################
 774
 775     # **WARNING**
 776     # NB: this function can remove entries from the 'files' index [if
 777     # the .orig.tar.gz is a duplicate of the one in the archive]; if
 778     # you're iterating over 'files' and call this function as part of
 779     # the loop, be sure to add a check to the top of the loop to
 780     # ensure you haven't just tried to derefernece the deleted entry.
 781     # **WARNING**
 782
 783     def check_dsc_against_db(self, file):
 784         self.reject_message = "";
 785         files = self.pkg.files;
 786         dsc_files = self.pkg.dsc_files;
 787         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 788         orig_tar_gz = None;
 789
 790         # Try and find all files mentioned in the .dsc.  This has
 791         # to work harder to cope with the multiple possible
 792         # locations of an .orig.tar.gz.
 793         for dsc_file in dsc_files.keys():
 794             found = None;
 795             if files.has_key(dsc_file):
 796                 actual_md5 = files[dsc_file]["md5sum"];
 797                 actual_size = int(files[dsc_file]["size"]);
 798                 found = "%s in incoming" % (dsc_file)
 799                 # Check the file does not already exist in the archive
 800                 q = self.projectB.query("SELECT size, md5sum, filename FROM files WHERE filename LIKE '%%%s%%'" % (dsc_file));
 801
 802                 ql = q.getresult();
 803                 # Strip out anything that isn't '%s' or '/%s$'
 804                 for i in ql:
 805                     if i[2] != dsc_file and i[2][-(len(dsc_file)+1):] != '/'+dsc_file:
 806                         self.Logger.log(["check_dsc_against_db",i[2],dsc_file]);
 807                         ql.remove(i);
 808
 809                 # "[katie] has not broken them.  [katie] has fixed a
 810                 # brokenness.  Your crappy hack exploited a bug in
 811                 # the old dinstall.
 812                 #
 813                 # "(Come on!  I thought it was always obvious that
 814                 # one just doesn't release different files with
 815                 # the same name and version.)"
 816                 #                        -- ajk@ on d-devel@l.d.o
 817
 818                 if ql:
 819                     # Ignore exact matches for .orig.tar.gz
 820                     match = 0;
 821                     if dsc_file.endswith(".orig.tar.gz"):
 822                         for i in ql:
 823                             if files.has_key(dsc_file) and \
 824                                int(files[dsc_file]["size"]) == int(i[0]) and \
 825                                files[dsc_file]["md5sum"] == i[1]:
 826                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
 827                                 del files[dsc_file];
 828                                 match = 1;
 829
 830                     if not match:
 831                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 832             elif dsc_file.endswith(".orig.tar.gz"):
 833                 # Check in the pool
 834                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file));
 835                 ql = q.getresult();
 836                 # Strip out anything that isn't '%s' or '/%s$'
 837                 for i in ql:
 838                     if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
 839                         self.Logger.log(["check_dsc_against_db",i[1],dsc_file]);
 840                         ql.remove(i);
 841
 842                 if ql:
 843                     # Unfortunately, we make get more than one
 844                     # match here if, for example, the package was
 845                     # in potato but had a -sa upload in woody.  So
 846                     # we need to choose the right one.
 847
 848                     x = ql[0]; # default to something sane in case we don't match any or have only one
 849
 850                     if len(ql) > 1:
 851                         for i in ql:
 852                             old_file = i[0] + i[1];
 853                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 854                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 855                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 856                                 x = i;
 857                             else:
 858                                 legacy_source_untouchable[i[3]] = "";
 859
 860                     old_file = x[0] + x[1];
 861                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 862                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 863                     found = old_file;
 864                     suite_type = x[2];
 865                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 866                     # See install() in katie...
 867                     self.pkg.orig_tar_id = x[3];
 868                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 869                         self.pkg.orig_tar_location = "legacy";
 870                     else:
 871                         self.pkg.orig_tar_location = x[4];
 872                 else:
 873                     # Not there? Check the queue directories...
 874
 875                     in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file);
 876                     # See process_it() in jennifer for explanation of this
 877                     if os.path.exists(in_unchecked):
 878                         return (self.reject_message, in_unchecked);
 879                     else:
 880                         for dir in [ "Accepted", "New", "Byhand" ]:
 881                             in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file);
 882                             if os.path.exists(in_otherdir):
 883                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 884                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 885                                 found = in_otherdir;
 886
 887                     if not found:
 888                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 889                         continue;
 890             else:
 891                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 892                 continue;
 893             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 894                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 895             if actual_size != int(dsc_files[dsc_file]["size"]):
 896                 self.reject("size for %s doesn't match %s." % (found, file));
 897
 898         return (self.reject_message, orig_tar_gz);
 899
 900     def do_query(self, q):
 901         sys.stderr.write("query: \"%s\" ... " % (q));
 902         before = time.time();
 903         r = self.projectB.query(q);
 904         time_diff = time.time()-before;
 905         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
 906         return r;