katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001, 2002, 2003  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.40 2003-09-17 23:36:17 troup Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28
  29 ###############################################################################
  30
  31 re_isanum = re.compile (r"^\d+$");
  32 re_default_answer = re.compile(r"\[(.*)\]");
  33 re_fdnic = re.compile("\n\n");
  34 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
  35 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
  36
  37 ###############################################################################
  38
  39 # Convenience wrapper to carry around all the package information in
  40
  41 class Pkg:
  42     def __init__(self, **kwds):
  43         self.__dict__.update(kwds);
  44
  45     def update(self, **kwds):
  46         self.__dict__.update(kwds);
  47
  48 ###############################################################################
  49
  50 class nmu_p:
  51     # Read in the group maintainer override file
  52     def __init__ (self, Cnf):
  53         self.group_maint = {};
  54         self.Cnf = Cnf;
  55         if Cnf.get("Dinstall::GroupOverrideFilename"):
  56             filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"];
  57             file = utils.open_file(filename);
  58             for line in file.readlines():
  59                 line = utils.re_comments.sub('', line).lower().strip();
  60                 if line != "":
  61                     self.group_maint[line] = 1;
  62             file.close();
  63
  64     def is_an_nmu (self, pkg):
  65         Cnf = self.Cnf;
  66         changes = pkg.changes;
  67         dsc = pkg.dsc;
  68
  69         (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]).lower());
  70         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  71         if dsc_name == changes["maintainername"].lower() and \
  72            (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name):
  73             return 0;
  74
  75         if dsc.has_key("uploaders"):
  76             uploaders = dsc["uploaders"].lower().split(",");
  77             uploadernames = {};
  78             for i in uploaders:
  79                 (rfc822, name, email) = utils.fix_maintainer (i.strip());
  80                 uploadernames[name] = "";
  81             if uploadernames.has_key(changes["changedbyname"].lower()):
  82                 return 0;
  83
  84         # Some group maintained packages (e.g. Debian QA) are never NMU's
  85         if self.group_maint.has_key(changes["maintaineremail"].lower()):
  86             return 0;
  87
  88         return 1;
  89
  90 ###############################################################################
  91
  92 class Katie:
  93
  94     def __init__(self, Cnf):
  95         self.Cnf = Cnf;
  96         self.values = {};
  97         # Read in the group-maint override file
  98         self.nmu = nmu_p(Cnf);
  99         self.accept_count = 0;
 100         self.accept_bytes = 0L;
 101         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 102                        legacy_source_untouchable = {});
 103
 104         # Initialize the substitution template mapping global
 105         Subst = self.Subst = {};
 106         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 107         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 108         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 109         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 110
 111         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 112         db_access.init(Cnf, self.projectB);
 113
 114     ###########################################################################
 115
 116     def init_vars (self):
 117         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 118             exec "self.pkg.%s.clear();" % (i);
 119         self.pkg.orig_tar_id = None;
 120         self.pkg.orig_tar_location = "";
 121
 122     ###########################################################################
 123
 124     def update_vars (self):
 125         dump_filename = self.pkg.changes_file[:-8]+".katie";
 126         dump_file = utils.open_file(dump_filename);
 127         p = cPickle.Unpickler(dump_file);
 128         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 129             exec "self.pkg.%s.update(p.load());" % (i);
 130         for i in [ "orig_tar_id", "orig_tar_location" ]:
 131             exec "self.pkg.%s = p.load();" % (i);
 132         dump_file.close();
 133
 134     ###########################################################################
 135
 136     # This could just dump the dictionaries as is, but I'd like to avoid
 137     # this so there's some idea of what katie & lisa use from jennifer
 138
 139     def dump_vars(self, dest_dir):
 140         for i in [ "changes", "dsc", "files", "dsc_files",
 141                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 142             exec "%s = self.pkg.%s;" % (i,i);
 143         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 144         dump_file = utils.open_file(dump_filename, 'w');
 145         try:
 146             os.chmod(dump_filename, 0660);
 147         except OSError, e:
 148             if errno.errorcode[e.errno] == 'EPERM':
 149                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]);
 150                 if perms & stat.S_IROTH:
 151                     utils.fubar("%s is world readable and chmod failed." % (dump_filename));
 152             else:
 153                 raise;
 154
 155         p = cPickle.Pickler(dump_file, 1);
 156         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 157             exec "%s = {}" % i;
 158         ## files
 159         for file in files.keys():
 160             d_files[file] = {};
 161             for i in [ "package", "version", "architecture", "type", "size",
 162                        "md5sum", "component", "location id", "source package",
 163                        "source version", "maintainer", "dbtype", "files id",
 164                        "new", "section", "priority", "othercomponents",
 165                        "pool name", "original component" ]:
 166                 if files[file].has_key(i):
 167                     d_files[file][i] = files[file][i];
 168         ## changes
 169         # Mandatory changes fields
 170         for i in [ "distribution", "source", "architecture", "version", "maintainer",
 171                    "urgency", "fingerprint", "changedby822", "changedbyname",
 172                    "maintainername", "maintaineremail", "closes" ]:
 173             d_changes[i] = changes[i];
 174         # Optional changes fields
 175         # FIXME: changes should be mandatory
 176         for i in [ "changed-by", "maintainer822", "filecontents", "format",
 177                    "changes", "lisa note" ]:
 178             if changes.has_key(i):
 179                 d_changes[i] = changes[i];
 180         ## dsc
 181         for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
 182             if dsc.has_key(i):
 183                 d_dsc[i] = dsc[i];
 184         ## dsc_files
 185         for file in dsc_files.keys():
 186             d_dsc_files[file] = {};
 187             # Mandatory dsc_files fields
 188             for i in [ "size", "md5sum" ]:
 189                 d_dsc_files[file][i] = dsc_files[file][i];
 190             # Optional dsc_files fields
 191             for i in [ "files id" ]:
 192                 if dsc_files[file].has_key(i):
 193                     d_dsc_files[file][i] = dsc_files[file][i];
 194
 195         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 196                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 197             p.dump(i);
 198         dump_file.close();
 199
 200     ###########################################################################
 201
 202     # Set up the per-package template substitution mappings
 203
 204     def update_subst (self, reject_message = ""):
 205         Subst = self.Subst;
 206         changes = self.pkg.changes;
 207         # If jennifer crashed out in the right place, architecture may still be a string.
 208         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 209             changes["architecture"] = { "Unknown" : "" };
 210         # and maintainer822 may not exist.
 211         if not changes.has_key("maintainer822"):
 212             changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
 213
 214         Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys());
 215         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 216         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 217
 218         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 219         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 220             Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
 221             Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
 222             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 223         else:
 224             Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
 225             Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
 226             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 227         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 228             Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 229
 230         # Apply any global override of the Maintainer field
 231         if self.Cnf.get("Dinstall::OverrideMaintainer"):
 232             Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 233             Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 234
 235         Subst["__REJECT_MESSAGE__"] = reject_message;
 236         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 237         Subst["__VERSION__"] = changes.get("version", "Unknown");
 238
 239     ###########################################################################
 240
 241     def build_summaries(self):
 242         changes = self.pkg.changes;
 243         files = self.pkg.files;
 244
 245         byhand = summary = new = "";
 246
 247         # changes["distribution"] may not exist in corner cases
 248         # (e.g. unreadable changes files)
 249         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 250             changes["distribution"] = {};
 251
 252         file_keys = files.keys();
 253         file_keys.sort();
 254         for file in file_keys:
 255             if files[file].has_key("byhand"):
 256                 byhand = 1
 257                 summary += file + " byhand\n"
 258             elif files[file].has_key("new"):
 259                 new = 1
 260                 summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 261                 if files[file].has_key("othercomponents"):
 262                     summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 263                 if files[file]["type"] == "deb":
 264                     summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 265             else:
 266                 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
 267                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 268                 summary += file + "\n  to " + destination + "\n"
 269
 270         short_summary = summary;
 271
 272         # This is for direport's benefit...
 273         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 274
 275         if byhand or new:
 276             summary += "Changes: " + f;
 277
 278         summary += self.announce(short_summary, 0)
 279
 280         return (summary, short_summary);
 281
 282     ###########################################################################
 283
 284     def close_bugs (self, summary, action):
 285         changes = self.pkg.changes;
 286         Subst = self.Subst;
 287         Cnf = self.Cnf;
 288
 289         bugs = changes["closes"].keys();
 290
 291         if not bugs:
 292             return summary;
 293
 294         bugs.sort();
 295         if not self.nmu.is_an_nmu(self.pkg):
 296             if changes["distribution"].has_key("experimental"):
 297                 # tag bugs as fixed-in-experimental for uploads to experimental
 298                 summary += "Setting bugs to severity fixed: ";
 299                 control_message = "";
 300                 for bug in bugs:
 301                     summary += "%s " % (bug);
 302                     control_message += "tag %s + fixed-in-experimental\n" % (bug);
 303                 if action and control_message != "":
 304                     Subst["__CONTROL_MESSAGE__"] = control_message;
 305                     mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-experimental-fixed");
 306                     utils.send_mail (mail_message);
 307                 if action:
 308                     self.Logger.log(["setting bugs to fixed"]+bugs);
 309
 310
 311             else:
 312                 summary += "Closing bugs: ";
 313                 for bug in bugs:
 314                     summary += "%s " % (bug);
 315                     if action:
 316                         Subst["__BUG_NUMBER__"] = bug;
 317                         if changes["distribution"].has_key("stable"):
 318                             Subst["__STABLE_WARNING__"] = """
 319 Note that this package is not part of the released stable Debian
 320 distribution.  It may have dependencies on other unreleased software,
 321 or other instabilities.  Please take care if you wish to install it.
 322 The update will eventually make its way into the next released Debian
 323 distribution.""";
 324                         else:
 325                             Subst["__STABLE_WARNING__"] = "";
 326                             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
 327                             utils.send_mail (mail_message);
 328                 if action:
 329                     self.Logger.log(["closing bugs"]+bugs);
 330
 331         else:                     # NMU
 332             summary += "Setting bugs to severity fixed: ";
 333             control_message = "";
 334             for bug in bugs:
 335                 summary += "%s " % (bug);
 336                 control_message += "tag %s + fixed\n" % (bug);
 337             if action and control_message != "":
 338                 Subst["__CONTROL_MESSAGE__"] = control_message;
 339                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed");
 340                 utils.send_mail (mail_message);
 341             if action:
 342                 self.Logger.log(["setting bugs to fixed"]+bugs);
 343         summary += "\n";
 344         return summary;
 345
 346     ###########################################################################
 347
 348     def announce (self, short_summary, action):
 349         Subst = self.Subst;
 350         Cnf = self.Cnf;
 351         changes = self.pkg.changes;
 352
 353         # Only do announcements for source uploads with a recent dpkg-dev installed
 354         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 355             return "";
 356
 357         lists_done = {};
 358         summary = "";
 359         Subst["__SHORT_SUMMARY__"] = short_summary;
 360
 361         for dist in changes["distribution"].keys():
 362             list = Cnf.Find("Suite::%s::Announce" % (dist));
 363             if list == "" or lists_done.has_key(list):
 364                 continue;
 365             lists_done[list] = 1;
 366             summary += "Announcing to %s\n" % (list);
 367
 368             if action:
 369                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 370                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 371                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]);
 372                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce");
 373                 utils.send_mail (mail_message);
 374
 375         if Cnf.FindB("Dinstall::CloseBugs"):
 376             summary = self.close_bugs(summary, action);
 377
 378         return summary;
 379
 380     ###########################################################################
 381
 382     def accept (self, summary, short_summary):
 383         Cnf = self.Cnf;
 384         Subst = self.Subst;
 385         files = self.pkg.files;
 386
 387         print "Accepting."
 388         self.Logger.log(["Accepting changes",self.pkg.changes_file]);
 389
 390         self.dump_vars(Cnf["Dir::Queue::Accepted"]);
 391
 392         # Move all the files into the accepted directory
 393         utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
 394         file_keys = files.keys();
 395         for file in file_keys:
 396             utils.move(file, Cnf["Dir::Queue::Accepted"]);
 397             self.accept_bytes += float(files[file]["size"])
 398         self.accept_count += 1;
 399
 400         # Send accept mail, announce to lists, close bugs and check for
 401         # override disparities
 402         if not Cnf["Dinstall::Options::No-Mail"]:
 403             Subst["__SUITE__"] = "";
 404             Subst["__SUMMARY__"] = summary;
 405             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted");
 406             utils.send_mail(mail_message)
 407             self.announce(short_summary, 1)
 408
 409         # Special support to enable clean auto-building of accepted packages
 410         self.projectB.query("BEGIN WORK");
 411         for suite in self.pkg.changes["distribution"].keys():
 412             if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
 413                 continue;
 414             suite_id = db_access.get_suite_id(suite);
 415             dest_dir = Cnf["Dir::AcceptedAutoBuild"];
 416             if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 417                 dest_dir = os.path.join(dest_dir, suite);
 418             for file in file_keys:
 419                 src = os.path.join(Cnf["Dir::Queue::Accepted"], file);
 420                 dest = os.path.join(dest_dir, file);
 421                 if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 422                     # Copy it since the original won't be readable by www-data
 423                     utils.copy(src, dest);
 424                 else:
 425                     # Create a symlink to it
 426                     os.symlink(src, dest);
 427                 # Add it to the list of packages for later processing by apt-ftparchive
 428                 self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 429             # If the .orig.tar.gz is in the pool, create a symlink to
 430             # it (if one doesn't already exist)
 431             if self.pkg.orig_tar_id:
 432                 # Determine the .orig.tar.gz file name
 433                 for dsc_file in self.pkg.dsc_files.keys():
 434                     if dsc_file.endswith(".orig.tar.gz"):
 435                         filename = dsc_file;
 436                 dest = os.path.join(dest_dir, filename);
 437                 # If it doesn't exist, create a symlink
 438                 if not os.path.exists(dest):
 439                     # Find the .orig.tar.gz in the pool
 440                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 441                     ql = q.getresult();
 442                     if not ql:
 443                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 444                     src = os.path.join(ql[0][0], ql[0][1]);
 445                     os.symlink(src, dest);
 446                     # Add it to the list of packages for later processing by apt-ftparchive
 447                     self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 448                 # if it does, update things to ensure it's not removed prematurely
 449                 else:
 450                     self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id));
 451
 452         self.projectB.query("COMMIT WORK");
 453
 454     ###########################################################################
 455
 456     def check_override (self):
 457         Subst = self.Subst;
 458         changes = self.pkg.changes;
 459         files = self.pkg.files;
 460         Cnf = self.Cnf;
 461
 462         # Abandon the check if:
 463         #  a) it's a non-sourceful upload
 464         #  b) override disparity checks have been disabled
 465         #  c) we're not sending mail
 466         if not changes["architecture"].has_key("source") or \
 467            not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 468            Cnf["Dinstall::Options::No-Mail"]:
 469             return;
 470
 471         summary = "";
 472         for file in files.keys():
 473             if not files[file].has_key("new") and files[file]["type"] == "deb":
 474                 section = files[file]["section"];
 475                 override_section = files[file]["override section"];
 476                 if section.lower() != override_section.lower() and section != "-":
 477                     # Ignore this; it's a common mistake and not worth whining about
 478                     if section.lower() == "non-us/main" and override_section.lower() == "non-us":
 479                         continue;
 480                     summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section);
 481                 priority = files[file]["priority"];
 482                 override_priority = files[file]["override priority"];
 483                 if priority != override_priority and priority != "-":
 484                     summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority);
 485
 486         if summary == "":
 487             return;
 488
 489         Subst["__SUMMARY__"] = summary;
 490         mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity");
 491         utils.send_mail(mail_message);
 492
 493     ###########################################################################
 494
 495     def force_reject (self, files):
 496         """Forcefully move files from the current directory to the
 497            reject directory.  If any file already exists in the reject
 498            directory it will be moved to the morgue to make way for
 499            the new file."""
 500
 501         Cnf = self.Cnf
 502
 503         for file in files:
 504             # Skip any files which don't exist or which we don't have permission to copy.
 505             if os.access(file,os.R_OK) == 0:
 506                 continue;
 507             dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file);
 508             try:
 509                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 510             except OSError, e:
 511                 # File exists?  Let's try and move it to the morgue
 512                 if errno.errorcode[e.errno] == 'EEXIST':
 513                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file);
 514                     try:
 515                         morgue_file = utils.find_next_free(morgue_file);
 516                     except utils.tried_too_hard_exc:
 517                         # Something's either gone badly Pete Tong, or
 518                         # someone is trying to exploit us.
 519                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 520                         return;
 521                     utils.move(dest_file, morgue_file, perms=0660);
 522                     try:
 523                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 524                     except OSError, e:
 525                         # Likewise
 526                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 527                         return;
 528                 else:
 529                     raise;
 530             # If we got here, we own the destination file, so we can
 531             # safely overwrite it.
 532             utils.move(file, dest_file, 1, perms=0660);
 533
 534     ###########################################################################
 535
 536     def do_reject (self, manual = 0, reject_message = ""):
 537         # If we weren't given a manual rejection message, spawn an
 538         # editor so the user can add one in...
 539         if manual and not reject_message:
 540             temp_filename = tempfile.mktemp();
 541             fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
 542             os.close(fd);
 543             editor = os.environ.get("EDITOR","vi")
 544             answer = 'E';
 545             while answer == 'E':
 546                 os.system("%s %s" % (editor, temp_filename))
 547                 file = utils.open_file(temp_filename);
 548                 reject_message = "".join(file.readlines());
 549                 file.close();
 550                 print "Reject message:";
 551                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1);
 552                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 553                 answer = "XXX";
 554                 while prompt.find(answer) == -1:
 555                     answer = utils.our_raw_input(prompt);
 556                     m = re_default_answer.search(prompt);
 557                     if answer == "":
 558                         answer = m.group(1);
 559                     answer = answer[:1].upper();
 560             os.unlink(temp_filename);
 561             if answer == 'A':
 562                 return 1;
 563             elif answer == 'Q':
 564                 sys.exit(0);
 565
 566         print "Rejecting.\n"
 567
 568         Cnf = self.Cnf;
 569         Subst = self.Subst;
 570         pkg = self.pkg;
 571
 572         reason_filename = pkg.changes_file[:-8] + ".reason";
 573         reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
 574
 575         # Move all the files into the reject directory
 576         reject_files = pkg.files.keys() + [pkg.changes_file];
 577         self.force_reject(reject_files);
 578
 579         # If we fail here someone is probably trying to exploit the race
 580         # so let's just raise an exception ...
 581         if os.path.exists(reject_filename):
 582             os.unlink(reject_filename);
 583         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 584
 585         if not manual:
 586             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 587             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 588             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 589             os.write(fd, reject_message);
 590             os.close(fd);
 591             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 592         else:
 593             # Build up the rejection email
 594             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 595
 596             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 597             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 598             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 599             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 600
 601             # Write the rejection email out as the <foo>.reason file
 602             os.write(fd, reject_mail_message);
 603             os.close(fd);
 604
 605         # Send the rejection mail if appropriate
 606         if not Cnf["Dinstall::Options::No-Mail"]:
 607             utils.send_mail(reject_mail_message);
 608
 609         self.Logger.log(["rejected", pkg.changes_file]);
 610         return 0;
 611
 612     ################################################################################
 613
 614     # Ensure that source exists somewhere in the archive for the binary
 615     # upload being processed.
 616     #
 617     # (1) exact match                      => 1.0-3
 618     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 619     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 620
 621     def source_exists (self, package, source_version, suites = ["any"]):
 622         okay = 1
 623         for suite in suites:
 624             if suite == "any":
 625                 que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
 626                     (package)
 627             else:
 628                 # source must exist in suite X, or in some other suite that's
 629                 # mapped to X, recursively... silent-maps are counted too,
 630                 # unreleased-maps aren't.
 631                 maps = self.Cnf.ValueList("SuiteMappings")[:]
 632                 maps.reverse()
 633                 maps = [ m.split() for m in maps ]
 634                 maps = [ (x[1], x[2]) for x in maps
 635                                 if x[0] == "map" or x[0] == "silent-map" ]
 636                 s = [suite]
 637                 for x in maps:
 638                         if x[1] in s and x[0] not in s:
 639                                 s.append(x[0])
 640
 641                 que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR "));
 642             q = self.projectB.query(que)
 643
 644             # Reduce the query results to a list of version numbers
 645             ql = map(lambda x: x[0], q.getresult());
 646
 647             # Try (1)
 648             if ql.count(source_version):
 649                 continue
 650
 651             # Try (2)
 652             orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version)
 653             if ql.count(orig_source_version):
 654                 continue
 655
 656             # Try (3)
 657             orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version)
 658             if ql.count(orig_source_version):
 659                 continue
 660
 661             # No source found...
 662             okay = 0
 663         return okay
 664
 665     ################################################################################
 666
 667     def in_override_p (self, package, component, suite, binary_type, file):
 668         files = self.pkg.files;
 669
 670         if binary_type == "": # must be source
 671             type = "dsc";
 672         else:
 673             type = binary_type;
 674
 675         # Override suite name; used for example with proposed-updates
 676         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 677             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 678
 679         # Avoid <undef> on unknown distributions
 680         suite_id = db_access.get_suite_id(suite);
 681         if suite_id == -1:
 682             return None;
 683         component_id = db_access.get_component_id(component);
 684         type_id = db_access.get_override_type_id(type);
 685
 686         # FIXME: nasty non-US speficic hack
 687         if component[:7].lower() == "non-us/":
 688             component = component[7:];
 689
 690         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 691                            % (package, suite_id, component_id, type_id));
 692         result = q.getresult();
 693         # If checking for a source package fall back on the binary override type
 694         if type == "dsc" and not result:
 695             deb_type_id = db_access.get_override_type_id("deb");
 696             udeb_type_id = db_access.get_override_type_id("udeb");
 697             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
 698                                % (package, suite_id, component_id, deb_type_id, udeb_type_id));
 699             result = q.getresult();
 700
 701         # Remember the section and priority so we can check them later if appropriate
 702         if result:
 703             files[file]["override section"] = result[0][0];
 704             files[file]["override priority"] = result[0][1];
 705
 706         return result;
 707
 708     ################################################################################
 709
 710     def reject (self, str, prefix="Rejected: "):
 711         if str:
 712             # Unlike other rejects we add new lines first to avoid trailing
 713             # new lines when this message is passed back up to a caller.
 714             if self.reject_message:
 715                 self.reject_message += "\n";
 716             self.reject_message += prefix + str;
 717
 718     ################################################################################
 719
 720     def cross_suite_version_check(self, query_result, file, new_version):
 721         """Ensure versions are newer than existing packages in target
 722         suites and that cross-suite version checking rules as
 723         set out in the conf file are satisfied."""
 724
 725         # Check versions for each target suite
 726         for target_suite in self.pkg.changes["distribution"].keys():
 727             must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
 728             must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
 729             # Enforce "must be newer than target suite" even if conffile omits it
 730             if target_suite not in must_be_newer_than:
 731                 must_be_newer_than.append(target_suite);
 732             for entry in query_result:
 733                 existent_version = entry[0];
 734                 suite = entry[1];
 735                 if suite in must_be_newer_than and \
 736                    apt_pkg.VersionCompare(new_version, existent_version) != 1:
 737                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 738                 if suite in must_be_older_than and \
 739                    apt_pkg.VersionCompare(new_version, existent_version) != -1:
 740                     self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 741
 742     ################################################################################
 743
 744     def check_binary_against_db(self, file):
 745         self.reject_message = "";
 746         files = self.pkg.files;
 747
 748         # Ensure version is sane
 749         q = self.projectB.query("""
 750 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
 751                                      architecture a
 752  WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
 753    AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
 754                                 % (files[file]["package"],
 755                                    files[file]["architecture"]));
 756         self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
 757
 758         # Check for any existing copies of the file
 759         q = self.projectB.query("""
 760 SELECT b.id FROM binaries b, architecture a
 761  WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
 762    AND a.id = b.architecture"""
 763                                 % (files[file]["package"],
 764                                    files[file]["version"],
 765                                    files[file]["architecture"]))
 766         if q.getresult():
 767             self.reject("%s: can not overwrite existing copy already in the archive." % (file));
 768
 769         return self.reject_message;
 770
 771     ################################################################################
 772
 773     def check_source_against_db(self, file):
 774         self.reject_message = "";
 775         dsc = self.pkg.dsc;
 776
 777         # Ensure version is sane
 778         q = self.projectB.query("""
 779 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
 780  WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
 781         self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
 782
 783         return self.reject_message;
 784
 785     ################################################################################
 786
 787     # **WARNING**
 788     # NB: this function can remove entries from the 'files' index [if
 789     # the .orig.tar.gz is a duplicate of the one in the archive]; if
 790     # you're iterating over 'files' and call this function as part of
 791     # the loop, be sure to add a check to the top of the loop to
 792     # ensure you haven't just tried to derefernece the deleted entry.
 793     # **WARNING**
 794
 795     def check_dsc_against_db(self, file):
 796         self.reject_message = "";
 797         files = self.pkg.files;
 798         dsc_files = self.pkg.dsc_files;
 799         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 800         orig_tar_gz = None;
 801
 802         # Try and find all files mentioned in the .dsc.  This has
 803         # to work harder to cope with the multiple possible
 804         # locations of an .orig.tar.gz.
 805         for dsc_file in dsc_files.keys():
 806             found = None;
 807             if files.has_key(dsc_file):
 808                 actual_md5 = files[dsc_file]["md5sum"];
 809                 actual_size = int(files[dsc_file]["size"]);
 810                 found = "%s in incoming" % (dsc_file)
 811                 # Check the file does not already exist in the archive
 812                 q = self.projectB.query("SELECT size, md5sum, filename FROM files WHERE filename LIKE '%%%s%%'" % (dsc_file));
 813
 814                 ql = q.getresult();
 815                 # Strip out anything that isn't '%s' or '/%s$'
 816                 for i in ql:
 817                     if i[2] != dsc_file and i[2][-(len(dsc_file)+1):] != '/'+dsc_file:
 818                         self.Logger.log(["check_dsc_against_db",i[2],dsc_file]);
 819                         ql.remove(i);
 820
 821                 # "[katie] has not broken them.  [katie] has fixed a
 822                 # brokenness.  Your crappy hack exploited a bug in
 823                 # the old dinstall.
 824                 #
 825                 # "(Come on!  I thought it was always obvious that
 826                 # one just doesn't release different files with
 827                 # the same name and version.)"
 828                 #                        -- ajk@ on d-devel@l.d.o
 829
 830                 if ql:
 831                     # Ignore exact matches for .orig.tar.gz
 832                     match = 0;
 833                     if dsc_file.endswith(".orig.tar.gz"):
 834                         for i in ql:
 835                             if files.has_key(dsc_file) and \
 836                                int(files[dsc_file]["size"]) == int(i[0]) and \
 837                                files[dsc_file]["md5sum"] == i[1]:
 838                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
 839                                 del files[dsc_file];
 840                                 match = 1;
 841
 842                     if not match:
 843                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 844             elif dsc_file.endswith(".orig.tar.gz"):
 845                 # Check in the pool
 846                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file));
 847                 ql = q.getresult();
 848                 # Strip out anything that isn't '%s' or '/%s$'
 849                 for i in ql:
 850                     if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
 851                         self.Logger.log(["check_dsc_against_db",i[1],dsc_file]);
 852                         ql.remove(i);
 853
 854                 if ql:
 855                     # Unfortunately, we make get more than one
 856                     # match here if, for example, the package was
 857                     # in potato but had a -sa upload in woody.  So
 858                     # we need to choose the right one.
 859
 860                     x = ql[0]; # default to something sane in case we don't match any or have only one
 861
 862                     if len(ql) > 1:
 863                         for i in ql:
 864                             old_file = i[0] + i[1];
 865                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 866                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 867                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 868                                 x = i;
 869                             else:
 870                                 legacy_source_untouchable[i[3]] = "";
 871
 872                     old_file = x[0] + x[1];
 873                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 874                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 875                     found = old_file;
 876                     suite_type = x[2];
 877                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 878                     # See install() in katie...
 879                     self.pkg.orig_tar_id = x[3];
 880                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 881                         self.pkg.orig_tar_location = "legacy";
 882                     else:
 883                         self.pkg.orig_tar_location = x[4];
 884                 else:
 885                     # Not there? Check the queue directories...
 886
 887                     in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file);
 888                     # See process_it() in jennifer for explanation of this
 889                     if os.path.exists(in_unchecked):
 890                         return (self.reject_message, in_unchecked);
 891                     else:
 892                         for dir in [ "Accepted", "New", "Byhand" ]:
 893                             in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file);
 894                             if os.path.exists(in_otherdir):
 895                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 896                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 897                                 found = in_otherdir;
 898
 899                     if not found:
 900                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 901                         continue;
 902             else:
 903                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 904                 continue;
 905             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 906                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 907             if actual_size != int(dsc_files[dsc_file]["size"]):
 908                 self.reject("size for %s doesn't match %s." % (found, file));
 909
 910         return (self.reject_message, orig_tar_gz);
 911
 912     def do_query(self, q):
 913         sys.stderr.write("query: \"%s\" ... " % (q));
 914         before = time.time();
 915         r = self.projectB.query(q);
 916         time_diff = time.time()-before;
 917         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
 918         return r;