katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001, 2002, 2003  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.42 2003-10-29 16:57:44 troup Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28
  29 ###############################################################################
  30
  31 re_isanum = re.compile (r"^\d+$");
  32 re_default_answer = re.compile(r"\[(.*)\]");
  33 re_fdnic = re.compile("\n\n");
  34 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
  35 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
  36
  37 ###############################################################################
  38
  39 # Convenience wrapper to carry around all the package information in
  40
  41 class Pkg:
  42     def __init__(self, **kwds):
  43         self.__dict__.update(kwds);
  44
  45     def update(self, **kwds):
  46         self.__dict__.update(kwds);
  47
  48 ###############################################################################
  49
  50 class nmu_p:
  51     # Read in the group maintainer override file
  52     def __init__ (self, Cnf):
  53         self.group_maint = {};
  54         self.Cnf = Cnf;
  55         if Cnf.get("Dinstall::GroupOverrideFilename"):
  56             filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"];
  57             file = utils.open_file(filename);
  58             for line in file.readlines():
  59                 line = utils.re_comments.sub('', line).lower().strip();
  60                 if line != "":
  61                     self.group_maint[line] = 1;
  62             file.close();
  63
  64     def is_an_nmu (self, pkg):
  65         Cnf = self.Cnf;
  66         changes = pkg.changes;
  67         dsc = pkg.dsc;
  68
  69         (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]).lower());
  70         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  71         if dsc_name == changes["maintainername"].lower() and \
  72            (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name):
  73             return 0;
  74
  75         if dsc.has_key("uploaders"):
  76             uploaders = dsc["uploaders"].lower().split(",");
  77             uploadernames = {};
  78             for i in uploaders:
  79                 (rfc822, name, email) = utils.fix_maintainer (i.strip());
  80                 uploadernames[name] = "";
  81             if uploadernames.has_key(changes["changedbyname"].lower()):
  82                 return 0;
  83
  84         # Some group maintained packages (e.g. Debian QA) are never NMU's
  85         if self.group_maint.has_key(changes["maintaineremail"].lower()):
  86             return 0;
  87
  88         return 1;
  89
  90 ###############################################################################
  91
  92 class Katie:
  93
  94     def __init__(self, Cnf):
  95         self.Cnf = Cnf;
  96         self.values = {};
  97         # Read in the group-maint override file
  98         self.nmu = nmu_p(Cnf);
  99         self.accept_count = 0;
 100         self.accept_bytes = 0L;
 101         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 102                        legacy_source_untouchable = {});
 103
 104         # Initialize the substitution template mapping global
 105         Subst = self.Subst = {};
 106         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 107         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 108         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 109         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 110
 111         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 112         db_access.init(Cnf, self.projectB);
 113
 114     ###########################################################################
 115
 116     def init_vars (self):
 117         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 118             exec "self.pkg.%s.clear();" % (i);
 119         self.pkg.orig_tar_id = None;
 120         self.pkg.orig_tar_location = "";
 121
 122     ###########################################################################
 123
 124     def update_vars (self):
 125         dump_filename = self.pkg.changes_file[:-8]+".katie";
 126         dump_file = utils.open_file(dump_filename);
 127         p = cPickle.Unpickler(dump_file);
 128         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 129             exec "self.pkg.%s.update(p.load());" % (i);
 130         for i in [ "orig_tar_id", "orig_tar_location" ]:
 131             exec "self.pkg.%s = p.load();" % (i);
 132         dump_file.close();
 133
 134     ###########################################################################
 135
 136     # This could just dump the dictionaries as is, but I'd like to avoid
 137     # this so there's some idea of what katie & lisa use from jennifer
 138
 139     def dump_vars(self, dest_dir):
 140         for i in [ "changes", "dsc", "files", "dsc_files",
 141                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 142             exec "%s = self.pkg.%s;" % (i,i);
 143         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 144         dump_file = utils.open_file(dump_filename, 'w');
 145         try:
 146             os.chmod(dump_filename, 0660);
 147         except OSError, e:
 148             if errno.errorcode[e.errno] == 'EPERM':
 149                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]);
 150                 if perms & stat.S_IROTH:
 151                     utils.fubar("%s is world readable and chmod failed." % (dump_filename));
 152             else:
 153                 raise;
 154
 155         p = cPickle.Pickler(dump_file, 1);
 156         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 157             exec "%s = {}" % i;
 158         ## files
 159         for file in files.keys():
 160             d_files[file] = {};
 161             for i in [ "package", "version", "architecture", "type", "size",
 162                        "md5sum", "component", "location id", "source package",
 163                        "source version", "maintainer", "dbtype", "files id",
 164                        "new", "section", "priority", "othercomponents",
 165                        "pool name", "original component" ]:
 166                 if files[file].has_key(i):
 167                     d_files[file][i] = files[file][i];
 168         ## changes
 169         # Mandatory changes fields
 170         for i in [ "distribution", "source", "architecture", "version", "maintainer",
 171                    "urgency", "fingerprint", "changedby822", "changedbyname",
 172                    "maintainername", "maintaineremail", "closes" ]:
 173             d_changes[i] = changes[i];
 174         # Optional changes fields
 175         # FIXME: changes should be mandatory
 176         for i in [ "changed-by", "maintainer822", "filecontents", "format",
 177                    "changes", "lisa note" ]:
 178             if changes.has_key(i):
 179                 d_changes[i] = changes[i];
 180         ## dsc
 181         for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
 182             if dsc.has_key(i):
 183                 d_dsc[i] = dsc[i];
 184         ## dsc_files
 185         for file in dsc_files.keys():
 186             d_dsc_files[file] = {};
 187             # Mandatory dsc_files fields
 188             for i in [ "size", "md5sum" ]:
 189                 d_dsc_files[file][i] = dsc_files[file][i];
 190             # Optional dsc_files fields
 191             for i in [ "files id" ]:
 192                 if dsc_files[file].has_key(i):
 193                     d_dsc_files[file][i] = dsc_files[file][i];
 194
 195         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 196                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 197             p.dump(i);
 198         dump_file.close();
 199
 200     ###########################################################################
 201
 202     # Set up the per-package template substitution mappings
 203
 204     def update_subst (self, reject_message = ""):
 205         Subst = self.Subst;
 206         changes = self.pkg.changes;
 207         # If jennifer crashed out in the right place, architecture may still be a string.
 208         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 209             changes["architecture"] = { "Unknown" : "" };
 210         # and maintainer822 may not exist.
 211         if not changes.has_key("maintainer822"):
 212             changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
 213
 214         Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys());
 215         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 216         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 217
 218         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 219         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 220             Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
 221             Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
 222             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 223         else:
 224             Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
 225             Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
 226             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 227         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 228             Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 229
 230         # Apply any global override of the Maintainer field
 231         if self.Cnf.get("Dinstall::OverrideMaintainer"):
 232             Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 233             Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"];
 234
 235         Subst["__REJECT_MESSAGE__"] = reject_message;
 236         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 237         Subst["__VERSION__"] = changes.get("version", "Unknown");
 238
 239     ###########################################################################
 240
 241     def build_summaries(self):
 242         changes = self.pkg.changes;
 243         files = self.pkg.files;
 244
 245         byhand = summary = new = "";
 246
 247         # changes["distribution"] may not exist in corner cases
 248         # (e.g. unreadable changes files)
 249         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 250             changes["distribution"] = {};
 251
 252         file_keys = files.keys();
 253         file_keys.sort();
 254         for file in file_keys:
 255             if files[file].has_key("byhand"):
 256                 byhand = 1
 257                 summary += file + " byhand\n"
 258             elif files[file].has_key("new"):
 259                 new = 1
 260                 summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 261                 if files[file].has_key("othercomponents"):
 262                     summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 263                 if files[file]["type"] == "deb":
 264                     summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 265             else:
 266                 files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"])
 267                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 268                 summary += file + "\n  to " + destination + "\n"
 269
 270         short_summary = summary;
 271
 272         # This is for direport's benefit...
 273         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 274
 275         if byhand or new:
 276             summary += "Changes: " + f;
 277
 278         summary += self.announce(short_summary, 0)
 279
 280         return (summary, short_summary);
 281
 282     ###########################################################################
 283
 284     def close_bugs (self, summary, action):
 285         changes = self.pkg.changes;
 286         Subst = self.Subst;
 287         Cnf = self.Cnf;
 288
 289         bugs = changes["closes"].keys();
 290
 291         if not bugs:
 292             return summary;
 293
 294         bugs.sort();
 295         if not self.nmu.is_an_nmu(self.pkg):
 296             if changes["distribution"].has_key("experimental"):
 297                 # tag bugs as fixed-in-experimental for uploads to experimental
 298                 summary += "Setting bugs to severity fixed: ";
 299                 control_message = "";
 300                 for bug in bugs:
 301                     summary += "%s " % (bug);
 302                     control_message += "tag %s + fixed-in-experimental\n" % (bug);
 303                 if action and control_message != "":
 304                     Subst["__CONTROL_MESSAGE__"] = control_message;
 305                     mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-experimental-fixed");
 306                     utils.send_mail (mail_message);
 307                 if action:
 308                     self.Logger.log(["setting bugs to fixed"]+bugs);
 309
 310
 311             else:
 312                 summary += "Closing bugs: ";
 313                 for bug in bugs:
 314                     summary += "%s " % (bug);
 315                     if action:
 316                         Subst["__BUG_NUMBER__"] = bug;
 317                         if changes["distribution"].has_key("stable"):
 318                             Subst["__STABLE_WARNING__"] = """
 319 Note that this package is not part of the released stable Debian
 320 distribution.  It may have dependencies on other unreleased software,
 321 or other instabilities.  Please take care if you wish to install it.
 322 The update will eventually make its way into the next released Debian
 323 distribution.""";
 324                         else:
 325                             Subst["__STABLE_WARNING__"] = "";
 326                             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
 327                             utils.send_mail (mail_message);
 328                 if action:
 329                     self.Logger.log(["closing bugs"]+bugs);
 330
 331         else:                     # NMU
 332             summary += "Setting bugs to severity fixed: ";
 333             control_message = "";
 334             for bug in bugs:
 335                 summary += "%s " % (bug);
 336                 control_message += "tag %s + fixed\n" % (bug);
 337             if action and control_message != "":
 338                 Subst["__CONTROL_MESSAGE__"] = control_message;
 339                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed");
 340                 utils.send_mail (mail_message);
 341             if action:
 342                 self.Logger.log(["setting bugs to fixed"]+bugs);
 343         summary += "\n";
 344         return summary;
 345
 346     ###########################################################################
 347
 348     def announce (self, short_summary, action):
 349         Subst = self.Subst;
 350         Cnf = self.Cnf;
 351         changes = self.pkg.changes;
 352
 353         # Only do announcements for source uploads with a recent dpkg-dev installed
 354         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 355             return "";
 356
 357         lists_done = {};
 358         summary = "";
 359         Subst["__SHORT_SUMMARY__"] = short_summary;
 360
 361         for dist in changes["distribution"].keys():
 362             list = Cnf.Find("Suite::%s::Announce" % (dist));
 363             if list == "" or lists_done.has_key(list):
 364                 continue;
 365             lists_done[list] = 1;
 366             summary += "Announcing to %s\n" % (list);
 367
 368             if action:
 369                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 370                 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
 371                     Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]);
 372                 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce");
 373                 utils.send_mail (mail_message);
 374
 375         if Cnf.FindB("Dinstall::CloseBugs"):
 376             summary = self.close_bugs(summary, action);
 377
 378         return summary;
 379
 380     ###########################################################################
 381
 382     def accept (self, summary, short_summary):
 383         Cnf = self.Cnf;
 384         Subst = self.Subst;
 385         files = self.pkg.files;
 386
 387         print "Accepting."
 388         self.Logger.log(["Accepting changes",self.pkg.changes_file]);
 389
 390         self.dump_vars(Cnf["Dir::Queue::Accepted"]);
 391
 392         # Move all the files into the accepted directory
 393         utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
 394         file_keys = files.keys();
 395         for file in file_keys:
 396             utils.move(file, Cnf["Dir::Queue::Accepted"]);
 397             self.accept_bytes += float(files[file]["size"])
 398         self.accept_count += 1;
 399
 400         # Send accept mail, announce to lists, close bugs and check for
 401         # override disparities
 402         if not Cnf["Dinstall::Options::No-Mail"]:
 403             Subst["__SUITE__"] = "";
 404             Subst["__SUMMARY__"] = summary;
 405             mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted");
 406             utils.send_mail(mail_message)
 407             self.announce(short_summary, 1)
 408
 409         # Special support to enable clean auto-building of accepted packages
 410         self.projectB.query("BEGIN WORK");
 411         for suite in self.pkg.changes["distribution"].keys():
 412             if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
 413                 continue;
 414             suite_id = db_access.get_suite_id(suite);
 415             dest_dir = Cnf["Dir::AcceptedAutoBuild"];
 416             if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 417                 dest_dir = os.path.join(dest_dir, suite);
 418             for file in file_keys:
 419                 src = os.path.join(Cnf["Dir::Queue::Accepted"], file);
 420                 dest = os.path.join(dest_dir, file);
 421                 if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"):
 422                     # Copy it since the original won't be readable by www-data
 423                     utils.copy(src, dest);
 424                 else:
 425                     # Create a symlink to it
 426                     os.symlink(src, dest);
 427                 # Add it to the list of packages for later processing by apt-ftparchive
 428                 self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 429             # If the .orig.tar.gz is in the pool, create a symlink to
 430             # it (if one doesn't already exist)
 431             if self.pkg.orig_tar_id:
 432                 # Determine the .orig.tar.gz file name
 433                 for dsc_file in self.pkg.dsc_files.keys():
 434                     if dsc_file.endswith(".orig.tar.gz"):
 435                         filename = dsc_file;
 436                 dest = os.path.join(dest_dir, filename);
 437                 # If it doesn't exist, create a symlink
 438                 if not os.path.exists(dest):
 439                     # Find the .orig.tar.gz in the pool
 440                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 441                     ql = q.getresult();
 442                     if not ql:
 443                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 444                     src = os.path.join(ql[0][0], ql[0][1]);
 445                     os.symlink(src, dest);
 446                     # Add it to the list of packages for later processing by apt-ftparchive
 447                     self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest));
 448                 # if it does, update things to ensure it's not removed prematurely
 449                 else:
 450                     self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id));
 451
 452         self.projectB.query("COMMIT WORK");
 453
 454     ###########################################################################
 455
 456     def check_override (self):
 457         Subst = self.Subst;
 458         changes = self.pkg.changes;
 459         files = self.pkg.files;
 460         Cnf = self.Cnf;
 461
 462         # Abandon the check if:
 463         #  a) it's a non-sourceful upload
 464         #  b) override disparity checks have been disabled
 465         #  c) we're not sending mail
 466         if not changes["architecture"].has_key("source") or \
 467            not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
 468            Cnf["Dinstall::Options::No-Mail"]:
 469             return;
 470
 471         summary = "";
 472         for file in files.keys():
 473             if not files[file].has_key("new") and files[file]["type"] == "deb":
 474                 section = files[file]["section"];
 475                 override_section = files[file]["override section"];
 476                 if section.lower() != override_section.lower() and section != "-":
 477                     # Ignore this; it's a common mistake and not worth whining about
 478                     if section.lower() == "non-us/main" and override_section.lower() == "non-us":
 479                         continue;
 480                     summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section);
 481                 priority = files[file]["priority"];
 482                 override_priority = files[file]["override priority"];
 483                 if priority != override_priority and priority != "-":
 484                     summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority);
 485
 486         if summary == "":
 487             return;
 488
 489         Subst["__SUMMARY__"] = summary;
 490         mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity");
 491         utils.send_mail(mail_message);
 492
 493     ###########################################################################
 494
 495     def force_reject (self, files):
 496         """Forcefully move files from the current directory to the
 497            reject directory.  If any file already exists in the reject
 498            directory it will be moved to the morgue to make way for
 499            the new file."""
 500
 501         Cnf = self.Cnf
 502
 503         for file in files:
 504             # Skip any files which don't exist or which we don't have permission to copy.
 505             if os.access(file,os.R_OK) == 0:
 506                 continue;
 507             dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file);
 508             try:
 509                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 510             except OSError, e:
 511                 # File exists?  Let's try and move it to the morgue
 512                 if errno.errorcode[e.errno] == 'EEXIST':
 513                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file);
 514                     try:
 515                         morgue_file = utils.find_next_free(morgue_file);
 516                     except utils.tried_too_hard_exc:
 517                         # Something's either gone badly Pete Tong, or
 518                         # someone is trying to exploit us.
 519                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 520                         return;
 521                     utils.move(dest_file, morgue_file, perms=0660);
 522                     try:
 523                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 524                     except OSError, e:
 525                         # Likewise
 526                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 527                         return;
 528                 else:
 529                     raise;
 530             # If we got here, we own the destination file, so we can
 531             # safely overwrite it.
 532             utils.move(file, dest_file, 1, perms=0660);
 533
 534     ###########################################################################
 535
 536     def do_reject (self, manual = 0, reject_message = ""):
 537         # If we weren't given a manual rejection message, spawn an
 538         # editor so the user can add one in...
 539         if manual and not reject_message:
 540             temp_filename = tempfile.mktemp();
 541             fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
 542             os.close(fd);
 543             editor = os.environ.get("EDITOR","vi")
 544             answer = 'E';
 545             while answer == 'E':
 546                 os.system("%s %s" % (editor, temp_filename))
 547                 file = utils.open_file(temp_filename);
 548                 reject_message = "".join(file.readlines());
 549                 file.close();
 550                 print "Reject message:";
 551                 print utils.prefix_multi_line_string(reject_message,"  ",include_blank_lines=1);
 552                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 553                 answer = "XXX";
 554                 while prompt.find(answer) == -1:
 555                     answer = utils.our_raw_input(prompt);
 556                     m = re_default_answer.search(prompt);
 557                     if answer == "":
 558                         answer = m.group(1);
 559                     answer = answer[:1].upper();
 560             os.unlink(temp_filename);
 561             if answer == 'A':
 562                 return 1;
 563             elif answer == 'Q':
 564                 sys.exit(0);
 565
 566         print "Rejecting.\n"
 567
 568         Cnf = self.Cnf;
 569         Subst = self.Subst;
 570         pkg = self.pkg;
 571
 572         reason_filename = pkg.changes_file[:-8] + ".reason";
 573         reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
 574
 575         # Move all the files into the reject directory
 576         reject_files = pkg.files.keys() + [pkg.changes_file];
 577         self.force_reject(reject_files);
 578
 579         # If we fail here someone is probably trying to exploit the race
 580         # so let's just raise an exception ...
 581         if os.path.exists(reason_filename):
 582             os.unlink(reason_filename);
 583         reason_file = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 584
 585         if not manual:
 586             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 587             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 588             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 589             os.write(reason_file, reject_message);
 590             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 591         else:
 592             # Build up the rejection email
 593             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 594
 595             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 596             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 597             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 598             reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
 599             # Write the rejection email out as the <foo>.reason file
 600             os.write(reason_file, reject_mail_message);
 601
 602         os.close(reason_file);
 603
 604         # Send the rejection mail if appropriate
 605         if not Cnf["Dinstall::Options::No-Mail"]:
 606             utils.send_mail(reject_mail_message);
 607
 608         self.Logger.log(["rejected", pkg.changes_file]);
 609         return 0;
 610
 611     ################################################################################
 612
 613     # Ensure that source exists somewhere in the archive for the binary
 614     # upload being processed.
 615     #
 616     # (1) exact match                      => 1.0-3
 617     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 618     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 619
 620     def source_exists (self, package, source_version, suites = ["any"]):
 621         okay = 1
 622         for suite in suites:
 623             if suite == "any":
 624                 que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \
 625                     (package)
 626             else:
 627                 # source must exist in suite X, or in some other suite that's
 628                 # mapped to X, recursively... silent-maps are counted too,
 629                 # unreleased-maps aren't.
 630                 maps = self.Cnf.ValueList("SuiteMappings")[:]
 631                 maps.reverse()
 632                 maps = [ m.split() for m in maps ]
 633                 maps = [ (x[1], x[2]) for x in maps
 634                                 if x[0] == "map" or x[0] == "silent-map" ]
 635                 s = [suite]
 636                 for x in maps:
 637                         if x[1] in s and x[0] not in s:
 638                                 s.append(x[0])
 639
 640                 que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR "));
 641             q = self.projectB.query(que)
 642
 643             # Reduce the query results to a list of version numbers
 644             ql = map(lambda x: x[0], q.getresult());
 645
 646             # Try (1)
 647             if ql.count(source_version):
 648                 continue
 649
 650             # Try (2)
 651             orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version)
 652             if ql.count(orig_source_version):
 653                 continue
 654
 655             # Try (3)
 656             orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version)
 657             if ql.count(orig_source_version):
 658                 continue
 659
 660             # No source found...
 661             okay = 0
 662         return okay
 663
 664     ################################################################################
 665
 666     def in_override_p (self, package, component, suite, binary_type, file):
 667         files = self.pkg.files;
 668
 669         if binary_type == "": # must be source
 670             type = "dsc";
 671         else:
 672             type = binary_type;
 673
 674         # Override suite name; used for example with proposed-updates
 675         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 676             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 677
 678         # Avoid <undef> on unknown distributions
 679         suite_id = db_access.get_suite_id(suite);
 680         if suite_id == -1:
 681             return None;
 682         component_id = db_access.get_component_id(component);
 683         type_id = db_access.get_override_type_id(type);
 684
 685         # FIXME: nasty non-US speficic hack
 686         if component[:7].lower() == "non-us/":
 687             component = component[7:];
 688
 689         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 690                            % (package, suite_id, component_id, type_id));
 691         result = q.getresult();
 692         # If checking for a source package fall back on the binary override type
 693         if type == "dsc" and not result:
 694             deb_type_id = db_access.get_override_type_id("deb");
 695             udeb_type_id = db_access.get_override_type_id("udeb");
 696             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id"
 697                                % (package, suite_id, component_id, deb_type_id, udeb_type_id));
 698             result = q.getresult();
 699
 700         # Remember the section and priority so we can check them later if appropriate
 701         if result:
 702             files[file]["override section"] = result[0][0];
 703             files[file]["override priority"] = result[0][1];
 704
 705         return result;
 706
 707     ################################################################################
 708
 709     def reject (self, str, prefix="Rejected: "):
 710         if str:
 711             # Unlike other rejects we add new lines first to avoid trailing
 712             # new lines when this message is passed back up to a caller.
 713             if self.reject_message:
 714                 self.reject_message += "\n";
 715             self.reject_message += prefix + str;
 716
 717     ################################################################################
 718
 719     def cross_suite_version_check(self, query_result, file, new_version):
 720         """Ensure versions are newer than existing packages in target
 721         suites and that cross-suite version checking rules as
 722         set out in the conf file are satisfied."""
 723
 724         # Check versions for each target suite
 725         for target_suite in self.pkg.changes["distribution"].keys():
 726             must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
 727             must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
 728             # Enforce "must be newer than target suite" even if conffile omits it
 729             if target_suite not in must_be_newer_than:
 730                 must_be_newer_than.append(target_suite);
 731             for entry in query_result:
 732                 existent_version = entry[0];
 733                 suite = entry[1];
 734                 if suite in must_be_newer_than and \
 735                    apt_pkg.VersionCompare(new_version, existent_version) != 1:
 736                     self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 737                 if suite in must_be_older_than and \
 738                    apt_pkg.VersionCompare(new_version, existent_version) != -1:
 739                     self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
 740
 741     ################################################################################
 742
 743     def check_binary_against_db(self, file):
 744         self.reject_message = "";
 745         files = self.pkg.files;
 746
 747         # Ensure version is sane
 748         q = self.projectB.query("""
 749 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
 750                                      architecture a
 751  WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
 752    AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
 753                                 % (files[file]["package"],
 754                                    files[file]["architecture"]));
 755         self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
 756
 757         # Check for any existing copies of the file
 758         q = self.projectB.query("""
 759 SELECT b.id FROM binaries b, architecture a
 760  WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
 761    AND a.id = b.architecture"""
 762                                 % (files[file]["package"],
 763                                    files[file]["version"],
 764                                    files[file]["architecture"]))
 765         if q.getresult():
 766             self.reject("%s: can not overwrite existing copy already in the archive." % (file));
 767
 768         return self.reject_message;
 769
 770     ################################################################################
 771
 772     def check_source_against_db(self, file):
 773         self.reject_message = "";
 774         dsc = self.pkg.dsc;
 775
 776         # Ensure version is sane
 777         q = self.projectB.query("""
 778 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
 779  WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
 780         self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
 781
 782         return self.reject_message;
 783
 784     ################################################################################
 785
 786     # **WARNING**
 787     # NB: this function can remove entries from the 'files' index [if
 788     # the .orig.tar.gz is a duplicate of the one in the archive]; if
 789     # you're iterating over 'files' and call this function as part of
 790     # the loop, be sure to add a check to the top of the loop to
 791     # ensure you haven't just tried to derefernece the deleted entry.
 792     # **WARNING**
 793
 794     def check_dsc_against_db(self, file):
 795         self.reject_message = "";
 796         files = self.pkg.files;
 797         dsc_files = self.pkg.dsc_files;
 798         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 799         orig_tar_gz = None;
 800
 801         # Try and find all files mentioned in the .dsc.  This has
 802         # to work harder to cope with the multiple possible
 803         # locations of an .orig.tar.gz.
 804         for dsc_file in dsc_files.keys():
 805             found = None;
 806             if files.has_key(dsc_file):
 807                 actual_md5 = files[dsc_file]["md5sum"];
 808                 actual_size = int(files[dsc_file]["size"]);
 809                 found = "%s in incoming" % (dsc_file)
 810                 # Check the file does not already exist in the archive
 811                 q = self.projectB.query("SELECT size, md5sum, filename FROM files WHERE filename LIKE '%%%s%%'" % (dsc_file));
 812
 813                 ql = q.getresult();
 814                 # Strip out anything that isn't '%s' or '/%s$'
 815                 for i in ql:
 816                     if i[2] != dsc_file and i[2][-(len(dsc_file)+1):] != '/'+dsc_file:
 817                         self.Logger.log(["check_dsc_against_db",i[2],dsc_file]);
 818                         ql.remove(i);
 819
 820                 # "[katie] has not broken them.  [katie] has fixed a
 821                 # brokenness.  Your crappy hack exploited a bug in
 822                 # the old dinstall.
 823                 #
 824                 # "(Come on!  I thought it was always obvious that
 825                 # one just doesn't release different files with
 826                 # the same name and version.)"
 827                 #                        -- ajk@ on d-devel@l.d.o
 828
 829                 if ql:
 830                     # Ignore exact matches for .orig.tar.gz
 831                     match = 0;
 832                     if dsc_file.endswith(".orig.tar.gz"):
 833                         for i in ql:
 834                             if files.has_key(dsc_file) and \
 835                                int(files[dsc_file]["size"]) == int(i[0]) and \
 836                                files[dsc_file]["md5sum"] == i[1]:
 837                                 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
 838                                 del files[dsc_file];
 839                                 match = 1;
 840
 841                     if not match:
 842                         self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 843             elif dsc_file.endswith(".orig.tar.gz"):
 844                 # Check in the pool
 845                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file));
 846                 ql = q.getresult();
 847                 # Strip out anything that isn't '%s' or '/%s$'
 848                 for i in ql:
 849                     if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
 850                         self.Logger.log(["check_dsc_against_db",i[1],dsc_file]);
 851                         ql.remove(i);
 852
 853                 if ql:
 854                     # Unfortunately, we make get more than one
 855                     # match here if, for example, the package was
 856                     # in potato but had a -sa upload in woody.  So
 857                     # we need to choose the right one.
 858
 859                     x = ql[0]; # default to something sane in case we don't match any or have only one
 860
 861                     if len(ql) > 1:
 862                         for i in ql:
 863                             old_file = i[0] + i[1];
 864                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 865                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 866                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 867                                 x = i;
 868                             else:
 869                                 legacy_source_untouchable[i[3]] = "";
 870
 871                     old_file = x[0] + x[1];
 872                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 873                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 874                     found = old_file;
 875                     suite_type = x[2];
 876                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 877                     # See install() in katie...
 878                     self.pkg.orig_tar_id = x[3];
 879                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 880                         self.pkg.orig_tar_location = "legacy";
 881                     else:
 882                         self.pkg.orig_tar_location = x[4];
 883                 else:
 884                     # Not there? Check the queue directories...
 885
 886                     in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file);
 887                     # See process_it() in jennifer for explanation of this
 888                     if os.path.exists(in_unchecked):
 889                         return (self.reject_message, in_unchecked);
 890                     else:
 891                         for dir in [ "Accepted", "New", "Byhand" ]:
 892                             in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file);
 893                             if os.path.exists(in_otherdir):
 894                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 895                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 896                                 found = in_otherdir;
 897
 898                     if not found:
 899                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 900                         continue;
 901             else:
 902                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 903                 continue;
 904             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 905                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 906             if actual_size != int(dsc_files[dsc_file]["size"]):
 907                 self.reject("size for %s doesn't match %s." % (found, file));
 908
 909         return (self.reject_message, orig_tar_gz);
 910
 911     def do_query(self, q):
 912         sys.stderr.write("query: \"%s\" ... " % (q));
 913         before = time.time();
 914         r = self.projectB.query(q);
 915         time_diff = time.time()-before;
 916         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
 917         return r;