katie.py

   1 #!/usr/bin/env python
   2
   3 # Utility functions for katie
   4 # Copyright (C) 2001  James Troup <james@nocrew.org>
   5 # $Id: katie.py,v 1.15 2002-04-21 15:38:29 troup Exp $
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ###############################################################################
  22
  23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
  24 import utils, db_access;
  25 import apt_inst, apt_pkg;
  26
  27 from types import *;
  28 from string import lower;
  29
  30 ###############################################################################
  31
  32 re_isanum = re.compile (r"^\d+$");
  33 re_default_answer = re.compile(r"\[(.*)\]");
  34 re_fdnic = re.compile("\n\n");
  35 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
  36 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
  37
  38 ###############################################################################
  39
  40 # Convenience wrapper to carry around all the package information in
  41
  42 class Pkg:
  43     def __init__(self, **kwds):
  44         self.__dict__.update(kwds);
  45
  46     def update(self, **kwds):
  47         self.__dict__.update(kwds);
  48
  49 ###############################################################################
  50
  51 class nmu_p:
  52     # Read in the group maintainer override file
  53     def __init__ (self, Cnf):
  54         self.group_maint = {};
  55         self.Cnf = Cnf;
  56         if Cnf.get("Dinstall::GroupOverrideFilename"):
  57             filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
  58             file = utils.open_file(filename);
  59             for line in file.readlines():
  60                 line = lower(string.strip(utils.re_comments.sub('', line)));
  61                 if line != "":
  62                     self.group_maint[line] = 1;
  63             file.close();
  64
  65     def is_an_nmu (self, pkg):
  66         Cnf = self.Cnf;
  67         changes = pkg.changes;
  68         dsc = pkg.dsc;
  69
  70         (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (lower(dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"])));
  71         # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
  72         if dsc_name == lower(changes["maintainername"]) and \
  73            (changes["changedby822"] == "" or lower(changes["changedbyname"]) == dsc_name):
  74             return 0;
  75
  76         if dsc.has_key("uploaders"):
  77             uploaders = string.split(lower(dsc["uploaders"]), ",");
  78             uploadernames = {};
  79             for i in uploaders:
  80                 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
  81                 uploadernames[name] = "";
  82             if uploadernames.has_key(lower(changes["changedbyname"])):
  83                 return 0;
  84
  85         # Some group maintained packages (e.g. Debian QA) are never NMU's
  86         if self.group_maint.has_key(lower(changes["maintaineremail"])):
  87             return 0;
  88
  89         return 1;
  90
  91 ###############################################################################
  92
  93 class Katie:
  94
  95     def __init__(self, Cnf):
  96         self.Cnf = Cnf;
  97         self.values = {};
  98         # Read in the group-maint override file
  99         self.nmu = nmu_p(Cnf);
 100         self.accept_count = 0;
 101         self.accept_bytes = 0L;
 102         self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
 103                        legacy_source_untouchable = {});
 104
 105         # Initialize the substitution template mapping global
 106         Subst = self.Subst = {};
 107         Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
 108         Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
 109         Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
 110         Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 111
 112         self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
 113         db_access.init(Cnf, self.projectB);
 114
 115     ###########################################################################
 116
 117     def init_vars (self):
 118         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 119             exec "self.pkg.%s.clear();" % (i);
 120         self.pkg.orig_tar_id = None;
 121         self.pkg.orig_tar_location = "";
 122
 123     ###########################################################################
 124
 125     def update_vars (self):
 126         dump_filename = self.pkg.changes_file[:-8]+".katie";
 127         dump_file = utils.open_file(dump_filename);
 128         p = cPickle.Unpickler(dump_file);
 129         for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
 130             exec "self.pkg.%s.update(p.load());" % (i);
 131         for i in [ "orig_tar_id", "orig_tar_location" ]:
 132             exec "self.pkg.%s = p.load();" % (i);
 133         dump_file.close();
 134
 135     ###########################################################################
 136
 137     # This could just dump the dictionaries as is, but I'd like to avoid
 138     # this so there's some idea of what katie & lisa use from jennifer
 139
 140     def dump_vars(self, dest_dir):
 141         for i in [ "changes", "dsc", "files", "dsc_files",
 142                    "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
 143             exec "%s = self.pkg.%s;" % (i,i);
 144         dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
 145         dump_file = utils.open_file(dump_filename, 'w');
 146         p = cPickle.Pickler(dump_file, 1);
 147         for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
 148             exec "%s = {}" % i;
 149         ## files
 150         for file in files.keys():
 151             d_files[file] = {};
 152             for i in [ "package", "version", "architecture", "type", "size",
 153                        "md5sum", "component", "location id", "source package",
 154                        "source version", "maintainer", "dbtype", "files id",
 155                        "new", "section", "priority", "oldfiles", "othercomponents" ]:
 156                 if files[file].has_key(i):
 157                     d_files[file][i] = files[file][i];
 158         ## changes
 159         # Mandatory changes fields
 160         for i in [ "distribution", "source", "architecture", "version", "maintainer",
 161                    "urgency", "fingerprint", "changedby822", "changedbyname",
 162                    "maintainername", "maintaineremail", "closes" ]:
 163             d_changes[i] = changes[i];
 164         # Optional changes fields
 165         # FIXME: changes should be mandatory
 166         for i in [ "changed-by", "maintainer822", "filecontents", "format", "changes" ]:
 167             if changes.has_key(i):
 168                 d_changes[i] = changes[i];
 169         ## dsc
 170         for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
 171             if dsc.has_key(i):
 172                 d_dsc[i] = dsc[i];
 173         ## dsc_files
 174         for file in dsc_files.keys():
 175             d_dsc_files[file] = {};
 176             # Mandatory dsc_files fields
 177             for i in [ "size", "md5sum" ]:
 178                 d_dsc_files[file][i] = dsc_files[file][i];
 179             # Optional dsc_files fields
 180             for i in [ "files id" ]:
 181                 if dsc_files[file].has_key(i):
 182                     d_dsc_files[file][i] = dsc_files[file][i];
 183
 184         for i in [ d_changes, d_dsc, d_files, d_dsc_files,
 185                    legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
 186             p.dump(i);
 187         dump_file.close();
 188
 189     ###########################################################################
 190
 191     # Set up the per-package template substitution mappings
 192
 193     def update_subst (self, reject_message = ""):
 194         Subst = self.Subst;
 195         changes = self.pkg.changes;
 196         # If jennifer crashed out in the right place, architecture may still be a string.
 197         if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
 198             changes["architecture"] = { "Unknown" : "" };
 199         # and maintainer822 may not exist.
 200         if not changes.has_key("maintainer822"):
 201             changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
 202
 203         Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
 204         Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
 205         Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
 206
 207         # For source uploads the Changed-By field wins; otherwise Maintainer wins.
 208         if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
 209             Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
 210             Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
 211             Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
 212         else:
 213             Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
 214             Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
 215             Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
 216         if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
 217             Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
 218
 219         Subst["__REJECT_MESSAGE__"] = reject_message;
 220         Subst["__SOURCE__"] = changes.get("source", "Unknown");
 221         Subst["__VERSION__"] = changes.get("version", "Unknown");
 222
 223     ###########################################################################
 224
 225     def build_summaries(self):
 226         changes = self.pkg.changes;
 227         files = self.pkg.files;
 228
 229         byhand = summary = new = "";
 230
 231         # changes["distribution"] may not exist in corner cases
 232         # (e.g. unreadable changes files)
 233         if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
 234             changes["distribution"] = {};
 235
 236         file_keys = files.keys();
 237         file_keys.sort();
 238         for file in file_keys:
 239             if files[file].has_key("byhand"):
 240                 byhand = 1
 241                 summary = summary + file + " byhand\n"
 242             elif files[file].has_key("new"):
 243                 new = 1
 244                 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
 245                 if files[file].has_key("othercomponents"):
 246                     summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
 247                 if files[file]["type"] == "deb":
 248                     summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
 249             else:
 250                 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
 251                 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
 252                 summary = summary + file + "\n  to " + destination + "\n"
 253
 254         short_summary = summary;
 255
 256         # This is for direport's benefit...
 257         f = re_fdnic.sub("\n .\n", changes.get("changes",""));
 258
 259         if byhand or new:
 260             summary = summary + "Changes: " + f;
 261
 262         summary = summary + self.announce(short_summary, 0)
 263
 264         return (summary, short_summary);
 265
 266     ###########################################################################
 267
 268     def announce (self, short_summary, action):
 269         Subst = self.Subst;
 270         Cnf = self.Cnf;
 271         changes = self.pkg.changes;
 272         dsc = self.pkg.dsc;
 273
 274         # Only do announcements for source uploads with a recent dpkg-dev installed
 275         if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
 276             return ""
 277
 278         lists_done = {}
 279         summary = ""
 280         Subst["__SHORT_SUMMARY__"] = short_summary;
 281
 282         for dist in changes["distribution"].keys():
 283             list = Cnf.Find("Suite::%s::Announce" % (dist))
 284             if list == "" or lists_done.has_key(list):
 285                 continue
 286             lists_done[list] = 1
 287             summary = summary + "Announcing to %s\n" % (list)
 288
 289             if action:
 290                 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
 291                 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.announce","r").read());
 292                 utils.send_mail (mail_message, "")
 293
 294         bugs = changes["closes"].keys()
 295         bugs.sort()
 296         if not self.nmu.is_an_nmu(self.pkg):
 297             summary = summary + "Closing bugs: "
 298             for bug in bugs:
 299                 summary = summary + "%s " % (bug)
 300                 if action:
 301                     Subst["__BUG_NUMBER__"] = bug;
 302                     if changes["distribution"].has_key("stable"):
 303                         Subst["__STABLE_WARNING__"] = """
 304     Note that this package is not part of the released stable Debian
 305     distribution.  It may have dependencies on other unreleased software,
 306     or other instabilities.  Please take care if you wish to install it.
 307     The update will eventually make its way into the next released Debian
 308     distribution."""
 309                     else:
 310                         Subst["__STABLE_WARNING__"] = "";
 311                     mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.bug-close","r").read());
 312                     utils.send_mail (mail_message, "")
 313             if action:
 314                 self.Logger.log(["closing bugs"]+bugs);
 315         else:                     # NMU
 316             summary = summary + "Setting bugs to severity fixed: "
 317             control_message = ""
 318             for bug in bugs:
 319                 summary = summary + "%s " % (bug)
 320                 control_message = control_message + "tag %s + fixed\n" % (bug)
 321             if action and control_message != "":
 322                 Subst["__CONTROL_MESSAGE__"] = control_message;
 323                 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.bug-nmu-fixed","r").read());
 324                 utils.send_mail (mail_message, "")
 325             if action:
 326                 self.Logger.log(["setting bugs to fixed"]+bugs);
 327         summary = summary + "\n"
 328
 329         return summary
 330
 331     ###########################################################################
 332
 333     def accept (self, summary, short_summary):
 334         Cnf = self.Cnf;
 335         Subst = self.Subst;
 336         files = self.pkg.files;
 337
 338         print "Accepting."
 339         self.Logger.log(["Accepting changes",self.pkg.changes_file]);
 340
 341         self.dump_vars(Cnf["Dir::QueueAcceptedDir"]);
 342
 343         # Move all the files into the accepted directory
 344         utils.move(self.pkg.changes_file, Cnf["Dir::QueueAcceptedDir"]);
 345         file_keys = files.keys();
 346         for file in file_keys:
 347             utils.move(file, Cnf["Dir::QueueAcceptedDir"]);
 348             self.accept_bytes = self.accept_bytes + float(files[file]["size"])
 349         self.accept_count = self.accept_count + 1;
 350
 351         # Send accept mail, announce to lists, close bugs and check for
 352         # override disparities
 353         if not Cnf["Dinstall::Options::No-Mail"]:
 354             Subst["__SUITE__"] = "";
 355             Subst["__SUMMARY__"] = summary;
 356             mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.accepted","r").read());
 357             utils.send_mail(mail_message, "")
 358             self.announce(short_summary, 1)
 359
 360         ## FIXME: this should go away to some Debian specific file
 361         # If we're accepting something for unstable do extra work to
 362         # make it auto-buildable from accepted
 363         if self.pkg.changes["distribution"].has_key("unstable"):
 364             self.projectB.query("BEGIN WORK");
 365             # Add it to the list of packages for later processing by apt-ftparchive
 366             for file in file_keys:
 367                 if files[file]["type"] == "dsc" or files[file]["type"] == "deb":
 368                     filename = os.path.join(Cnf["Dir::QueueAcceptedDir"], file);
 369                     self.projectB.query("INSERT INTO unstable_accepted (filename) VALUES ('%s')" % (filename));
 370             # If the .orig.tar.gz is in the pool, create a symlink (if
 371             # one doesn't already exist)
 372             if self.pkg.orig_tar_id:
 373                 # Determine the .orig.tar.gz file name
 374                 for dsc_file in self.pkg.dsc_files.keys():
 375                     if dsc_file[-12:] == ".orig.tar.gz":
 376                         filename = dsc_file;
 377                 dest = os.path.join(Cnf["Dir::QueueAcceptedDir"],filename);
 378                 # If it doesn't exist, create a symlink
 379                 if not os.path.exists(dest):
 380                     # Find the .orig.tar.gz in the pool
 381                     q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
 382                     ql = q.getresult();
 383                     if not ql:
 384                         utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
 385                     src = os.path.join(ql[0][0], ql[0][1]);
 386                     os.symlink(src, dest);
 387             self.projectB.query("COMMIT WORK");
 388
 389     ###########################################################################
 390
 391     def check_override (self):
 392         Subst = self.Subst;
 393         changes = self.pkg.changes;
 394         files = self.pkg.files;
 395
 396         # Only check section & priority on sourceful uploads
 397         if not changes["architecture"].has_key("source"):
 398             return;
 399
 400         summary = "";
 401         for file in files.keys():
 402             if not files[file].has_key("new") and files[file]["type"] == "deb":
 403                 section = files[file]["section"];
 404                 override_section = files[file]["override section"];
 405                 if lower(section) != lower(override_section) and section != "-":
 406                     # Ignore this; it's a common mistake and not worth whining about
 407                     if lower(section) == "non-us/main" and lower(override_section) == "non-us":
 408                         continue;
 409                     summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
 410                 priority = files[file]["priority"];
 411                 override_priority = files[file]["override priority"];
 412                 if priority != override_priority and priority != "-":
 413                     summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
 414
 415         if summary == "":
 416             return;
 417
 418         Subst["__SUMMARY__"] = summary;
 419         mail_message = utils.TemplateSubst(Subst,utils.open_file(self.Cnf["Dir::TemplatesDir"]+"/jennifer.override-disparity").read());
 420         utils.send_mail (mail_message, "");
 421
 422     ###########################################################################
 423
 424     def force_move (self, files):
 425         """Forcefully move files from the current directory to the reject
 426            directory.  If any file already exists it will be moved to the
 427            morgue to make way for the new file."""
 428
 429         Cnf = self.Cnf
 430
 431         for file in files:
 432             # Skip any files which don't exist or which we don't have permission to copy.
 433             if os.access(file,os.R_OK) == 0:
 434                 continue;
 435             dest_file = os.path.join(Cnf["Dir::QueueRejectDir"], file);
 436             try:
 437                 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 438             except OSError, e:
 439                 # File exists?  Let's try and move it to the morgue
 440                 if errno.errorcode[e.errno] == 'EEXIST':
 441                     morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueRejectDir"],file);
 442                     try:
 443                         morgue_file = utils.find_next_free(morgue_file);
 444                     except utils.tried_too_hard_exc:
 445                         # Something's either gone badly Pete Tong, or
 446                         # someone is trying to exploit us.
 447                         utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
 448                         return;
 449                     utils.move(dest_file, morgue_file, perms=0660);
 450                     try:
 451                         os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 452                     except OSError, e:
 453                         # Likewise
 454                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
 455                         return;
 456                 else:
 457                     raise;
 458             # If we got here, we own the destination file, so we can
 459             # safely overwrite it.
 460             utils.move(file, dest_file, 1, perms=0660);
 461
 462
 463     ###########################################################################
 464
 465     def do_reject (self, manual = 0, reject_message = ""):
 466         # If we weren't given a manual rejection message, spawn an
 467         # editor so the user can add one in...
 468         if manual and not reject_message:
 469             temp_filename = tempfile.mktemp();
 470             fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
 471             os.close(fd);
 472             editor = os.environ.get("EDITOR","vi")
 473             answer = 'E';
 474             while answer == 'E':
 475                 os.system("%s %s" % (editor, temp_filename))
 476                 file = utils.open_file(temp_filename);
 477                 reject_message = string.join(file.readlines());
 478                 file.close();
 479                 print "Reject message:";
 480                 print utils.prefix_multi_line_string(reject_message,"  ");
 481                 prompt = "[R]eject, Edit, Abandon, Quit ?"
 482                 answer = "XXX";
 483                 while string.find(prompt, answer) == -1:
 484                     answer = utils.our_raw_input(prompt);
 485                     m = re_default_answer.search(prompt);
 486                     if answer == "":
 487                         answer = m.group(1);
 488                     answer = string.upper(answer[:1]);
 489             os.unlink(temp_filename);
 490             if answer == 'A':
 491                 return 1;
 492             elif answer == 'Q':
 493                 sys.exit(0);
 494
 495         print "Rejecting.\n"
 496
 497         Cnf = self.Cnf;
 498         Subst = self.Subst;
 499         pkg = self.pkg;
 500
 501         reason_filename = pkg.changes_file[:-8] + ".reason";
 502         reject_filename = Cnf["Dir::QueueRejectDir"] + '/' + reason_filename;
 503
 504         # Move all the files into the reject directory
 505         reject_files = pkg.files.keys() + [pkg.changes_file];
 506         self.force_move(reject_files);
 507
 508         # If we fail here someone is probably trying to exploit the race
 509         # so let's just raise an exception ...
 510         if os.path.exists(reject_filename):
 511             os.unlink(reject_filename);
 512         fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
 513
 514         if not manual:
 515             Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
 516             Subst["__MANUAL_REJECT_MESSAGE__"] = "";
 517             Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
 518             os.write(fd, reject_message);
 519             os.close(fd);
 520             reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.rejected").read());
 521         else:
 522             # Build up the rejection email
 523             user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
 524
 525             Subst["__REJECTOR_ADDRESS__"] = user_email_address;
 526             Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
 527             Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
 528             reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.rejected").read());
 529
 530             # Write the rejection email out as the <foo>.reason file
 531             os.write(fd, reject_mail_message);
 532             os.close(fd);
 533
 534         # Send the rejection mail if appropriate
 535         if not Cnf["Dinstall::Options::No-Mail"]:
 536             utils.send_mail (reject_mail_message, "");
 537
 538         self.Logger.log(["rejected", pkg.changes_file]);
 539         return 0;
 540
 541     ################################################################################
 542
 543     # Ensure that source exists somewhere in the archive for the binary
 544     # upload being processed.
 545     #
 546     # (1) exact match                      => 1.0-3
 547     # (2) Bin-only NMU of an MU            => 1.0-3.0.1
 548     # (3) Bin-only NMU of a sourceful-NMU  => 1.0-3.1.1
 549
 550     def source_exists (self, package, source_version):
 551         q = self.projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
 552
 553         # Reduce the query results to a list of version numbers
 554         ql = map(lambda x: x[0], q.getresult());
 555
 556         # Try (1)
 557         if ql.count(source_version):
 558             return 1;
 559
 560         # Try (2)
 561         orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
 562         if ql.count(orig_source_version):
 563             return 1;
 564
 565         # Try (3)
 566         orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
 567         if ql.count(orig_source_version):
 568             return 1;
 569
 570         # No source found...
 571         return 0;
 572
 573     ################################################################################
 574
 575     def in_override_p (self, package, component, suite, binary_type, file):
 576         files = self.pkg.files;
 577
 578         if binary_type == "": # must be source
 579             type = "dsc";
 580         else:
 581             type = binary_type;
 582
 583         # Override suite name; used for example with proposed-updates
 584         if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
 585             suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
 586
 587         # Avoid <undef> on unknown distributions
 588         suite_id = db_access.get_suite_id(suite);
 589         if suite_id == -1:
 590             return None;
 591         component_id = db_access.get_component_id(component);
 592         type_id = db_access.get_override_type_id(type);
 593
 594         # FIXME: nasty non-US speficic hack
 595         if lower(component[:7]) == "non-us/":
 596             component = component[7:];
 597
 598         q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 599                            % (package, suite_id, component_id, type_id));
 600         result = q.getresult();
 601         # If checking for a source package fall back on the binary override type
 602         if type == "dsc" and not result:
 603             type_id = db_access.get_override_type_id("deb");
 604             q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
 605                                % (package, suite_id, component_id, type_id));
 606             result = q.getresult();
 607
 608         # Remember the section and priority so we can check them later if appropriate
 609         if result != []:
 610             files[file]["override section"] = result[0][0];
 611             files[file]["override priority"] = result[0][1];
 612
 613         return result;
 614
 615     ################################################################################
 616
 617     def reject (self, str, prefix="Rejected: "):
 618         if str:
 619             # Unlike other rejects we add new lines first to avoid trailing
 620             # new lines when this message is passed back up to a caller.
 621             if self.reject_message:
 622                 self.reject_message = self.reject_message + "\n";
 623             self.reject_message = self.reject_message + prefix + str;
 624
 625     def check_binaries_against_db(self, file, suite):
 626         self.reject_message = "";
 627         files = self.pkg.files;
 628
 629         # Find any old binary packages
 630         q = self.projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name  FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
 631                            % (files[file]["package"], suite, files[file]["architecture"]))
 632         for oldfile in q.dictresult():
 633             files[file]["oldfiles"][suite] = oldfile;
 634             # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
 635             if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
 636                 self.reject("%s: old version (%s) >= new version (%s)." % (file, oldfile["version"], files[file]["version"]));
 637         # Check for any existing copies of the file
 638         q = self.projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
 639         if q.getresult() != []:
 640             self.reject("can not overwrite existing copy of '%s' already in the archive." % (file));
 641
 642         return self.reject_message;
 643
 644     ################################################################################
 645
 646     def check_source_against_db(self, file):
 647         """Ensure source is newer than existing source in target suites."""
 648         self.reject_message = "";
 649         changes = self.pkg.changes;
 650         dsc = self.pkg.dsc;
 651
 652         package = dsc.get("source");
 653         new_version = dsc.get("version");
 654         for suite in changes["distribution"].keys():
 655             q = self.projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
 656                                % (package, suite));
 657             ql = map(lambda x: x[0], q.getresult());
 658             for old_version in ql:
 659                 if apt_pkg.VersionCompare(new_version, old_version) != 1:
 660                     self.reject("%s: Old version `%s' >= new version `%s'." % (file, old_version, new_version));
 661         return self.reject_message;
 662
 663     ################################################################################
 664
 665     def check_dsc_against_db(self, file):
 666         self.reject_message = "";
 667         files = self.pkg.files;
 668         dsc_files = self.pkg.dsc_files;
 669         legacy_source_untouchable = self.pkg.legacy_source_untouchable;
 670         orig_tar_gz = None;
 671
 672         # Try and find all files mentioned in the .dsc.  This has
 673         # to work harder to cope with the multiple possible
 674         # locations of an .orig.tar.gz.
 675         for dsc_file in dsc_files.keys():
 676             found = None;
 677             if files.has_key(dsc_file):
 678                 actual_md5 = files[dsc_file]["md5sum"];
 679                 actual_size = int(files[dsc_file]["size"]);
 680                 found = "%s in incoming" % (dsc_file)
 681                 # Check the file does not already exist in the archive
 682                 q = self.projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
 683
 684                 # "It has not broken them.  It has fixed a
 685                 # brokenness.  Your crappy hack exploited a bug in
 686                 # the old dinstall.
 687                 #
 688                 # "(Come on!  I thought it was always obvious that
 689                 # one just doesn't release different files with
 690                 # the same name and version.)"
 691                 #                        -- ajk@ on d-devel@l.d.o
 692
 693                 if q.getresult() != []:
 694                     self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
 695             elif dsc_file[-12:] == ".orig.tar.gz":
 696                 # Check in the pool
 697                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
 698                 ql = q.getresult();
 699
 700                 if ql != []:
 701                     # Unfortunately, we make get more than one
 702                     # match here if, for example, the package was
 703                     # in potato but had a -sa upload in woody.  So
 704                     # we need to choose the right one.
 705
 706                     x = ql[0]; # default to something sane in case we don't match any or have only one
 707
 708                     if len(ql) > 1:
 709                         for i in ql:
 710                             old_file = i[0] + i[1];
 711                             actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 712                             actual_size = os.stat(old_file)[stat.ST_SIZE];
 713                             if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
 714                                 x = i;
 715                             else:
 716                                 legacy_source_untouchable[i[3]] = "";
 717
 718                     old_file = x[0] + x[1];
 719                     actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
 720                     actual_size = os.stat(old_file)[stat.ST_SIZE];
 721                     found = old_file;
 722                     suite_type = x[2];
 723                     dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
 724                     # See install() in katie...
 725                     self.pkg.orig_tar_id = x[3];
 726                     if suite_type == "legacy" or suite_type == "legacy-mixed":
 727                         self.pkg.orig_tar_location = "legacy";
 728                     else:
 729                         self.pkg.orig_tar_location = x[4];
 730                 else:
 731                     # Not there? Check the queue directories...
 732
 733                     in_unchecked = os.path.join(self.Cnf["Dir::QueueUncheckedDir"],dsc_file);
 734                     # See process_it() in jennifer for explanation of this
 735                     if os.path.exists(in_unchecked):
 736                         return (self.reject_message, in_unchecked);
 737                     else:
 738                         for dir in [ "Accepted", "New", "Byhand" ]:
 739                             in_otherdir = os.path.join(self.Cnf["Dir::Queue%sDir" % (dir)],dsc_file);
 740                             if os.path.exists(in_otherdir):
 741                                 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
 742                                 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
 743                                 found = in_otherdir;
 744
 745                     if not found:
 746                         self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
 747                         continue;
 748             else:
 749                 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
 750                 continue;
 751             if actual_md5 != dsc_files[dsc_file]["md5sum"]:
 752                 self.reject("md5sum for %s doesn't match %s." % (found, file));
 753             if actual_size != int(dsc_files[dsc_file]["size"]):
 754                 self.reject("size for %s doesn't match %s." % (found, file));
 755
 756         return (self.reject_message, orig_tar_gz);
 757
 758     def do_query(self, q):
 759         sys.stderr.write("query: \"%s\" ... " % (q));
 760         before = time.time();
 761         r = self.projectB.query(q);
 762         time_diff = time.time()-before;
 763         sys.stderr.write("took %.3f seconds.\n" % (time_diff));
 764         return r;