3 # Utility functions for katie
4 # Copyright (C) 2001, 2002 James Troup <james@nocrew.org>
5 # $Id: katie.py,v 1.22 2002-05-19 00:47:07 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ###############################################################################
23 import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
24 import utils, db_access;
25 import apt_inst, apt_pkg;
28 from string import lower;
30 ###############################################################################
32 re_isanum = re.compile (r"^\d+$");
33 re_default_answer = re.compile(r"\[(.*)\]");
34 re_fdnic = re.compile("\n\n");
35 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
36 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
38 ###############################################################################
40 # Convenience wrapper to carry around all the package information in
43 def __init__(self, **kwds):
44 self.__dict__.update(kwds);
46 def update(self, **kwds):
47 self.__dict__.update(kwds);
49 ###############################################################################
52 # Read in the group maintainer override file
53 def __init__ (self, Cnf):
54 self.group_maint = {};
56 if Cnf.get("Dinstall::GroupOverrideFilename"):
57 filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"];
58 file = utils.open_file(filename);
59 for line in file.readlines():
60 line = lower(string.strip(utils.re_comments.sub('', line)));
62 self.group_maint[line] = 1;
65 def is_an_nmu (self, pkg):
67 changes = pkg.changes;
70 (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (lower(dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"])));
71 # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
72 if dsc_name == lower(changes["maintainername"]) and \
73 (changes["changedby822"] == "" or lower(changes["changedbyname"]) == dsc_name):
76 if dsc.has_key("uploaders"):
77 uploaders = string.split(lower(dsc["uploaders"]), ",");
80 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
81 uploadernames[name] = "";
82 if uploadernames.has_key(lower(changes["changedbyname"])):
85 # Some group maintained packages (e.g. Debian QA) are never NMU's
86 if self.group_maint.has_key(lower(changes["maintaineremail"])):
91 ###############################################################################
95 def __init__(self, Cnf):
98 # Read in the group-maint override file
99 self.nmu = nmu_p(Cnf);
100 self.accept_count = 0;
101 self.accept_bytes = 0L;
102 self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
103 legacy_source_untouchable = {});
105 # Initialize the substitution template mapping global
106 Subst = self.Subst = {};
107 Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
108 Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
109 Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
110 Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
112 self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
113 db_access.init(Cnf, self.projectB);
115 ###########################################################################
117 def init_vars (self):
118 for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
119 exec "self.pkg.%s.clear();" % (i);
120 self.pkg.orig_tar_id = None;
121 self.pkg.orig_tar_location = "";
123 ###########################################################################
125 def update_vars (self):
126 dump_filename = self.pkg.changes_file[:-8]+".katie";
127 dump_file = utils.open_file(dump_filename);
128 p = cPickle.Unpickler(dump_file);
129 for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]:
130 exec "self.pkg.%s.update(p.load());" % (i);
131 for i in [ "orig_tar_id", "orig_tar_location" ]:
132 exec "self.pkg.%s = p.load();" % (i);
135 ###########################################################################
137 # This could just dump the dictionaries as is, but I'd like to avoid
138 # this so there's some idea of what katie & lisa use from jennifer
140 def dump_vars(self, dest_dir):
141 for i in [ "changes", "dsc", "files", "dsc_files",
142 "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]:
143 exec "%s = self.pkg.%s;" % (i,i);
144 dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie");
145 dump_file = utils.open_file(dump_filename, 'w');
147 os.chmod(dump_filename, 0660);
149 if errno.errorcode[e.errno] == 'EPERM':
150 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]);
151 if perms & stat.S_IROTH:
152 utils.fubar("%s is world readable and chmod failed." % (dump_filename));
156 p = cPickle.Pickler(dump_file, 1);
157 for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]:
160 for file in files.keys():
162 for i in [ "package", "version", "architecture", "type", "size",
163 "md5sum", "component", "location id", "source package",
164 "source version", "maintainer", "dbtype", "files id",
165 "new", "section", "priority", "othercomponents",
167 if files[file].has_key(i):
168 d_files[file][i] = files[file][i];
170 # Mandatory changes fields
171 for i in [ "distribution", "source", "architecture", "version", "maintainer",
172 "urgency", "fingerprint", "changedby822", "changedbyname",
173 "maintainername", "maintaineremail", "closes" ]:
174 d_changes[i] = changes[i];
175 # Optional changes fields
176 # FIXME: changes should be mandatory
177 for i in [ "changed-by", "maintainer822", "filecontents", "format",
178 "changes", "lisa note" ]:
179 if changes.has_key(i):
180 d_changes[i] = changes[i];
182 for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
186 for file in dsc_files.keys():
187 d_dsc_files[file] = {};
188 # Mandatory dsc_files fields
189 for i in [ "size", "md5sum" ]:
190 d_dsc_files[file][i] = dsc_files[file][i];
191 # Optional dsc_files fields
192 for i in [ "files id" ]:
193 if dsc_files[file].has_key(i):
194 d_dsc_files[file][i] = dsc_files[file][i];
196 for i in [ d_changes, d_dsc, d_files, d_dsc_files,
197 legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
201 ###########################################################################
203 # Set up the per-package template substitution mappings
205 def update_subst (self, reject_message = ""):
207 changes = self.pkg.changes;
208 # If jennifer crashed out in the right place, architecture may still be a string.
209 if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
210 changes["architecture"] = { "Unknown" : "" };
211 # and maintainer822 may not exist.
212 if not changes.has_key("maintainer822"):
213 changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
215 Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
216 Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
217 Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
219 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
220 if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
221 Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
222 Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
223 Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
225 Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
226 Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
227 Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
228 if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
229 Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
231 # Apply any global override of the Maintainer field
232 if self.Cnf.get("Dinstall::OverrideMaintainer"):
233 Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"];
234 Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"];
236 Subst["__REJECT_MESSAGE__"] = reject_message;
237 Subst["__SOURCE__"] = changes.get("source", "Unknown");
238 Subst["__VERSION__"] = changes.get("version", "Unknown");
240 ###########################################################################
242 def build_summaries(self):
243 changes = self.pkg.changes;
244 files = self.pkg.files;
246 byhand = summary = new = "";
248 # changes["distribution"] may not exist in corner cases
249 # (e.g. unreadable changes files)
250 if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
251 changes["distribution"] = {};
253 file_keys = files.keys();
255 for file in file_keys:
256 if files[file].has_key("byhand"):
258 summary = summary + file + " byhand\n"
259 elif files[file].has_key("new"):
261 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
262 if files[file].has_key("othercomponents"):
263 summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
264 if files[file]["type"] == "deb":
265 summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
267 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
268 destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
269 summary = summary + file + "\n to " + destination + "\n"
271 short_summary = summary;
273 # This is for direport's benefit...
274 f = re_fdnic.sub("\n .\n", changes.get("changes",""));
277 summary = summary + "Changes: " + f;
279 summary = summary + self.announce(short_summary, 0)
281 return (summary, short_summary);
283 ###########################################################################
285 def close_bugs (self, summary, action):
286 changes = self.pkg.changes;
290 bugs = changes["closes"].keys();
296 if not self.nmu.is_an_nmu(self.pkg):
297 summary = summary + "Closing bugs: ";
299 summary = summary + "%s " % (bug);
301 Subst["__BUG_NUMBER__"] = bug;
302 if changes["distribution"].has_key("stable"):
303 Subst["__STABLE_WARNING__"] = """
304 Note that this package is not part of the released stable Debian
305 distribution. It may have dependencies on other unreleased software,
306 or other instabilities. Please take care if you wish to install it.
307 The update will eventually make its way into the next released Debian
310 Subst["__STABLE_WARNING__"] = "";
311 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
312 utils.send_mail (mail_message, "");
314 self.Logger.log(["closing bugs"]+bugs);
316 summary = summary + "Setting bugs to severity fixed: ";
317 control_message = "";
319 summary = summary + "%s " % (bug);
320 control_message = control_message + "tag %s + fixed\n" % (bug);
321 if action and control_message != "":
322 Subst["__CONTROL_MESSAGE__"] = control_message;
323 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed");
324 utils.send_mail (mail_message, "");
326 self.Logger.log(["setting bugs to fixed"]+bugs);
327 summary = summary + "\n";
330 ###########################################################################
332 def announce (self, short_summary, action):
335 changes = self.pkg.changes;
337 # Only do announcements for source uploads with a recent dpkg-dev installed
338 if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
343 Subst["__SHORT_SUMMARY__"] = short_summary;
345 for dist in changes["distribution"].keys():
346 list = Cnf.Find("Suite::%s::Announce" % (dist));
347 if list == "" or lists_done.has_key(list):
349 lists_done[list] = 1;
350 summary = summary + "Announcing to %s\n" % (list);
353 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
354 if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"):
355 Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]);
356 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce");
357 utils.send_mail (mail_message, "");
359 if Cnf.get("Dinstall::CloseBugs"):
360 summary = self.close_bugs(summary, action);
364 ###########################################################################
366 def accept (self, summary, short_summary):
369 files = self.pkg.files;
372 self.Logger.log(["Accepting changes",self.pkg.changes_file]);
374 self.dump_vars(Cnf["Dir::Queue::Accepted"]);
376 # Move all the files into the accepted directory
377 utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
378 file_keys = files.keys();
379 for file in file_keys:
380 utils.move(file, Cnf["Dir::Queue::Accepted"]);
381 self.accept_bytes = self.accept_bytes + float(files[file]["size"])
382 self.accept_count = self.accept_count + 1;
384 # Send accept mail, announce to lists, close bugs and check for
385 # override disparities
386 if not Cnf["Dinstall::Options::No-Mail"]:
387 Subst["__SUITE__"] = "";
388 Subst["__SUMMARY__"] = summary;
389 mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted");
390 utils.send_mail(mail_message, "")
391 self.announce(short_summary, 1)
393 # Special support to enable clean auto-building of accepted packages
394 if Cnf.FindB("Dinstall::SpecialAcceptedAutoBuild") and \
395 self.pkg.changes["distribution"].has_key("unstable"):
396 self.projectB.query("BEGIN WORK");
397 for file in file_keys:
398 src = os.path.join(Cnf["Dir::Queue::Accepted"], file);
399 dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"], file);
400 # Create a symlink to it
401 os.symlink(src, dest);
402 # Add it to the list of packages for later processing by apt-ftparchive
403 self.projectB.query("INSERT INTO unstable_accepted (filename, in_accepted) VALUES ('%s', 't')" % (dest));
404 # If the .orig.tar.gz is in the pool, create a symlink to
405 # it (if one doesn't already exist)
406 if self.pkg.orig_tar_id:
407 # Determine the .orig.tar.gz file name
408 for dsc_file in self.pkg.dsc_files.keys():
409 if dsc_file[-12:] == ".orig.tar.gz":
411 dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"],filename);
412 # If it doesn't exist, create a symlink
413 if not os.path.exists(dest):
414 # Find the .orig.tar.gz in the pool
415 q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id));
418 utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id));
419 src = os.path.join(ql[0][0], ql[0][1]);
420 os.symlink(src, dest);
421 # Add it to the list of packages for later processing by apt-ftparchive
422 self.projectB.query("INSERT INTO unstable_accepted (filename, in_accepted) VALUES ('%s', 't')" % (dest));
424 self.projectB.query("COMMIT WORK");
426 ###########################################################################
428 def check_override (self):
430 changes = self.pkg.changes;
431 files = self.pkg.files;
434 # Abandon the check if:
435 # a) it's a non-sourceful upload
436 # b) override disparity checks have been disabled
437 # c) we're not sending mail
438 if not changes["architecture"].has_key("source") or \
439 not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \
440 Cnf["Dinstall::Options::No-Mail"]:
444 for file in files.keys():
445 if not files[file].has_key("new") and files[file]["type"] == "deb":
446 section = files[file]["section"];
447 override_section = files[file]["override section"];
448 if lower(section) != lower(override_section) and section != "-":
449 # Ignore this; it's a common mistake and not worth whining about
450 if lower(section) == "non-us/main" and lower(override_section) == "non-us":
452 summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
453 priority = files[file]["priority"];
454 override_priority = files[file]["override priority"];
455 if priority != override_priority and priority != "-":
456 summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
461 Subst["__SUMMARY__"] = summary;
462 mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity");
463 utils.send_mail (mail_message, "");
465 ###########################################################################
467 def force_move (self, files):
468 """Forcefully move files from the current directory to the reject
469 directory. If any file already exists it will be moved to the
470 morgue to make way for the new file."""
475 # Skip any files which don't exist or which we don't have permission to copy.
476 if os.access(file,os.R_OK) == 0:
478 dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file);
480 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
482 # File exists? Let's try and move it to the morgue
483 if errno.errorcode[e.errno] == 'EEXIST':
484 morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file);
486 morgue_file = utils.find_next_free(morgue_file);
487 except utils.tried_too_hard_exc:
488 # Something's either gone badly Pete Tong, or
489 # someone is trying to exploit us.
490 utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file));
492 utils.move(dest_file, morgue_file, perms=0660);
494 os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
497 utils.warn("**WARNING** failed to claim %s in the reject directory." % (file));
501 # If we got here, we own the destination file, so we can
502 # safely overwrite it.
503 utils.move(file, dest_file, 1, perms=0660);
505 ###########################################################################
507 def do_reject (self, manual = 0, reject_message = ""):
508 # If we weren't given a manual rejection message, spawn an
509 # editor so the user can add one in...
510 if manual and not reject_message:
511 temp_filename = tempfile.mktemp();
512 fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
514 editor = os.environ.get("EDITOR","vi")
517 os.system("%s %s" % (editor, temp_filename))
518 file = utils.open_file(temp_filename);
519 reject_message = string.join(file.readlines());
521 print "Reject message:";
522 print utils.prefix_multi_line_string(reject_message," ");
523 prompt = "[R]eject, Edit, Abandon, Quit ?"
525 while string.find(prompt, answer) == -1:
526 answer = utils.our_raw_input(prompt);
527 m = re_default_answer.search(prompt);
530 answer = string.upper(answer[:1]);
531 os.unlink(temp_filename);
543 reason_filename = pkg.changes_file[:-8] + ".reason";
544 reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
546 # Move all the files into the reject directory
547 reject_files = pkg.files.keys() + [pkg.changes_file];
548 self.force_move(reject_files);
550 # If we fail here someone is probably trying to exploit the race
551 # so let's just raise an exception ...
552 if os.path.exists(reject_filename):
553 os.unlink(reject_filename);
554 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
557 Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
558 Subst["__MANUAL_REJECT_MESSAGE__"] = "";
559 Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
560 os.write(fd, reject_message);
562 reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
564 # Build up the rejection email
565 user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]);
567 Subst["__REJECTOR_ADDRESS__"] = user_email_address;
568 Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
569 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
570 reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
572 # Write the rejection email out as the <foo>.reason file
573 os.write(fd, reject_mail_message);
576 # Send the rejection mail if appropriate
577 if not Cnf["Dinstall::Options::No-Mail"]:
578 utils.send_mail (reject_mail_message, "");
580 self.Logger.log(["rejected", pkg.changes_file]);
583 ################################################################################
585 # Ensure that source exists somewhere in the archive for the binary
586 # upload being processed.
588 # (1) exact match => 1.0-3
589 # (2) Bin-only NMU of an MU => 1.0-3.0.1
590 # (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
592 def source_exists (self, package, source_version):
593 q = self.projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
595 # Reduce the query results to a list of version numbers
596 ql = map(lambda x: x[0], q.getresult());
599 if ql.count(source_version):
603 orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
604 if ql.count(orig_source_version):
608 orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
609 if ql.count(orig_source_version):
615 ################################################################################
617 def in_override_p (self, package, component, suite, binary_type, file):
618 files = self.pkg.files;
620 if binary_type == "": # must be source
625 # Override suite name; used for example with proposed-updates
626 if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
627 suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)];
629 # Avoid <undef> on unknown distributions
630 suite_id = db_access.get_suite_id(suite);
633 component_id = db_access.get_component_id(component);
634 type_id = db_access.get_override_type_id(type);
636 # FIXME: nasty non-US speficic hack
637 if lower(component[:7]) == "non-us/":
638 component = component[7:];
640 q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
641 % (package, suite_id, component_id, type_id));
642 result = q.getresult();
643 # If checking for a source package fall back on the binary override type
644 if type == "dsc" and not result:
645 type_id = db_access.get_override_type_id("deb");
646 q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
647 % (package, suite_id, component_id, type_id));
648 result = q.getresult();
650 # Remember the section and priority so we can check them later if appropriate
652 files[file]["override section"] = result[0][0];
653 files[file]["override priority"] = result[0][1];
657 ################################################################################
659 def reject (self, str, prefix="Rejected: "):
661 # Unlike other rejects we add new lines first to avoid trailing
662 # new lines when this message is passed back up to a caller.
663 if self.reject_message:
664 self.reject_message = self.reject_message + "\n";
665 self.reject_message = self.reject_message + prefix + str;
667 ################################################################################
669 def cross_suite_version_check(self, query_result, file, new_version):
670 """Ensure versions are newer than existing packages in target
671 suites and that cross-suite version checking rules as
672 set out in the conf file are satisfied."""
674 # Check versions for each target suite
675 for target_suite in self.pkg.changes["distribution"].keys():
676 must_be_newer_than = map(lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
677 must_be_older_than = map(lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
678 # Enforce "must be newer than target suite" even if conffile omits it
679 if target_suite not in must_be_newer_than:
680 must_be_newer_than.append(target_suite);
681 for entry in query_result:
682 existent_version = entry[0];
684 if suite in must_be_newer_than and \
685 apt_pkg.VersionCompare(new_version, existent_version) != 1:
686 self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
687 if suite in must_be_older_than and \
688 apt_pkg.VersionCompare(new_version, existent_version) != -1:
689 self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
691 ################################################################################
693 def check_binary_against_db(self, file):
694 self.reject_message = "";
695 files = self.pkg.files;
697 # Ensure version is sane
698 q = self.projectB.query("""
699 SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
701 WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
702 AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
703 % (files[file]["package"],
704 files[file]["architecture"]));
705 self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
707 # Check for any existing copies of the file
708 q = self.projectB.query("""
709 SELECT b.id FROM binaries b, architecture a
710 WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
711 AND a.id = b.architecture"""
712 % (files[file]["package"],
713 files[file]["version"],
714 files[file]["architecture"]))
716 self.reject("can not overwrite existing copy of '%s' already in the archive." % (file));
718 return self.reject_message;
720 ################################################################################
722 def check_source_against_db(self, file):
723 self.reject_message = "";
726 # Ensure version is sane
727 q = self.projectB.query("""
728 SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
729 WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
730 self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
732 return self.reject_message;
734 ################################################################################
736 def check_dsc_against_db(self, file):
737 self.reject_message = "";
738 files = self.pkg.files;
739 dsc_files = self.pkg.dsc_files;
740 legacy_source_untouchable = self.pkg.legacy_source_untouchable;
743 # Try and find all files mentioned in the .dsc. This has
744 # to work harder to cope with the multiple possible
745 # locations of an .orig.tar.gz.
746 for dsc_file in dsc_files.keys():
748 if files.has_key(dsc_file):
749 actual_md5 = files[dsc_file]["md5sum"];
750 actual_size = int(files[dsc_file]["size"]);
751 found = "%s in incoming" % (dsc_file)
752 # Check the file does not already exist in the archive
753 q = self.projectB.query("SELECT f.size, f.md5sum FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
755 # "It has not broken them. It has fixed a
756 # brokenness. Your crappy hack exploited a bug in
759 # "(Come on! I thought it was always obvious that
760 # one just doesn't release different files with
761 # the same name and version.)"
762 # -- ajk@ on d-devel@l.d.o
766 # Ignore exact matches for .orig.tar.gz
768 if dsc_file[-12:] == ".orig.tar.gz":
770 if files.has_key(dsc_file) and \
771 int(files[dsc_file]["size"]) == int(i[0]) and \
772 files[dsc_file]["md5sum"] == i[1]:
773 self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
778 self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file));
779 elif dsc_file[-12:] == ".orig.tar.gz":
781 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
785 # Unfortunately, we make get more than one
786 # match here if, for example, the package was
787 # in potato but had a -sa upload in woody. So
788 # we need to choose the right one.
790 x = ql[0]; # default to something sane in case we don't match any or have only one
794 old_file = i[0] + i[1];
795 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
796 actual_size = os.stat(old_file)[stat.ST_SIZE];
797 if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
800 legacy_source_untouchable[i[3]] = "";
802 old_file = x[0] + x[1];
803 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
804 actual_size = os.stat(old_file)[stat.ST_SIZE];
807 dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
808 # See install() in katie...
809 self.pkg.orig_tar_id = x[3];
810 if suite_type == "legacy" or suite_type == "legacy-mixed":
811 self.pkg.orig_tar_location = "legacy";
813 self.pkg.orig_tar_location = x[4];
815 # Not there? Check the queue directories...
817 in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file);
818 # See process_it() in jennifer for explanation of this
819 if os.path.exists(in_unchecked):
820 return (self.reject_message, in_unchecked);
822 for dir in [ "Accepted", "New", "Byhand" ]:
823 in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file);
824 if os.path.exists(in_otherdir):
825 actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
826 actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
830 self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
833 self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
835 if actual_md5 != dsc_files[dsc_file]["md5sum"]:
836 self.reject("md5sum for %s doesn't match %s." % (found, file));
837 if actual_size != int(dsc_files[dsc_file]["size"]):
838 self.reject("size for %s doesn't match %s." % (found, file));
840 return (self.reject_message, orig_tar_gz);
842 def do_query(self, q):
843 sys.stderr.write("query: \"%s\" ... " % (q));
844 before = time.time();
845 r = self.projectB.query(q);
846 time_diff = time.time()-before;
847 sys.stderr.write("took %.3f seconds.\n" % (time_diff));