3 # Installs Debian packages
4 # Copyright (C) 2000, 2001 James Troup <james@nocrew.org>
5 # $Id: katie,v 1.67 2001-11-25 01:21:34 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 # Originally based almost entirely on dinstall by Guy Maor <maor@debian.org>
23 ###############################################################################
25 # Cartman: "I'm trying to make the best of a bad situation, I don't
26 # need to hear crap from a bunch of hippy freaks living in
27 # denial. Screw you guys, I'm going home."
29 # Kyle: "But Cartman, we're trying to..."
31 # Cartman: "uhh.. screw you guys... home."
33 ###############################################################################
35 import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback
36 import apt_inst, apt_pkg
37 import utils, db_access, logging
41 ###############################################################################
43 re_isanum = re.compile (r"^\d+$");
44 re_changes = re.compile (r"changes$");
45 re_default_answer = re.compile(r"\[(.*)\]");
46 re_fdnic = re.compile("\n\n");
47 re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null");
48 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
49 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
51 #########################################################################################
69 orig_tar_location = "";
70 legacy_source_untouchable = {};
73 katie_version = "$Revision: 1.67 $";
75 ###############################################################################
80 Cnf = utils.get_conf()
82 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
83 ('h',"help","Dinstall::Options::Help"),
84 ('k',"ack-new","Dinstall::Options::Ack-New"),
85 ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
86 ('n',"no-action","Dinstall::Options::No-Action"),
87 ('p',"no-lock", "Dinstall::Options::No-Lock"),
88 ('s',"no-mail", "Dinstall::Options::No-Mail"),
89 ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
90 ('V',"version","Dinstall::Options::Version")];
92 for i in ["automatic", "help", "ack-new", "manual-reject", "no-action",
93 "no-lock", "no-mail", "override-distribution", "version"]:
94 if not Cnf.has_key("Dinstall::Options::%s" % (i)):
95 Cnf["Dinstall::Options::%s" % (i)] = "";
97 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
98 Options = Cnf.SubTree("Dinstall::Options")
100 return changes_files;
102 ###############################################################################
104 def usage (exit_code=0):
105 print """Usage: dinstall [OPTION]... [CHANGES]...
106 -a, --automatic automatic run
107 -h, --help show this help and exit.
108 -k, --ack-new acknowledge new packages !! for cron.daily only !!
109 -m, --manual-reject=MSG manual reject with `msg'
110 -n, --no-action don't do anything
111 -p, --no-lock don't check lockfile !! for cron.daily only !!
112 -s, --no-mail don't send any mail
113 -u, --distribution=DIST override distribution to `dist'
114 -V, --version display the version number and exit"""
117 #########################################################################################
119 def check_signature (filename):
120 global reject_message
122 (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename))
124 reject_message = reject_message + "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output)
128 ######################################################################################################
131 # Read in the group maintainer override file
133 self.group_maint = {};
134 if Cnf.get("Dinstall::GroupOverrideFilename"):
135 filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
136 file = utils.open_file(filename);
137 for line in file.readlines():
138 line = string.strip(utils.re_comments.sub('', line));
140 self.group_maint[line] = 1;
143 def is_an_nmu (self, changes, dsc):
144 (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
145 # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
146 if dsc_name == changes["maintainername"] and (changes["changedby822"] == "" or changes["changedbyname"] == dsc_name):
149 if dsc.has_key("uploaders"):
150 uploaders = string.split(dsc["uploaders"], ",");
153 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
154 uploadernames[name] = "";
155 if uploadernames.has_key(changes["changedbyname"]):
158 # Some group maintained packages (e.g. Debian QA) are never NMU's
159 if self.group_maint.has_key(changes["maintaineremail"]):
164 ######################################################################################################
166 # Ensure that source exists somewhere in the archive for the binary
167 # upload being processed.
169 # (1) exact match => 1.0-3
170 # (2) Bin-only NMU of an MU => 1.0-3.0.1
171 # (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
173 def source_exists (package, source_version):
174 q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
176 # Reduce the query results to a list of version numbers
177 ql = map(lambda x: x[0], q.getresult());
180 if ql.count(source_version):
184 orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
185 if ql.count(orig_source_version):
189 orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
190 if ql.count(orig_source_version):
196 ######################################################################################################
198 # See if a given package is in the override table
200 def in_override_p (package, component, suite, binary_type, file):
203 if binary_type == "": # must be source
208 # Override suite name; used for example with proposed-updates
209 if Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
210 suite = Cnf["Suite::%s::OverrideSuite" % (suite)];
212 # Avoid <undef> on unknown distributions
213 suite_id = db_access.get_suite_id(suite);
216 component_id = db_access.get_component_id(component);
217 type_id = db_access.get_override_type_id(type);
219 # FIXME: nasty non-US speficic hack
220 if string.lower(component[:7]) == "non-us/":
221 component = component[7:];
223 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
224 % (package, suite_id, component_id, type_id));
225 result = q.getresult();
226 # If checking for a source package fall back on the binary override type
227 if type == "dsc" and not result:
228 type_id = db_access.get_override_type_id("deb");
229 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
230 % (package, suite_id, component_id, type_id));
231 result = q.getresult();
233 # Remember the section and priority so we can check them later if appropriate
235 files[file]["override section"] = result[0][0];
236 files[file]["override priority"] = result[0][1];
240 #####################################################################################################################
242 def check_changes(filename):
243 global reject_message, changes, files
245 # Default in case we bail out
246 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
247 changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"];
248 changes["architecture"] = {};
250 # Parse the .changes field into a dictionary
252 changes = utils.parse_changes(filename, 0)
253 except utils.cant_open_exc:
254 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
256 except utils.changes_parse_error_exc, line:
257 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
260 # Parse the Files field from the .changes into another dictionary
262 files = utils.build_file_list(changes, "");
263 except utils.changes_parse_error_exc, line:
264 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line);
265 except utils.nk_format_exc, format:
266 reject_message = reject_message + "Rejected: unknown format '%s' of changes file '%s'.\n" % (format, filename);
269 # Check for mandatory fields
270 for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"):
271 if not changes.has_key(i):
272 reject_message = reject_message + "Rejected: Missing field `%s' in changes file.\n" % (i)
273 return 0 # Avoid <undef> errors during later tests
275 # Override the Distribution: field if appropriate
276 if Options["Override-Distribution"] != "":
277 reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Options["Override-Distribution"])
278 changes["distribution"] = Options["Override-Distribution"]
280 # Split multi-value fields into a lower-level dictionary
281 for i in ("architecture", "distribution", "binary", "closes"):
282 o = changes.get(i, "")
286 for j in string.split(o):
289 # Fix the Maintainer: field to be RFC822 compatible
290 (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
292 # Fix the Changed-By: field to be RFC822 compatible; if it exists.
293 (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
295 # Ensure all the values in Closes: are numbers
296 if changes.has_key("closes"):
297 for i in changes["closes"].keys():
298 if re_isanum.match (i) == None:
299 reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i)
301 # Ensure there _is_ a target distribution
302 if not changes["distribution"].keys():
303 reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n";
305 # Map frozen to unstable if frozen doesn't exist
306 if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"):
307 del changes["distribution"]["frozen"]
308 changes["distribution"]["unstable"] = 1;
309 reject_message = reject_message + "Mapping frozen to unstable.\n"
311 # Map testing to unstable
312 if changes["distribution"].has_key("testing"):
313 if len(changes["distribution"].keys()) > 1:
314 del changes["distribution"]["testing"];
315 reject_message = reject_message + "Warning: Ignoring testing as a target suite.\n";
317 reject_message = reject_message + "Rejected: invalid distribution 'testing'.\n";
319 # Ensure target distributions exist
320 for i in changes["distribution"].keys():
321 if not Cnf.has_key("Suite::%s" % (i)):
322 reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i)
324 # Map unreleased arches from stable to unstable
325 if changes["distribution"].has_key("stable"):
326 for i in changes["architecture"].keys():
327 if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)):
328 reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i)
329 del changes["distribution"]["stable"]
330 changes["distribution"]["unstable"] = 1;
332 # Map arches not being released from frozen to unstable
333 if changes["distribution"].has_key("frozen"):
334 for i in changes["architecture"].keys():
335 if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)):
336 reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i)
337 del changes["distribution"]["frozen"]
338 changes["distribution"]["unstable"] = 1;
340 # Handle uploads to stable
341 if changes["distribution"].has_key("stable"):
342 # If running from within proposed-updates; assume an install to stable
343 if string.find(os.getcwd(), 'proposed-updates') != -1:
344 # Remove non-stable target distributions
345 for dist in changes["distribution"].keys():
347 reject_message = reject_message + "Removing %s from distribution list.\n" % (dist);
348 del changes["distribution"][dist];
349 changes["stable install"] = 1;
350 # If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool
351 file = files.keys()[0];
352 if os.access(file, os.R_OK) == 0:
353 pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]);
354 changes["installing from the pool"] = 1;
356 # Otherwise (normal case) map stable to updates
358 reject_message = reject_message + "Mapping stable to updates.\n";
359 del changes["distribution"]["stable"];
360 changes["distribution"]["proposed-updates"] = 1;
362 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
363 changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"])
364 changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"])
366 if string.find(reject_message, "Rejected:") != -1:
372 global reject_message
374 archive = utils.where_am_i();
376 for file in files.keys():
377 # Check the file is readable
378 if os.access(file,os.R_OK) == 0:
379 if os.path.exists(file):
380 reject_message = reject_message + "Rejected: Can't read `%s'. [permission denied]\n" % (file)
382 reject_message = reject_message + "Rejected: Can't read `%s'. [file not found]\n" % (file)
384 files[file]["type"] = "unreadable";
386 # If it's byhand skip remaining checks
387 if files[file]["section"] == "byhand":
388 files[file]["byhand"] = 1;
389 files[file]["type"] = "byhand";
390 # Checks for a binary package...
391 elif utils.re_isadeb.match(file) != None:
392 files[file]["type"] = "deb";
394 # Extract package information using dpkg-deb
396 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
398 reject_message = reject_message + "Rejected: %s: debExtractControl() raised %s.\n" % (file, sys.exc_type);
399 # Can't continue, none of the checks on control would work.
402 # Check for mandatory fields
403 if control.Find("Package") == None:
404 reject_message = reject_message + "Rejected: %s: No package field in control.\n" % (file)
405 if control.Find("Architecture") == None:
406 reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file)
407 if control.Find("Version") == None:
408 reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file)
410 # Ensure the package name matches the one give in the .changes
411 if not changes["binary"].has_key(control.Find("Package", "")):
412 reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", ""))
414 # Validate the architecture
415 if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))):
416 reject_message = reject_message + "Rejected: Unknown architecture '%s'.\n" % (control.Find("Architecture", ""))
418 # Check the architecture matches the one given in the .changes
419 if not changes["architecture"].has_key(control.Find("Architecture", "")):
420 reject_message = reject_message + "Rejected: %s: control file lists arch as `%s', which isn't in changes file.\n" % (file, control.Find("Architecture", ""))
421 # Check the section & priority match those given in the .changes (non-fatal)
422 if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"):
423 reject_message = reject_message + "Warning: %s control file lists section as `%s', but changes file has `%s'.\n" % (file, control.Find("Section", ""), files[file]["section"])
424 if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"):
425 reject_message = reject_message + "Warning: %s control file lists priority as `%s', but changes file has `%s'.\n" % (file, control.Find("Priority", ""), files[file]["priority"])
427 epochless_version = utils.re_no_epoch.sub('', control.Find("Version", ""))
429 files[file]["package"] = control.Find("Package");
430 files[file]["architecture"] = control.Find("Architecture");
431 files[file]["version"] = control.Find("Version");
432 files[file]["maintainer"] = control.Find("Maintainer", "");
433 if file[-5:] == ".udeb":
434 files[file]["dbtype"] = "udeb";
435 elif file[-4:] == ".deb":
436 files[file]["dbtype"] = "deb";
438 reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file);
439 files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", ""))
440 files[file]["source"] = control.Find("Source", "");
441 if files[file]["source"] == "":
442 files[file]["source"] = files[file]["package"];
443 # Get the source version
444 source = files[file]["source"];
446 if string.find(source, "(") != -1:
447 m = utils.re_extract_src_version.match(source)
449 source_version = m.group(2)
450 if not source_version:
451 source_version = files[file]["version"];
452 files[file]["source package"] = source;
453 files[file]["source version"] = source_version;
455 # Checks for a source package...
457 m = utils.re_issource.match(file)
459 files[file]["package"] = m.group(1)
460 files[file]["version"] = m.group(2)
461 files[file]["type"] = m.group(3)
463 # Ensure the source package name matches the Source filed in the .changes
464 if changes["source"] != files[file]["package"]:
465 reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"])
467 # Ensure the source version matches the version in the .changes file
468 if files[file]["type"] == "orig.tar.gz":
469 changes_version = changes["chopversion2"]
471 changes_version = changes["chopversion"]
472 if changes_version != files[file]["version"]:
473 reject_message = reject_message + "Rejected: %s: should be %s according to changes file.\n" % (file, changes_version)
475 # Ensure the .changes lists source in the Architecture field
476 if not changes["architecture"].has_key("source"):
477 reject_message = reject_message + "Rejected: %s: changes file doesn't list `source' in Architecture field.\n" % (file)
479 # Check the signature of a .dsc file
480 if files[file]["type"] == "dsc":
481 check_signature(file)
483 files[file]["fullname"] = file
484 files[file]["architecture"] = "source";
486 # Not a binary or source package? Assume byhand...
488 files[file]["byhand"] = 1;
489 files[file]["type"] = "byhand";
491 files[file]["oldfiles"] = {}
492 for suite in changes["distribution"].keys():
494 if files[file].has_key("byhand"):
497 if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])):
498 reject_message = reject_message + "Rejected: unknown component `%s' for suite `%s'.\n" % (files[file]["component"], suite)
501 # See if the package is NEW
502 if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
503 files[file]["new"] = 1
505 if files[file]["type"] == "deb":
506 # Find any old binary packages
507 q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
508 % (files[file]["package"], suite, files[file]["architecture"]))
509 oldfiles = q.dictresult()
510 for oldfile in oldfiles:
511 files[file]["oldfiles"][suite] = oldfile
512 # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
513 if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
514 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
515 # Check for existing copies of the file
516 if not changes.has_key("stable install"):
517 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
518 if q.getresult() != []:
519 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file)
521 # Check for existent source
522 # FIXME: this is no longer per suite
523 if changes["architecture"].has_key("source"):
524 source_version = files[file]["source version"];
525 if source_version != changes["version"]:
526 reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]);
528 if not source_exists (files[file]["source package"], source_version):
529 reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file);
531 # Find any old .dsc files
532 elif files[file]["type"] == "dsc":
533 q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
534 % (files[file]["package"], suite))
535 oldfiles = q.dictresult()
536 if len(oldfiles) >= 1:
537 files[file]["oldfiles"][suite] = oldfiles[0]
539 # Validate the component
540 component = files[file]["component"];
541 component_id = db_access.get_component_id(component);
542 if component_id == -1:
543 reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component);
546 # Validate the priority
547 if string.find(files[file]["priority"],'/') != -1:
548 reject_message = reject_message + "Rejected: file '%s' has invalid priority '%s' [contains '/'].\n" % (file, files[file]["priority"]);
550 # Check the md5sum & size against existing files (if any)
551 location = Cnf["Dir::PoolDir"];
552 files[file]["location id"] = db_access.get_location_id (location, component, archive);
554 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]);
555 files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]);
557 reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file)
559 reject_message = reject_message + "Rejected: md5sum and/or size mismatch on existing copy of %s.\n" % (file)
560 files[file]["files id"] = files_id
562 # Check for packages that have moved from one component to another
563 if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
564 files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
567 if string.find(reject_message, "Rejected:") != -1:
572 ###############################################################################
575 global dsc, dsc_files, reject_message, reprocess, orig_tar_id, orig_tar_location, legacy_source_untouchable;
577 for file in files.keys():
578 if files[file]["type"] == "dsc":
580 dsc = utils.parse_changes(file, 1)
581 except utils.cant_open_exc:
582 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (file)
584 except utils.changes_parse_error_exc, line:
585 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (file, line)
587 except utils.invalid_dsc_format_exc, line:
588 reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (file, line)
591 dsc_files = utils.build_file_list(dsc, 1)
592 except utils.no_files_exc:
593 reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n";
595 except utils.changes_parse_error_exc, line:
596 reject_message = reject_message + "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line);
599 # Enforce mandatory fields
600 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
601 if not dsc.has_key(i):
602 reject_message = reject_message + "Rejected: Missing field `%s' in dsc file.\n" % (i)
604 # The dpkg maintainer from hell strikes again! Bumping the
605 # version number of the .dsc breaks extraction by stable's
607 if dsc["format"] != "1.0":
608 reject_message = reject_message + """Rejected: [dpkg-sucks] source package was produced by a broken version
609 of dpkg-dev 1.9.1{3,4}; please rebuild with >= 1.9.15 version
613 # Ensure the version number in the .dsc matches the version number in the .changes
614 epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version"));
615 changes_version = files[file]["version"];
616 if epochless_dsc_version != files[file]["version"]:
617 reject_message = reject_message + "Rejected: version ('%s') in .dsc does not match version ('%s') in .changes\n" % (epochless_dsc_version, changes_version);
619 # Ensure source is newer than existing source in target suites
620 package = dsc.get("source");
621 new_version = dsc.get("version");
622 for suite in changes["distribution"].keys():
623 q = projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
625 ql = map(lambda x: x[0], q.getresult());
626 for old_version in ql:
627 if apt_pkg.VersionCompare(new_version, old_version) != 1:
628 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, old_version, new_version)
630 # Try and find all files mentioned in the .dsc. This has
631 # to work harder to cope with the multiple possible
632 # locations of an .orig.tar.gz.
633 for dsc_file in dsc_files.keys():
634 if files.has_key(dsc_file):
635 actual_md5 = files[dsc_file]["md5sum"];
636 actual_size = int(files[dsc_file]["size"]);
637 found = "%s in incoming" % (dsc_file)
638 # Check the file does not already exist in the archive
639 if not changes.has_key("stable install"):
640 q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
642 # "It has not broken them. It has fixed a
643 # brokenness. Your crappy hack exploited a
644 # bug in the old dinstall.
646 # "(Come on! I thought it was always obvious
647 # that one just doesn't release different
648 # files with the same name and version.)"
649 # -- ajk@ on d-devel@l.d.o
651 if q.getresult() != []:
652 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file)
653 elif dsc_file[-12:] == ".orig.tar.gz":
655 q = projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
659 # Unfortunately, we make get more than one match
660 # here if, for example, the package was in potato
661 # but had a -sa upload in woody. So we need to a)
662 # choose the right one and b) mark all wrong ones
663 # as excluded from the source poolification (to
664 # avoid file overwrites).
666 x = ql[0]; # default to something sane in case we don't match any or have only one
670 old_file = i[0] + i[1];
671 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
672 actual_size = os.stat(old_file)[stat.ST_SIZE];
673 if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
676 legacy_source_untouchable[i[3]] = "";
678 old_file = x[0] + x[1];
679 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
680 actual_size = os.stat(old_file)[stat.ST_SIZE];
683 dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
686 if suite_type == "legacy" or suite_type == "legacy-mixed":
687 orig_tar_location = "legacy";
689 orig_tar_location = x[4];
691 # Not there? Check in Incoming...
692 # [See comment above process_it() for explanation
693 # of why this is necessary...]
694 if os.path.exists(dsc_file):
695 files[dsc_file] = {};
696 files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
697 files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
698 files[dsc_file]["section"] = files[file]["section"];
699 files[dsc_file]["priority"] = files[file]["priority"];
700 files[dsc_file]["component"] = files[file]["component"];
701 files[dsc_file]["type"] = "orig.tar.gz";
705 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file);
708 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming.\n" % (file, dsc_file);
710 if actual_md5 != dsc_files[dsc_file]["md5sum"]:
711 reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file);
712 if actual_size != int(dsc_files[dsc_file]["size"]):
713 reject_message = reject_message + "Rejected: size for %s doesn't match %s.\n" % (found, file);
715 if string.find(reject_message, "Rejected:") != -1:
720 ###############################################################################
722 # Some cunning stunt broke dpkg-source in dpkg 1.8{,.1}; detect the
723 # resulting bad source packages and reject them.
725 # Even more amusingly the fix in 1.8.1.1 didn't actually fix the
726 # problem just changed the symptoms.
729 global dsc, dsc_files, reject_message, reprocess;
731 for filename in files.keys():
732 if files[filename]["type"] == "diff.gz":
733 file = gzip.GzipFile(filename, 'r');
734 for line in file.readlines():
735 if re_bad_diff.search(line):
736 reject_message = reject_message + "Rejected: [dpkg-sucks] source package was produced by a broken version of dpkg-dev 1.8.x; please rebuild with >= 1.8.3 version installed.\n";
739 if string.find(reject_message, "Rejected:") != -1:
744 ###############################################################################
746 def check_md5sums ():
747 global reject_message;
749 for file in files.keys():
751 file_handle = utils.open_file(file);
752 except utils.cant_open_exc:
755 if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
756 reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file);
758 def check_override ():
761 # Only check section & priority on sourceful non-stable installs
762 if not changes["architecture"].has_key("source") or changes.has_key("stable install"):
766 for file in files.keys():
767 if not files[file].has_key("new") and files[file]["type"] == "deb":
768 section = files[file]["section"];
769 override_section = files[file]["override section"];
770 if section != override_section and section != "-":
771 # Ignore this; it's a common mistake and not worth whining about
772 if string.lower(section) == "non-us/main" and string.lower(override_section) == "non-us":
774 summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
775 priority = files[file]["priority"];
776 override_priority = files[file]["override priority"];
777 if priority != override_priority and priority != "-":
778 summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
783 Subst["__SUMMARY__"] = summary;
784 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.override-disparity","r").read());
785 utils.send_mail (mail_message, "")
787 #####################################################################################################################
789 # Set up the per-package template substitution mappings
791 def update_subst (changes_filename):
794 # If katie crashed out in the right place, architecture may still be a string.
795 if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
796 changes["architecture"] = { "Unknown" : "" };
797 # and maintainer822 may not exist.
798 if not changes.has_key("maintainer822"):
799 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
801 Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
802 Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename);
803 Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
805 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
806 if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
807 Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
808 Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
809 Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
811 Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
812 Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
813 Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
815 Subst["__REJECT_MESSAGE__"] = reject_message;
816 Subst["__SOURCE__"] = changes.get("source", "Unknown");
817 Subst["__VERSION__"] = changes.get("version", "Unknown");
819 #####################################################################################################################
821 def action (changes_filename):
822 byhand = summary = new = "";
824 # changes["distribution"] may not exist in corner cases
825 # (e.g. unreadable changes files)
826 if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
827 changes["distribution"] = {};
831 for suite in changes["distribution"].keys():
832 if Cnf.has_key("Suite::%s::Confirm"):
833 confirm.append(suite)
836 file_keys = files.keys();
838 for file in file_keys:
839 if files[file].has_key("byhand"):
841 summary = summary + file + " byhand\n"
842 elif files[file].has_key("new"):
844 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
845 if files[file].has_key("othercomponents"):
846 summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
847 if files[file]["type"] == "deb":
848 summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
850 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
851 destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
852 summary = summary + file + "\n to " + destination + "\n"
854 short_summary = summary;
856 # This is for direport's benefit...
857 f = re_fdnic.sub("\n .\n", changes.get("changes",""));
859 if confirm or byhand or new:
860 summary = summary + "Changes: " + f;
862 summary = summary + announce (short_summary, 0)
864 (prompt, answer) = ("", "XXX")
865 if Options["No-Action"] or Options["Automatic"]:
868 if string.find(reject_message, "Rejected") != -1:
870 modified_time = time.time()-os.path.getmtime(changes_filename);
871 except: # i.e. ignore errors like 'file does not exist';
873 if modified_time < 86400:
874 print "SKIP (too new)\n" + reject_message,;
875 prompt = "[S]kip, Manual reject, Quit ?";
877 print "REJECT\n" + reject_message,;
878 prompt = "[R]eject, Manual reject, Skip, Quit ?";
879 if Options["Automatic"]:
882 print "NEW to %s\n%s%s" % (string.join(suites, ", "), reject_message, summary),;
883 prompt = "[S]kip, New ack, Manual reject, Quit ?";
884 if Options["Automatic"] and Options["Ack-New"]:
887 print "BYHAND\n" + reject_message + summary,;
888 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
890 print "CONFIRM to %s\n%s%s" % (string.join(confirm, ", "), reject_message, summary),
891 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
893 print "INSTALL\n" + reject_message + summary,;
894 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
895 if Options["Automatic"]:
898 while string.find(prompt, answer) == -1:
900 answer = utils.our_raw_input()
901 m = re_default_answer.match(prompt)
904 answer = string.upper(answer[:1])
907 reject (changes_filename, "");
909 manual_reject (changes_filename);
911 install (changes_filename, summary, short_summary);
913 acknowledge_new (changes_filename, summary);
917 ###############################################################################
919 def install (changes_filename, summary, short_summary):
920 global install_count, install_bytes, Subst;
922 # stable installs are a special case
923 if changes.has_key("stable install"):
924 stable_install (changes_filename, summary, short_summary);
929 Logger.log(["installing changes",changes_filename]);
931 archive = utils.where_am_i();
933 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
934 projectB.query("BEGIN WORK");
936 # Add the .dsc file to the DB
937 for file in files.keys():
938 if files[file]["type"] == "dsc":
939 package = dsc["source"]
940 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
941 maintainer = dsc["maintainer"]
942 maintainer = string.replace(maintainer, "'", "\\'")
943 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
944 filename = files[file]["pool name"] + file;
945 dsc_location_id = files[file]["location id"];
946 if not files[file]["files id"]:
947 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id)
948 projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)"
949 % (package, version, maintainer_id, files[file]["files id"]))
951 for suite in changes["distribution"].keys():
952 suite_id = db_access.get_suite_id(suite);
953 projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
955 # Add the source files to the DB (files and dsc_files)
956 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"]));
957 for dsc_file in dsc_files.keys():
958 filename = files[file]["pool name"] + dsc_file;
959 # If the .orig.tar.gz is already in the pool, it's
960 # files id is stored in dsc_files by check_dsc().
961 files_id = dsc_files[dsc_file].get("files id", None);
963 files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
964 # FIXME: needs to check for -1/-2 and or handle exception
966 files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
967 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id));
969 # Add the .deb files to the DB
970 for file in files.keys():
971 if files[file]["type"] == "deb":
972 package = files[file]["package"]
973 version = files[file]["version"]
974 maintainer = files[file]["maintainer"]
975 maintainer = string.replace(maintainer, "'", "\\'")
976 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
977 architecture = files[file]["architecture"]
978 architecture_id = db_access.get_architecture_id (architecture);
979 type = files[file]["dbtype"];
980 dsc_component = files[file]["component"]
981 source = files[file]["source package"]
982 source_version = files[file]["source version"];
983 filename = files[file]["pool name"] + file;
984 if not files[file]["files id"]:
985 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
986 source_id = db_access.get_source_id (source, source_version);
988 projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, %d, '%s')"
989 % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type));
991 projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, '%s')"
992 % (package, version, maintainer_id, architecture_id, files[file]["files id"], type));
993 for suite in changes["distribution"].keys():
994 suite_id = db_access.get_suite_id(suite);
995 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id));
997 # If the .orig.tar.gz is in a legacy directory we need to poolify
998 # it, so that apt-get source (and anything else that goes by the
999 # "Directory:" field in the Sources.gz file) works.
1000 if orig_tar_id != None and orig_tar_location == "legacy":
1001 q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id));
1002 qd = q.dictresult();
1004 # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details)
1005 if legacy_source_untouchable.has_key(qid["files_id"]):
1007 # First move the files to the new location
1008 legacy_filename = qid["path"]+qid["filename"];
1009 pool_location = utils.poolify (changes["source"], files[file]["component"]);
1010 pool_filename = pool_location + os.path.basename(qid["filename"]);
1011 destination = Cnf["Dir::PoolDir"] + pool_location
1012 utils.move(legacy_filename, destination);
1013 # Then Update the DB's files table
1014 q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"]));
1016 # If this is a sourceful diff only upload that is moving non-legacy
1017 # cross-component we need to copy the .orig.tar.gz into the new
1018 # component too for the same reasons as above.
1020 if changes["architecture"].has_key("source") and orig_tar_id != None and \
1021 orig_tar_location != "legacy" and orig_tar_location != dsc_location_id:
1022 q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id));
1023 ql = q.getresult()[0];
1024 old_filename = ql[0] + ql[1];
1026 file_md5sum = ql[3];
1027 new_filename = utils.poolify (changes["source"], dsc_component) + os.path.basename(old_filename);
1028 new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1029 if new_files_id == None:
1030 utils.copy(old_filename, Cnf["Dir::PoolDir"] + new_filename);
1031 new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1032 projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id));
1034 # Install the files into the pool
1035 for file in files.keys():
1036 if files[file].has_key("byhand"):
1038 destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file
1039 destdir = os.path.dirname(destination)
1040 utils.move (file, destination)
1041 Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]);
1042 install_bytes = install_bytes + float(files[file]["size"])
1044 # Copy the .changes file across for suite which need it.
1045 for suite in changes["distribution"].keys():
1046 if Cnf.has_key("Suite::%s::CopyChanges" % (suite)):
1047 utils.copy (changes_filename, Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)]);
1049 projectB.query("COMMIT WORK");
1052 utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
1054 utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(changes_filename), sys.exc_type));
1056 install_count = install_count + 1;
1058 if not Options["No-Mail"]:
1059 Subst["__SUITE__"] = "";
1060 Subst["__SUMMARY__"] = summary;
1061 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1062 utils.send_mail (mail_message, "")
1063 announce (short_summary, 1)
1066 ################################################################################
1068 def stable_install (changes_filename, summary, short_summary):
1069 global install_count, install_bytes, Subst;
1071 print "Installing to stable."
1073 archive = utils.where_am_i();
1075 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
1076 projectB.query("BEGIN WORK");
1078 # Add the .dsc file to the DB
1079 for file in files.keys():
1080 if files[file]["type"] == "dsc":
1081 package = dsc["source"]
1082 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
1083 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
1086 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1087 source_id = ql[0][0];
1088 suite_id = db_access.get_suite_id('proposed-updates');
1089 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1090 suite_id = db_access.get_suite_id('stable');
1091 projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id));
1092 install_bytes = install_bytes + float(files[file]["size"])
1094 # Add the .deb files to the DB
1095 for file in files.keys():
1096 if files[file]["type"] == "deb":
1097 package = files[file]["package"]
1098 version = files[file]["version"]
1099 architecture = files[file]["architecture"]
1100 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
1103 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1104 binary_id = ql[0][0];
1105 suite_id = db_access.get_suite_id('proposed-updates');
1106 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1107 suite_id = db_access.get_suite_id('stable');
1108 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id));
1109 install_bytes = install_bytes + float(files[file]["size"])
1111 projectB.query("COMMIT WORK");
1114 utils.move (changes_filename, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(changes_filename));
1116 # Update the Stable ChangeLog file
1118 new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog";
1119 changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog";
1120 if os.path.exists(new_changelog_filename):
1121 os.unlink (new_changelog_filename);
1123 new_changelog = utils.open_file(new_changelog_filename, 'w');
1124 for file in files.keys():
1125 if files[file]["type"] == "deb":
1126 new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file));
1127 elif utils.re_issource.match(file) != None:
1128 new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file));
1130 new_changelog.write("%s\n" % (file));
1131 chop_changes = re_fdnic.sub("\n", changes["changes"]);
1132 new_changelog.write(chop_changes + '\n\n');
1133 if os.access(changelog_filename, os.R_OK) != 0:
1134 changelog = utils.open_file(changelog_filename);
1135 new_changelog.write(changelog.read());
1136 new_changelog.close();
1137 if os.access(changelog_filename, os.R_OK) != 0:
1138 os.unlink(changelog_filename);
1139 utils.move(new_changelog_filename, changelog_filename);
1141 install_count = install_count + 1;
1143 if not Options["No-Mail"]:
1144 Subst["__SUITE__"] = " into stable";
1145 Subst["__SUMMARY__"] = summary;
1146 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1147 utils.send_mail (mail_message, "")
1148 announce (short_summary, 1)
1150 ################################################################################
1152 def reject (changes_filename, manual_reject_mail_filename):
1155 print "Rejecting.\n"
1157 base_changes_filename = os.path.basename(changes_filename);
1158 reason_filename = re_changes.sub("reason", base_changes_filename);
1159 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename);
1161 # Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored)
1163 utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
1165 utils.warn("couldn't reject changes file '%s'. [Got %s]" % (base_changes_filename, sys.exc_type));
1167 if not changes.has_key("stable install"):
1168 for file in files.keys():
1169 if os.path.exists(file):
1171 utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
1173 utils.warn("couldn't reject file '%s'. [Got %s]" % (file, sys.exc_type));
1176 suite_id = db_access.get_suite_id('proposed-updates');
1177 # Remove files from proposed-updates suite
1178 for file in files.keys():
1179 if files[file]["type"] == "dsc":
1180 package = dsc["source"];
1181 version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
1182 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version));
1185 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1186 source_id = ql[0][0];
1187 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1188 elif files[file]["type"] == "deb":
1189 package = files[file]["package"];
1190 version = files[file]["version"];
1191 architecture = files[file]["architecture"];
1192 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture));
1195 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1196 binary_id = ql[0][0];
1197 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1199 # If this is not a manual rejection generate the .reason file and rejection mail message
1200 if manual_reject_mail_filename == "":
1201 if os.path.exists(reject_filename):
1202 os.unlink(reject_filename);
1203 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1204 os.write(fd, reject_message);
1206 Subst["__MANUAL_REJECT_MESSAGE__"] = "";
1207 Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
1208 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
1209 else: # Have a manual rejection file to use
1210 reject_mail_message = ""; # avoid <undef>'s
1212 # Send the rejection mail if appropriate
1213 if not Options["No-Mail"]:
1214 utils.send_mail (reject_mail_message, manual_reject_mail_filename);
1216 Logger.log(["rejected", changes_filename]);
1218 ##################################################################
1220 def manual_reject (changes_filename):
1223 # Build up the rejection email
1224 user_email_address = utils.whoami() + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
1225 manual_reject_message = Options.get("Manual-Reject", "")
1227 Subst["__MANUAL_REJECT_MESSAGE__"] = manual_reject_message;
1228 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
1229 if changes.has_key("stable install"):
1230 template = "katie.stable-rejected";
1232 template = "katie.rejected";
1233 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/"+template,"r").read());
1235 # Write the rejection email out as the <foo>.reason file
1236 reason_filename = re_changes.sub("reason", os.path.basename(changes_filename));
1237 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename)
1238 if os.path.exists(reject_filename):
1239 os.unlink(reject_filename);
1240 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1241 os.write(fd, reject_mail_message);
1244 # If we weren't given one, spawn an editor so the user can add one in
1245 if manual_reject_message == "":
1246 result = os.system("vi +6 %s" % (reject_filename))
1248 utils.fubar("vi invocation failed for `%s'!" % (reject_filename), result);
1250 # Then process it as if it were an automatic rejection
1251 reject (changes_filename, reject_filename)
1253 #####################################################################################################################
1255 def acknowledge_new (changes_filename, summary):
1256 global new_ack_new, Subst;
1258 changes_filename = os.path.basename(changes_filename);
1260 Logger.log(["new",changes_filename]);
1262 new_ack_new[changes_filename] = 1;
1264 if new_ack_old.has_key(changes_filename):
1265 print "Ack already sent.";
1268 print "Sending new ack.";
1269 if not Options["No-Mail"]:
1270 Subst["__SUMMARY__"] = summary;
1271 new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.new","r").read());
1272 utils.send_mail(new_ack_message,"");
1274 #####################################################################################################################
1276 def announce (short_summary, action):
1279 # Only do announcements for source uploads with a recent dpkg-dev installed
1280 if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
1285 Subst["__SHORT_SUMMARY__"] = short_summary;
1287 for dist in changes["distribution"].keys():
1288 list = Cnf.Find("Suite::%s::Announce" % (dist))
1289 if list == "" or lists_done.has_key(list):
1291 lists_done[list] = 1
1292 summary = summary + "Announcing to %s\n" % (list)
1295 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
1296 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.announce","r").read());
1297 utils.send_mail (mail_message, "")
1299 bugs = changes["closes"].keys()
1301 if not nmu.is_an_nmu(changes, dsc):
1302 summary = summary + "Closing bugs: "
1304 summary = summary + "%s " % (bug)
1306 Subst["__BUG_NUMBER__"] = bug;
1307 if changes["distribution"].has_key("stable"):
1308 Subst["__STABLE_WARNING__"] = """
1309 Note that this package is not part of the released stable Debian
1310 distribution. It may have dependencies on other unreleased software,
1311 or other instabilities. Please take care if you wish to install it.
1312 The update will eventually make its way into the next released Debian
1315 Subst["__STABLE_WARNING__"] = "";
1316 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-close","r").read());
1317 utils.send_mail (mail_message, "")
1319 Logger.log(["closing bugs"]+bugs);
1321 summary = summary + "Setting bugs to severity fixed: "
1322 control_message = ""
1324 summary = summary + "%s " % (bug)
1325 control_message = control_message + "tag %s + fixed\n" % (bug)
1326 if action and control_message != "":
1327 Subst["__CONTROL_MESSAGE__"] = control_message;
1328 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-nmu-fixed","r").read());
1329 utils.send_mail (mail_message, "")
1331 Logger.log(["setting bugs to fixed"]+bugs);
1332 summary = summary + "\n"
1336 ###############################################################################
1338 # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in
1339 # Incoming. -1 will reference the .orig.tar.gz, but -2 will not.
1340 # dsccheckdistrib() can find the .orig.tar.gz but it will not have
1341 # processed it during it's checks of -2. If -1 has been deleted or
1342 # otherwise not checked by da-install, the .orig.tar.gz will not have
1343 # been checked at all. To get round this, we force the .orig.tar.gz
1344 # into the .changes structure and reprocess the .changes file.
1346 def process_it (changes_file):
1347 global reprocess, orig_tar_id, orig_tar_location, changes, dsc, dsc_files, files, reject_message;
1349 # Reset some globals
1356 orig_tar_location = "";
1357 legacy_source_untouchable = {};
1358 reject_message = "";
1360 # Absolutize the filename to avoid the requirement of being in the
1361 # same directory as the .changes file.
1362 changes_file = os.path.abspath(changes_file);
1364 # And since handling of installs to stable munges with the CWD;
1365 # save and restore it.
1369 check_signature (changes_file);
1370 check_changes (changes_file);
1379 traceback.print_exc(file=sys.stdout);
1382 update_subst(changes_file);
1383 action(changes_file);
1388 ###############################################################################
1391 global Cnf, Options, projectB, install_bytes, new_ack_old, Subst, nmu, Logger
1393 changes_files = init();
1398 if Options["Version"]:
1399 print "katie %s" % (katie_version);
1402 # -n/--dry-run invalidates some other options which would involve things happening
1403 if Options["No-Action"]:
1404 Options["Automatic"] = "";
1405 Options["Ack-New"] = "";
1407 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
1409 db_access.init(Cnf, projectB);
1411 # Check that we aren't going to clash with the daily cron job
1413 if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]:
1414 utils.fubar("Archive maintenance in progress. Try again later.");
1416 # Obtain lock if not in no-action mode and initialize the log
1418 if not Options["No-Action"]:
1419 lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT);
1420 fcntl.lockf(lock_fd, FCNTL.F_TLOCK);
1421 Logger = logging.Logger(Cnf, "katie");
1423 if Options["Ack-New"]:
1424 # Read in the list of already-acknowledged NEW packages
1425 if not os.path.exists(Cnf["Dinstall::NewAckList"]):
1426 utils.touch_file(Cnf["Dinstall::NewAckList"]);
1427 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"]);
1429 for line in new_ack_list.readlines():
1430 new_ack_old[line[:-1]] = 1;
1431 new_ack_list.close();
1433 # Initialize the substitution template mapping global
1435 Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
1436 Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
1437 bcc = "X-Katie: %s" % (katie_version);
1438 if Cnf.has_key("Dinstall::Bcc"):
1439 Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
1441 Subst["__BCC__"] = bcc;
1442 Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
1443 Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
1444 Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"];
1446 # Read in the group-maint override file
1449 # Sort the .changes files so that we process sourceful ones first
1450 changes_files.sort(utils.changes_compare);
1452 # Process the changes files
1453 for changes_file in changes_files:
1454 print "\n" + changes_file;
1455 process_it (changes_file);
1459 if install_count > 1:
1461 sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))));
1462 Logger.log(["total",install_count,install_bytes]);
1464 # Write out the list of already-acknowledged NEW packages
1465 if Options["Ack-New"]:
1466 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w')
1467 for i in new_ack_new.keys():
1468 new_ack_list.write(i+'\n')
1469 new_ack_list.close()
1471 if not Options["No-Action"]:
1474 if __name__ == '__main__':