3 # Installs Debian packages
4 # Copyright (C) 2000, 2001 James Troup <james@nocrew.org>
5 # $Id: katie,v 1.62 2001-11-04 22:41:31 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 # Originally based almost entirely on dinstall by Guy Maor <maor@debian.org>
23 #########################################################################################
25 # Cartman: "I'm trying to make the best of a bad situation, I don't
26 # need to hear crap from a bunch of hippy freaks living in
27 # denial. Screw you guys, I'm going home."
29 # Kyle: "But Cartman, we're trying to..."
31 # Cartman: "uhh.. screw you guys... home."
33 #########################################################################################
35 import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback
36 import apt_inst, apt_pkg
37 import utils, db_access, logging
41 ###############################################################################
43 re_isanum = re.compile (r"^\d+$");
44 re_changes = re.compile (r"changes$");
45 re_default_answer = re.compile(r"\[(.*)\]");
46 re_fdnic = re.compile("\n\n");
47 re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null");
48 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
49 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
51 #########################################################################################
69 orig_tar_location = "";
70 legacy_source_untouchable = {};
73 katie_version = "$Revision: 1.62 $";
75 ###############################################################################
82 Cnf = apt_pkg.newConfiguration();
83 apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file());
85 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
86 ('h',"help","Dinstall::Options::Help"),
87 ('k',"ack-new","Dinstall::Options::Ack-New"),
88 ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
89 ('n',"no-action","Dinstall::Options::No-Action"),
90 ('p',"no-lock", "Dinstall::Options::No-Lock"),
91 ('s',"no-mail", "Dinstall::Options::No-Mail"),
92 ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
93 ('V',"version","Dinstall::Options::Version")];
95 for i in ["automatic", "help", "ack-new", "manual-reject", "no-action",
96 "no-lock", "no-mail", "override-distribution", "version"]:
97 Cnf["Dinstall::Options::%s" % (i)] = "";
99 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
100 Options = Cnf.SubTree("Dinstall::Options")
102 return changes_files;
104 #########################################################################################
106 def usage (exit_code=0):
107 print """Usage: dinstall [OPTION]... [CHANGES]...
108 -a, --automatic automatic run
109 -h, --help show this help and exit.
110 -k, --ack-new acknowledge new packages !! for cron.daily only !!
111 -m, --manual-reject=MSG manual reject with `msg'
112 -n, --no-action don't do anything
113 -p, --no-lock don't check lockfile !! for cron.daily only !!
114 -s, --no-mail don't send any mail
115 -u, --distribution=DIST override distribution to `dist'
116 -V, --version display the version number and exit"""
119 #########################################################################################
121 def check_signature (filename):
122 global reject_message
124 (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename))
126 reject_message = reject_message + "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output)
130 ######################################################################################################
133 # Read in the group maintainer override file
135 self.group_maint = {};
136 if Cnf.get("Dinstall::GroupOverrideFilename"):
137 filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
138 file = utils.open_file(filename);
139 for line in file.readlines():
140 line = string.strip(utils.re_comments.sub('', line));
142 self.group_maint[line] = 1;
145 def is_an_nmu (self, changes, dsc):
146 (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
147 # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
148 if dsc_name == changes["maintainername"] and (changes["changedby822"] == "" or changes["changedbyname"] == dsc_name):
151 if dsc.has_key("uploaders"):
152 uploaders = string.split(dsc["uploaders"], ",");
155 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
156 uploadernames[name] = "";
157 if uploadernames.has_key(changes["changedbyname"]):
160 # Some group maintained packages (e.g. Debian QA) are never NMU's
161 if self.group_maint.has_key(changes["maintaineremail"]):
166 ######################################################################################################
168 # Ensure that source exists somewhere in the archive for the binary
169 # upload being processed.
171 # (1) exact match => 1.0-3
172 # (2) Bin-only NMU of an MU => 1.0-3.0.1
173 # (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
175 def source_exists (package, source_version):
176 q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
178 # Reduce the query results to a list of version numbers
179 ql = map(lambda x: x[0], q.getresult());
182 if ql.count(source_version):
186 orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
187 if ql.count(orig_source_version):
191 orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
192 if ql.count(orig_source_version):
198 ######################################################################################################
200 # See if a given package is in the override table
202 def in_override_p (package, component, suite, binary_type, file):
205 if binary_type == "": # must be source
210 # Override suite name; used for example with proposed-updates
211 if Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
212 suite = Cnf["Suite::%s::OverrideSuite" % (suite)];
214 # Avoid <undef> on unknown distributions
215 suite_id = db_access.get_suite_id(suite);
218 component_id = db_access.get_component_id(component);
219 type_id = db_access.get_override_type_id(type);
221 # FIXME: nasty non-US speficic hack
222 if string.lower(component[:7]) == "non-us/":
223 component = component[7:];
225 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
226 % (package, suite_id, component_id, type_id));
227 result = q.getresult();
228 # If checking for a source package fall back on the binary override type
229 if type == "dsc" and not result:
230 type_id = db_access.get_override_type_id("deb");
231 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
232 % (package, suite_id, component_id, type_id));
233 result = q.getresult();
235 # Remember the section and priority so we can check them later if appropriate
237 files[file]["override section"] = result[0][0];
238 files[file]["override priority"] = result[0][1];
242 #####################################################################################################################
244 def check_changes(filename):
245 global reject_message, changes, files
247 # Default in case we bail out
248 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
249 changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"];
250 changes["architecture"] = {};
252 # Parse the .changes field into a dictionary
254 changes = utils.parse_changes(filename, 0)
255 except utils.cant_open_exc:
256 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
258 except utils.changes_parse_error_exc, line:
259 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
262 # Parse the Files field from the .changes into another dictionary
264 files = utils.build_file_list(changes, "");
265 except utils.changes_parse_error_exc, line:
266 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line);
267 except utils.nk_format_exc, format:
268 reject_message = reject_message + "Rejected: unknown format '%s' of changes file '%s'.\n" % (format, filename);
271 # Check for mandatory fields
272 for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"):
273 if not changes.has_key(i):
274 reject_message = reject_message + "Rejected: Missing field `%s' in changes file.\n" % (i)
275 return 0 # Avoid <undef> errors during later tests
277 # Override the Distribution: field if appropriate
278 if Options["Override-Distribution"] != "":
279 reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Options["Override-Distribution"])
280 changes["distribution"] = Options["Override-Distribution"]
282 # Split multi-value fields into a lower-level dictionary
283 for i in ("architecture", "distribution", "binary", "closes"):
284 o = changes.get(i, "")
288 for j in string.split(o):
291 # Fix the Maintainer: field to be RFC822 compatible
292 (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
294 # Fix the Changed-By: field to be RFC822 compatible; if it exists.
295 (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
297 # Ensure all the values in Closes: are numbers
298 if changes.has_key("closes"):
299 for i in changes["closes"].keys():
300 if re_isanum.match (i) == None:
301 reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i)
303 # Ensure there _is_ a target distribution
304 if not changes["distribution"].keys():
305 reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n";
307 # Map frozen to unstable if frozen doesn't exist
308 if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"):
309 del changes["distribution"]["frozen"]
310 changes["distribution"]["unstable"] = 1;
311 reject_message = reject_message + "Mapping frozen to unstable.\n"
313 # Map testing to unstable
314 if changes["distribution"].has_key("testing"):
315 if len(changes["distribution"].keys()) > 1:
316 del changes["distribution"]["testing"];
317 reject_message = reject_message + "Warning: Ignoring testing as a target suite.\n";
319 reject_message = reject_message + "Rejected: invalid distribution 'testing'.\n";
321 # Ensure target distributions exist
322 for i in changes["distribution"].keys():
323 if not Cnf.has_key("Suite::%s" % (i)):
324 reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i)
326 # Map unreleased arches from stable to unstable
327 if changes["distribution"].has_key("stable"):
328 for i in changes["architecture"].keys():
329 if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)):
330 reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i)
331 del changes["distribution"]["stable"]
332 changes["distribution"]["unstable"] = 1;
334 # Map arches not being released from frozen to unstable
335 if changes["distribution"].has_key("frozen"):
336 for i in changes["architecture"].keys():
337 if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)):
338 reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i)
339 del changes["distribution"]["frozen"]
340 changes["distribution"]["unstable"] = 1;
342 # Handle uploads to stable
343 if changes["distribution"].has_key("stable"):
344 # If running from within proposed-updates; assume an install to stable
345 if string.find(os.getcwd(), 'proposed-updates') != -1:
346 # Remove non-stable target distributions
347 for dist in changes["distribution"].keys():
349 reject_message = reject_message + "Removing %s from distribution list.\n" % (dist);
350 del changes["distribution"][dist];
351 changes["stable install"] = 1;
352 # If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool
353 file = files.keys()[0];
354 if os.access(file, os.R_OK) == 0:
355 pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]);
356 changes["installing from the pool"] = 1;
358 # Otherwise (normal case) map stable to updates
360 reject_message = reject_message + "Mapping stable to updates.\n";
361 del changes["distribution"]["stable"];
362 changes["distribution"]["proposed-updates"] = 1;
364 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
365 changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"])
366 changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"])
368 if string.find(reject_message, "Rejected:") != -1:
374 global reject_message
376 archive = utils.where_am_i();
378 for file in files.keys():
379 # Check the file is readable
380 if os.access(file,os.R_OK) == 0:
381 if os.path.exists(file):
382 reject_message = reject_message + "Rejected: Can't read `%s'. [permission denied]\n" % (file)
384 reject_message = reject_message + "Rejected: Can't read `%s'. [file not found]\n" % (file)
386 files[file]["type"] = "unreadable";
388 # If it's byhand skip remaining checks
389 if files[file]["section"] == "byhand":
390 files[file]["byhand"] = 1;
391 files[file]["type"] = "byhand";
392 # Checks for a binary package...
393 elif utils.re_isadeb.match(file) != None:
394 files[file]["type"] = "deb";
396 # Extract package information using dpkg-deb
398 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
400 reject_message = reject_message + "Rejected: %s: debExtractControl() raised %s.\n" % (file, sys.exc_type);
401 # Can't continue, none of the checks on control would work.
404 # Check for mandatory fields
405 if control.Find("Package") == None:
406 reject_message = reject_message + "Rejected: %s: No package field in control.\n" % (file)
407 if control.Find("Architecture") == None:
408 reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file)
409 if control.Find("Version") == None:
410 reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file)
412 # Ensure the package name matches the one give in the .changes
413 if not changes["binary"].has_key(control.Find("Package", "")):
414 reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", ""))
416 # Validate the architecture
417 if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))):
418 reject_message = reject_message + "Rejected: Unknown architecture '%s'.\n" % (control.Find("Architecture", ""))
420 # Check the architecture matches the one given in the .changes
421 if not changes["architecture"].has_key(control.Find("Architecture", "")):
422 reject_message = reject_message + "Rejected: %s: control file lists arch as `%s', which isn't in changes file.\n" % (file, control.Find("Architecture", ""))
423 # Check the section & priority match those given in the .changes (non-fatal)
424 if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"):
425 reject_message = reject_message + "Warning: %s control file lists section as `%s', but changes file has `%s'.\n" % (file, control.Find("Section", ""), files[file]["section"])
426 if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"):
427 reject_message = reject_message + "Warning: %s control file lists priority as `%s', but changes file has `%s'.\n" % (file, control.Find("Priority", ""), files[file]["priority"])
429 epochless_version = utils.re_no_epoch.sub('', control.Find("Version", ""))
431 files[file]["package"] = control.Find("Package");
432 files[file]["architecture"] = control.Find("Architecture");
433 files[file]["version"] = control.Find("Version");
434 files[file]["maintainer"] = control.Find("Maintainer", "");
435 if file[-5:] == ".udeb":
436 files[file]["dbtype"] = "udeb";
437 elif file[-4:] == ".deb":
438 files[file]["dbtype"] = "deb";
440 reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file);
441 files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", ""))
442 files[file]["source"] = control.Find("Source", "");
443 if files[file]["source"] == "":
444 files[file]["source"] = files[file]["package"];
445 # Get the source version
446 source = files[file]["source"];
448 if string.find(source, "(") != -1:
449 m = utils.re_extract_src_version.match(source)
451 source_version = m.group(2)
452 if not source_version:
453 source_version = files[file]["version"];
454 files[file]["source package"] = source;
455 files[file]["source version"] = source_version;
457 # Checks for a source package...
459 m = utils.re_issource.match(file)
461 files[file]["package"] = m.group(1)
462 files[file]["version"] = m.group(2)
463 files[file]["type"] = m.group(3)
465 # Ensure the source package name matches the Source filed in the .changes
466 if changes["source"] != files[file]["package"]:
467 reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"])
469 # Ensure the source version matches the version in the .changes file
470 if files[file]["type"] == "orig.tar.gz":
471 changes_version = changes["chopversion2"]
473 changes_version = changes["chopversion"]
474 if changes_version != files[file]["version"]:
475 reject_message = reject_message + "Rejected: %s: should be %s according to changes file.\n" % (file, changes_version)
477 # Ensure the .changes lists source in the Architecture field
478 if not changes["architecture"].has_key("source"):
479 reject_message = reject_message + "Rejected: %s: changes file doesn't list `source' in Architecture field.\n" % (file)
481 # Check the signature of a .dsc file
482 if files[file]["type"] == "dsc":
483 check_signature(file)
485 files[file]["fullname"] = file
486 files[file]["architecture"] = "source";
488 # Not a binary or source package? Assume byhand...
490 files[file]["byhand"] = 1;
491 files[file]["type"] = "byhand";
493 files[file]["oldfiles"] = {}
494 for suite in changes["distribution"].keys():
496 if files[file].has_key("byhand"):
499 if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])):
500 reject_message = reject_message + "Rejected: unknown component `%s' for suite `%s'.\n" % (files[file]["component"], suite)
503 # See if the package is NEW
504 if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
505 files[file]["new"] = 1
507 if files[file]["type"] == "deb":
508 # Find any old binary packages
509 q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
510 % (files[file]["package"], suite, files[file]["architecture"]))
511 oldfiles = q.dictresult()
512 for oldfile in oldfiles:
513 files[file]["oldfiles"][suite] = oldfile
514 # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
515 if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
516 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
517 # Check for existing copies of the file
518 if not changes.has_key("stable install"):
519 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
520 if q.getresult() != []:
521 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file)
523 # Check for existent source
524 # FIXME: this is no longer per suite
525 if changes["architecture"].has_key("source"):
526 source_version = files[file]["source version"];
527 if source_version != changes["version"]:
528 reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]);
530 if not source_exists (files[file]["source package"], source_version):
531 reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file);
533 # Find any old .dsc files
534 elif files[file]["type"] == "dsc":
535 q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
536 % (files[file]["package"], suite))
537 oldfiles = q.dictresult()
538 if len(oldfiles) >= 1:
539 files[file]["oldfiles"][suite] = oldfiles[0]
541 # Validate the component
542 component = files[file]["component"];
543 component_id = db_access.get_component_id(component);
544 if component_id == -1:
545 reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component);
548 # Validate the priority
549 if string.find(files[file]["priority"],'/') != -1:
550 reject_message = reject_message + "Rejected: file '%s' has invalid priority '%s' [contains '/'].\n" % (file, files[file]["priority"]);
552 # Check the md5sum & size against existing files (if any)
553 location = Cnf["Dir::PoolDir"];
554 files[file]["location id"] = db_access.get_location_id (location, component, archive);
556 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]);
557 files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]);
559 reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file)
561 reject_message = reject_message + "Rejected: md5sum and/or size mismatch on existing copy of %s.\n" % (file)
562 files[file]["files id"] = files_id
564 # Check for packages that have moved from one component to another
565 if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
566 files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
569 if string.find(reject_message, "Rejected:") != -1:
574 ###############################################################################
577 global dsc, dsc_files, reject_message, reprocess, orig_tar_id, orig_tar_location, legacy_source_untouchable;
579 for file in files.keys():
580 if files[file]["type"] == "dsc":
582 dsc = utils.parse_changes(file, 1)
583 except utils.cant_open_exc:
584 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (file)
586 except utils.changes_parse_error_exc, line:
587 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (file, line)
589 except utils.invalid_dsc_format_exc, line:
590 reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (file, line)
593 dsc_files = utils.build_file_list(dsc, 1)
594 except utils.no_files_exc:
595 reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n";
597 except utils.changes_parse_error_exc, line:
598 reject_message = reject_message + "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line);
601 # Enforce mandatory fields
602 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
603 if not dsc.has_key(i):
604 reject_message = reject_message + "Rejected: Missing field `%s' in dsc file.\n" % (i)
606 # The dpkg maintainer from hell strikes again! Bumping the
607 # version number of the .dsc breaks extraction by stable's
609 if dsc["format"] != "1.0":
610 reject_message = reject_message + """Rejected: [dpkg-sucks] source package was produced by a broken version
611 of dpkg-dev 1.9.1{3,4}; please rebuild with >= 1.9.15 version
615 # Ensure the version number in the .dsc matches the version number in the .changes
616 epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version"));
617 changes_version = files[file]["version"];
618 if epochless_dsc_version != files[file]["version"]:
619 reject_message = reject_message + "Rejected: version ('%s') in .dsc does not match version ('%s') in .changes\n" % (epochless_dsc_version, changes_version);
621 # Ensure source is newer than existing source in target suites
622 package = dsc.get("source");
623 new_version = dsc.get("version");
624 for suite in changes["distribution"].keys():
625 q = projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
627 ql = map(lambda x: x[0], q.getresult());
628 for old_version in ql:
629 if apt_pkg.VersionCompare(new_version, old_version) != 1:
630 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, old_version, new_version)
632 # Try and find all files mentioned in the .dsc. This has
633 # to work harder to cope with the multiple possible
634 # locations of an .orig.tar.gz.
635 for dsc_file in dsc_files.keys():
636 if files.has_key(dsc_file):
637 actual_md5 = files[dsc_file]["md5sum"];
638 actual_size = int(files[dsc_file]["size"]);
639 found = "%s in incoming" % (dsc_file)
640 # Check the file does not already exist in the archive
641 if not changes.has_key("stable install"):
642 q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
644 # "It has not broken them. It has fixed a
645 # brokenness. Your crappy hack exploited a
646 # bug in the old dinstall.
648 # "(Come on! I thought it was always obvious
649 # that one just doesn't release different
650 # files with the same name and version.)"
651 # -- ajk@ on d-devel@l.d.o
653 if q.getresult() != []:
654 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file)
655 elif dsc_file[-12:] == ".orig.tar.gz":
657 q = projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
661 # Unfortunately, we make get more than one match
662 # here if, for example, the package was in potato
663 # but had a -sa upload in woody. So we need to a)
664 # choose the right one and b) mark all wrong ones
665 # as excluded from the source poolification (to
666 # avoid file overwrites).
668 x = ql[0]; # default to something sane in case we don't match any or have only one
672 old_file = i[0] + i[1];
673 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
674 actual_size = os.stat(old_file)[stat.ST_SIZE];
675 if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
678 legacy_source_untouchable[i[3]] = "";
680 old_file = x[0] + x[1];
681 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
682 actual_size = os.stat(old_file)[stat.ST_SIZE];
685 dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
688 if suite_type == "legacy" or suite_type == "legacy-mixed":
689 orig_tar_location = "legacy";
691 orig_tar_location = x[4];
693 # Not there? Check in Incoming...
694 # [See comment above process_it() for explanation
695 # of why this is necessary...]
696 if os.path.exists(dsc_file):
697 files[dsc_file] = {};
698 files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
699 files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
700 files[dsc_file]["section"] = files[file]["section"];
701 files[dsc_file]["priority"] = files[file]["priority"];
702 files[dsc_file]["component"] = files[file]["component"];
703 files[dsc_file]["type"] = "orig.tar.gz";
707 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file);
710 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming.\n" % (file, dsc_file);
712 if actual_md5 != dsc_files[dsc_file]["md5sum"]:
713 reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file);
714 if actual_size != int(dsc_files[dsc_file]["size"]):
715 reject_message = reject_message + "Rejected: size for %s doesn't match %s.\n" % (found, file);
717 if string.find(reject_message, "Rejected:") != -1:
722 ###############################################################################
724 # Some cunning stunt broke dpkg-source in dpkg 1.8{,.1}; detect the
725 # resulting bad source packages and reject them.
727 # Even more amusingly the fix in 1.8.1.1 didn't actually fix the
728 # problem just changed the symptoms.
731 global dsc, dsc_files, reject_message, reprocess;
733 for filename in files.keys():
734 if files[filename]["type"] == "diff.gz":
735 file = gzip.GzipFile(filename, 'r');
736 for line in file.readlines():
737 if re_bad_diff.search(line):
738 reject_message = reject_message + "Rejected: [dpkg-sucks] source package was produced by a broken version of dpkg-dev 1.8.x; please rebuild with >= 1.8.3 version installed.\n";
741 if string.find(reject_message, "Rejected:") != -1:
746 ###############################################################################
748 def check_md5sums ():
749 global reject_message;
751 for file in files.keys():
753 file_handle = utils.open_file(file);
754 except utils.cant_open_exc:
757 if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
758 reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file);
760 def check_override ():
763 # Only check section & priority on sourceful non-stable installs
764 if not changes["architecture"].has_key("source") or changes.has_key("stable install"):
768 for file in files.keys():
769 if not files[file].has_key("new") and files[file]["type"] == "deb":
770 section = files[file]["section"];
771 override_section = files[file]["override section"];
772 if section != override_section and section != "-":
773 # Ignore this; it's a common mistake and not worth whining about
774 if string.lower(section) == "non-us/main" and string.lower(override_section) == "non-us":
776 summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
777 priority = files[file]["priority"];
778 override_priority = files[file]["override priority"];
779 if priority != override_priority and priority != "-":
780 summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
785 Subst["__SUMMARY__"] = summary;
786 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.override-disparity","r").read());
787 utils.send_mail (mail_message, "")
789 #####################################################################################################################
791 # Set up the per-package template substitution mappings
793 def update_subst (changes_filename):
796 # If katie crashed out in the right place, architecture may still be a string.
797 if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
798 changes["architecture"] = { "Unknown" : "" };
799 # and maintainer822 may not exist.
800 if not changes.has_key("maintainer822"):
801 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
803 Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
804 Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename);
805 Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
807 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
808 if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
809 Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
810 Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
811 Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
813 Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
814 Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
815 Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
817 Subst["__REJECT_MESSAGE__"] = reject_message;
818 Subst["__SOURCE__"] = changes.get("source", "Unknown");
819 Subst["__VERSION__"] = changes.get("version", "Unknown");
821 #####################################################################################################################
823 def action (changes_filename):
824 byhand = summary = new = "";
826 # changes["distribution"] may not exist in corner cases
827 # (e.g. unreadable changes files)
828 if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
829 changes["distribution"] = {};
833 for suite in changes["distribution"].keys():
834 if Cnf.has_key("Suite::%s::Confirm"):
835 confirm.append(suite)
838 for file in files.keys():
839 if files[file].has_key("byhand"):
841 summary = summary + file + " byhand\n"
842 elif files[file].has_key("new"):
844 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
845 if files[file].has_key("othercomponents"):
846 summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
847 if files[file]["type"] == "deb":
848 summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
850 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
851 destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
852 summary = summary + file + "\n to " + destination + "\n"
854 short_summary = summary;
856 # This is for direport's benefit...
857 f = re_fdnic.sub("\n .\n", changes.get("changes",""));
859 if confirm or byhand or new:
860 summary = summary + "Changes: " + f;
862 summary = summary + announce (short_summary, 0)
864 (prompt, answer) = ("", "XXX")
865 if Options["No-Action"] or Options["Automatic"]:
868 if string.find(reject_message, "Rejected") != -1:
870 modified_time = time.time()-os.path.getmtime(changes_filename);
871 except: # i.e. ignore errors like 'file does not exist';
873 if modified_time < 86400:
874 print "SKIP (too new)\n" + reject_message,;
875 prompt = "[S]kip, Manual reject, Quit ?";
877 print "REJECT\n" + reject_message,;
878 prompt = "[R]eject, Manual reject, Skip, Quit ?";
879 if Options["Automatic"]:
882 print "NEW to %s\n%s%s" % (string.join(suites, ", "), reject_message, summary),;
883 prompt = "[S]kip, New ack, Manual reject, Quit ?";
884 if Options["Automatic"] and Options["Ack-New"]:
887 print "BYHAND\n" + reject_message + summary,;
888 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
890 print "CONFIRM to %s\n%s%s" % (string.join(confirm, ", "), reject_message, summary),
891 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
893 print "INSTALL\n" + reject_message + summary,;
894 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
895 if Options["Automatic"]:
898 while string.find(prompt, answer) == -1:
900 answer = utils.our_raw_input()
901 m = re_default_answer.match(prompt)
904 answer = string.upper(answer[:1])
907 reject (changes_filename, "");
909 manual_reject (changes_filename);
911 install (changes_filename, summary, short_summary);
913 acknowledge_new (changes_filename, summary);
917 #####################################################################################################################
919 def install (changes_filename, summary, short_summary):
920 global install_count, install_bytes, Subst;
922 # stable installs are a special case
923 if changes.has_key("stable install"):
924 stable_install (changes_filename, summary, short_summary);
929 Logger.log(["installing changes",changes_filename]);
931 archive = utils.where_am_i();
933 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
934 projectB.query("BEGIN WORK");
936 # Add the .dsc file to the DB
937 for file in files.keys():
938 if files[file]["type"] == "dsc":
939 package = dsc["source"]
940 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
941 maintainer = dsc["maintainer"]
942 maintainer = string.replace(maintainer, "'", "\\'")
943 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
944 filename = files[file]["pool name"] + file;
945 dsc_location_id = files[file]["location id"];
946 if not files[file]["files id"]:
947 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id)
948 projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)"
949 % (package, version, maintainer_id, files[file]["files id"]))
951 for suite in changes["distribution"].keys():
952 suite_id = db_access.get_suite_id(suite);
953 projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
955 # Add the source files to the DB (files and dsc_files)
956 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"]));
957 for dsc_file in dsc_files.keys():
958 filename = files[file]["pool name"] + dsc_file;
959 # If the .orig.tar.gz is already in the pool, it's
960 # files id is stored in dsc_files by check_dsc().
961 files_id = dsc_files[dsc_file].get("files id", None);
963 files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
964 # FIXME: needs to check for -1/-2 and or handle exception
966 files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
967 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id));
969 # Add the .deb files to the DB
970 for file in files.keys():
971 if files[file]["type"] == "deb":
972 package = files[file]["package"]
973 version = files[file]["version"]
974 maintainer = files[file]["maintainer"]
975 maintainer = string.replace(maintainer, "'", "\\'")
976 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
977 architecture = files[file]["architecture"]
978 architecture_id = db_access.get_architecture_id (architecture);
979 type = files[file]["dbtype"];
980 dsc_component = files[file]["component"]
981 source = files[file]["source package"]
982 source_version = files[file]["source version"];
983 filename = files[file]["pool name"] + file;
984 if not files[file]["files id"]:
985 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
986 source_id = db_access.get_source_id (source, source_version);
988 projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, %d, '%s')"
989 % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type));
991 projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, '%s')"
992 % (package, version, maintainer_id, architecture_id, files[file]["files id"], type));
993 for suite in changes["distribution"].keys():
994 suite_id = db_access.get_suite_id(suite);
995 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id));
997 # If the .orig.tar.gz is in a legacy directory we need to poolify
998 # it, so that apt-get source (and anything else that goes by the
999 # "Directory:" field in the Sources.gz file) works.
1000 if orig_tar_id != None and orig_tar_location == "legacy":
1001 q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id));
1002 qd = q.dictresult();
1004 # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details)
1005 if legacy_source_untouchable.has_key(qid["files_id"]):
1007 # First move the files to the new location
1008 legacy_filename = qid["path"]+qid["filename"];
1009 pool_location = utils.poolify (changes["source"], files[file]["component"]);
1010 pool_filename = pool_location + os.path.basename(qid["filename"]);
1011 destination = Cnf["Dir::PoolDir"] + pool_location
1012 utils.move(legacy_filename, destination);
1013 # Then Update the DB's files table
1014 q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"]));
1016 # If this is a sourceful diff only upload that is moving non-legacy
1017 # cross-component we need to copy the .orig.tar.gz into the new
1018 # component too for the same reasons as above.
1020 if changes["architecture"].has_key("source") and orig_tar_id != None and \
1021 orig_tar_location != "legacy" and orig_tar_location != dsc_location_id:
1022 q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id));
1023 ql = q.getresult()[0];
1024 old_filename = ql[0] + ql[1];
1026 file_md5sum = ql[3];
1027 new_filename = utils.poolify (changes["source"], dsc_component) + os.path.basename(old_filename);
1028 new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1029 if new_files_id == None:
1030 utils.copy(old_filename, Cnf["Dir::PoolDir"] + new_filename);
1031 new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1032 projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id));
1034 # Install the files into the pool
1035 for file in files.keys():
1036 if files[file].has_key("byhand"):
1038 destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file
1039 destdir = os.path.dirname(destination)
1040 utils.move (file, destination)
1041 Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]);
1042 install_bytes = install_bytes + float(files[file]["size"])
1044 # Copy the .changes file across for suite which need it.
1045 for suite in changes["distribution"].keys():
1046 if Cnf.has_key("Suite::%s::CopyChanges" % (suite)):
1047 utils.copy (changes_filename, Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)]);
1049 projectB.query("COMMIT WORK");
1052 utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
1054 utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(changes_filename), sys.exc_type));
1056 install_count = install_count + 1;
1058 if not Options["No-Mail"]:
1059 Subst["__SUITE__"] = "";
1060 Subst["__SUMMARY__"] = summary;
1061 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1062 utils.send_mail (mail_message, "")
1063 announce (short_summary, 1)
1066 #####################################################################################################################
1068 def stable_install (changes_filename, summary, short_summary):
1069 global install_count, install_bytes, Subst;
1071 print "Installing to stable."
1073 archive = utils.where_am_i();
1075 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
1076 projectB.query("BEGIN WORK");
1078 # Add the .dsc file to the DB
1079 for file in files.keys():
1080 if files[file]["type"] == "dsc":
1081 package = dsc["source"]
1082 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
1083 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
1086 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1087 source_id = ql[0][0];
1088 suite_id = db_access.get_suite_id('proposed-updates');
1089 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1090 suite_id = db_access.get_suite_id('stable');
1091 projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id));
1092 install_bytes = install_bytes + float(files[file]["size"])
1094 # Add the .deb files to the DB
1095 for file in files.keys():
1096 if files[file]["type"] == "deb":
1097 package = files[file]["package"]
1098 version = files[file]["version"]
1099 architecture = files[file]["architecture"]
1100 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
1103 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1104 binary_id = ql[0][0];
1105 suite_id = db_access.get_suite_id('proposed-updates');
1106 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1107 suite_id = db_access.get_suite_id('stable');
1108 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id));
1109 install_bytes = install_bytes + float(files[file]["size"])
1111 projectB.query("COMMIT WORK");
1114 utils.move (changes_filename, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(changes_filename));
1116 # Update the Stable ChangeLog file
1118 new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog";
1119 changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog";
1120 if os.path.exists(new_changelog_filename):
1121 os.unlink (new_changelog_filename);
1123 new_changelog = utils.open_file(new_changelog_filename, 'w');
1124 for file in files.keys():
1125 if files[file]["type"] == "deb":
1126 new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file));
1127 elif utils.re_issource.match(file) != None:
1128 new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file));
1130 new_changelog.write("%s\n" % (file));
1131 chop_changes = re_fdnic.sub("\n", changes["changes"]);
1132 new_changelog.write(chop_changes + '\n\n');
1133 if os.access(changelog_filename, os.R_OK) != 0:
1134 changelog = utils.open_file(changelog_filename);
1135 new_changelog.write(changelog.read());
1136 new_changelog.close();
1137 if os.access(changelog_filename, os.R_OK) != 0:
1138 os.unlink(changelog_filename);
1139 utils.move(new_changelog_filename, changelog_filename);
1141 install_count = install_count + 1;
1143 if not Options["No-Mail"]:
1144 Subst["__SUITE__"] = " into stable";
1145 Subst["__SUMMARY__"] = summary;
1146 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1147 utils.send_mail (mail_message, "")
1148 announce (short_summary, 1)
1150 ################################################################################
1152 def reject (changes_filename, manual_reject_mail_filename):
1155 print "Rejecting.\n"
1157 base_changes_filename = os.path.basename(changes_filename);
1158 reason_filename = re_changes.sub("reason", base_changes_filename);
1159 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename);
1161 # Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored)
1163 utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
1165 utils.warn("couldn't reject changes file '%s'. [Got %s]" % (base_changes_filename, sys.exc_type));
1167 if not changes.has_key("stable install"):
1168 for file in files.keys():
1169 if os.path.exists(file):
1171 utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
1173 utils.warn("couldn't reject file '%s'. [Got %s]" % (file, sys.exc_type));
1176 suite_id = db_access.get_suite_id('proposed-updates');
1177 # Remove files from proposed-updates suite
1178 for file in files.keys():
1179 if files[file]["type"] == "dsc":
1180 package = dsc["source"];
1181 version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
1182 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version));
1185 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1186 source_id = ql[0][0];
1187 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1188 elif files[file]["type"] == "deb":
1189 package = files[file]["package"];
1190 version = files[file]["version"];
1191 architecture = files[file]["architecture"];
1192 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture));
1195 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1196 binary_id = ql[0][0];
1197 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1199 # If this is not a manual rejection generate the .reason file and rejection mail message
1200 if manual_reject_mail_filename == "":
1201 if os.path.exists(reject_filename):
1202 os.unlink(reject_filename);
1203 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1204 os.write(fd, reject_message);
1206 Subst["__MANUAL_REJECT_MESSAGE__"] = "";
1207 Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
1208 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
1209 else: # Have a manual rejection file to use
1210 reject_mail_message = ""; # avoid <undef>'s
1212 # Send the rejection mail if appropriate
1213 if not Options["No-Mail"]:
1214 utils.send_mail (reject_mail_message, manual_reject_mail_filename);
1216 Logger.log(["rejected", changes_filename]);
1218 ##################################################################
1220 def manual_reject (changes_filename):
1223 # Build up the rejection email
1224 user_email_address = utils.whoami() + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
1225 manual_reject_message = Options.get("Manual-Reject", "")
1227 Subst["__MANUAL_REJECT_MESSAGE__"] = manual_reject_message;
1228 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
1229 if changes.has_key("stable install"):
1230 template = "katie.stable-rejected";
1232 template = "katie.rejected";
1233 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/"+template,"r").read());
1235 # Write the rejection email out as the <foo>.reason file
1236 reason_filename = re_changes.sub("reason", os.path.basename(changes_filename));
1237 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename)
1238 if os.path.exists(reject_filename):
1239 os.unlink(reject_filename);
1240 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1241 os.write(fd, reject_mail_message);
1244 # If we weren't given one, spawn an editor so the user can add one in
1245 if manual_reject_message == "":
1246 result = os.system("vi +6 %s" % (reject_filename))
1248 utils.fubar("vi invocation failed for `%s'!" % (reject_filename), result);
1250 # Then process it as if it were an automatic rejection
1251 reject (changes_filename, reject_filename)
1253 #####################################################################################################################
1255 def acknowledge_new (changes_filename, summary):
1256 global new_ack_new, Subst;
1258 changes_filename = os.path.basename(changes_filename);
1260 new_ack_new[changes_filename] = 1;
1262 if new_ack_old.has_key(changes_filename):
1263 print "Ack already sent.";
1266 print "Sending new ack.";
1267 if not Options["No-Mail"]:
1268 Subst["__SUMMARY__"] = summary;
1269 new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.new","r").read());
1270 utils.send_mail(new_ack_message,"");
1272 #####################################################################################################################
1274 def announce (short_summary, action):
1277 # Only do announcements for source uploads with a recent dpkg-dev installed
1278 if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
1283 Subst["__SHORT_SUMMARY__"] = short_summary;
1285 for dist in changes["distribution"].keys():
1286 list = Cnf.Find("Suite::%s::Announce" % (dist))
1287 if list == "" or lists_done.has_key(list):
1289 lists_done[list] = 1
1290 summary = summary + "Announcing to %s\n" % (list)
1293 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
1294 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.announce","r").read());
1295 utils.send_mail (mail_message, "")
1297 bugs = changes["closes"].keys()
1299 if not nmu.is_an_nmu(changes, dsc):
1300 summary = summary + "Closing bugs: "
1302 summary = summary + "%s " % (bug)
1304 Subst["__BUG_NUMBER__"] = bug;
1305 if changes["distribution"].has_key("stable"):
1306 Subst["__STABLE_WARNING__"] = """
1307 Note that this package is not part of the released stable Debian
1308 distribution. It may have dependencies on other unreleased software,
1309 or other instabilities. Please take care if you wish to install it.
1310 The update will eventually make its way into the next released Debian
1313 Subst["__STABLE_WARNING__"] = "";
1314 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-close","r").read());
1315 utils.send_mail (mail_message, "")
1317 Logger.log(["closing bugs"]+bugs);
1319 summary = summary + "Setting bugs to severity fixed: "
1320 control_message = ""
1322 summary = summary + "%s " % (bug)
1323 control_message = control_message + "tag %s + fixed\n" % (bug)
1324 if action and control_message != "":
1325 Subst["__CONTROL_MESSAGE__"] = control_message;
1326 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-nmu-fixed","r").read());
1327 utils.send_mail (mail_message, "")
1329 Logger.log(["setting bugs to fixed"]+bugs);
1330 summary = summary + "\n"
1334 ###############################################################################
1336 # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in
1337 # Incoming. -1 will reference the .orig.tar.gz, but -2 will not.
1338 # dsccheckdistrib() can find the .orig.tar.gz but it will not have
1339 # processed it during it's checks of -2. If -1 has been deleted or
1340 # otherwise not checked by da-install, the .orig.tar.gz will not have
1341 # been checked at all. To get round this, we force the .orig.tar.gz
1342 # into the .changes structure and reprocess the .changes file.
1344 def process_it (changes_file):
1345 global reprocess, orig_tar_id, orig_tar_location, changes, dsc, dsc_files, files, reject_message;
1347 # Reset some globals
1354 orig_tar_location = "";
1355 legacy_source_untouchable = {};
1356 reject_message = "";
1358 # Absolutize the filename to avoid the requirement of being in the
1359 # same directory as the .changes file.
1360 changes_file = os.path.abspath(changes_file);
1362 # And since handling of installs to stable munges with the CWD;
1363 # save and restore it.
1367 check_signature (changes_file);
1368 check_changes (changes_file);
1377 traceback.print_exc(file=sys.stdout);
1380 update_subst(changes_file);
1381 action(changes_file);
1386 ###############################################################################
1389 global Cnf, Options, projectB, install_bytes, new_ack_old, Subst, nmu, Logger
1391 changes_files = init();
1396 if Options["Version"]:
1397 print "katie %s" % (katie_version);
1400 # -n/--dry-run invalidates some other options which would involve things happening
1401 if Options["No-Action"]:
1402 Options["Automatic"] = "";
1403 Options["Ack-New"] = "";
1405 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
1407 db_access.init(Cnf, projectB);
1409 # Check that we aren't going to clash with the daily cron job
1411 if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]:
1412 utils.fubar("Archive maintenance in progress. Try again later.");
1414 # Obtain lock if not in no-action mode and initialize the log
1416 if not Options["No-Action"]:
1417 lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT);
1418 fcntl.lockf(lock_fd, FCNTL.F_TLOCK);
1419 Logger = logging.Logger(Cnf, "katie");
1421 if Options["Ack-New"]:
1422 # Read in the list of already-acknowledged NEW packages
1423 if not os.path.exists(Cnf["Dinstall::NewAckList"]):
1424 utils.touch_file(Cnf["Dinstall::NewAckList"]);
1425 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"]);
1427 for line in new_ack_list.readlines():
1428 new_ack_old[line[:-1]] = 1;
1429 new_ack_list.close();
1431 # Initialize the substitution template mapping global
1433 Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
1434 Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
1435 bcc = "X-Katie: %s" % (katie_version);
1436 if Cnf.has_key("Dinstall::Bcc"):
1437 Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
1439 Subst["__BCC__"] = bcc;
1440 Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
1441 Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
1442 Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"];
1444 # Read in the group-maint override file
1447 # Sort the .changes files so that we process sourceful ones first
1448 changes_files.sort(utils.changes_compare);
1450 # Process the changes files
1451 for changes_file in changes_files:
1452 print "\n" + changes_file;
1453 process_it (changes_file);
1457 if install_count > 1:
1459 sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))));
1460 Logger.log(["total",install_count,install_bytes]);
1462 # Write out the list of already-acknowledged NEW packages
1463 if Options["Ack-New"]:
1464 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w')
1465 for i in new_ack_new.keys():
1466 new_ack_list.write(i+'\n')
1467 new_ack_list.close()
1469 if not Options["No-Action"]:
1472 if __name__ == '__main__':