3 # Installs Debian packages
4 # Copyright (C) 2000, 2001 James Troup <james@nocrew.org>
5 # $Id: katie,v 1.63 2001-11-18 19:57:58 rmurray Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 # Originally based almost entirely on dinstall by Guy Maor <maor@debian.org>
23 #########################################################################################
25 # Cartman: "I'm trying to make the best of a bad situation, I don't
26 # need to hear crap from a bunch of hippy freaks living in
27 # denial. Screw you guys, I'm going home."
29 # Kyle: "But Cartman, we're trying to..."
31 # Cartman: "uhh.. screw you guys... home."
33 #########################################################################################
35 import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback
36 import apt_inst, apt_pkg
37 import utils, db_access, logging
41 ###############################################################################
43 re_isanum = re.compile (r"^\d+$");
44 re_changes = re.compile (r"changes$");
45 re_default_answer = re.compile(r"\[(.*)\]");
46 re_fdnic = re.compile("\n\n");
47 re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null");
48 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
49 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
51 #########################################################################################
69 orig_tar_location = "";
70 legacy_source_untouchable = {};
73 katie_version = "$Revision: 1.63 $";
75 ###############################################################################
80 Cnf = utils.get_conf()
82 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
83 ('h',"help","Dinstall::Options::Help"),
84 ('k',"ack-new","Dinstall::Options::Ack-New"),
85 ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
86 ('n',"no-action","Dinstall::Options::No-Action"),
87 ('p',"no-lock", "Dinstall::Options::No-Lock"),
88 ('s',"no-mail", "Dinstall::Options::No-Mail"),
89 ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
90 ('V',"version","Dinstall::Options::Version")];
92 for i in ["automatic", "help", "ack-new", "manual-reject", "no-action",
93 "no-lock", "no-mail", "override-distribution", "version"]:
94 if not Cnf.has_key("Dinstall::Options::%s" % (i)):
95 Cnf["Dinstall::Options::%s" % (i)] = "";
97 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
98 Options = Cnf.SubTree("Dinstall::Options")
100 return changes_files;
102 #########################################################################################
104 def usage (exit_code=0):
105 print """Usage: dinstall [OPTION]... [CHANGES]...
106 -a, --automatic automatic run
107 -h, --help show this help and exit.
108 -k, --ack-new acknowledge new packages !! for cron.daily only !!
109 -m, --manual-reject=MSG manual reject with `msg'
110 -n, --no-action don't do anything
111 -p, --no-lock don't check lockfile !! for cron.daily only !!
112 -s, --no-mail don't send any mail
113 -u, --distribution=DIST override distribution to `dist'
114 -V, --version display the version number and exit"""
117 #########################################################################################
119 def check_signature (filename):
120 global reject_message
122 (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename))
124 reject_message = reject_message + "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output)
128 ######################################################################################################
131 # Read in the group maintainer override file
133 self.group_maint = {};
134 if Cnf.get("Dinstall::GroupOverrideFilename"):
135 filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
136 file = utils.open_file(filename);
137 for line in file.readlines():
138 line = string.strip(utils.re_comments.sub('', line));
140 self.group_maint[line] = 1;
143 def is_an_nmu (self, changes, dsc):
144 (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
145 # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
146 if dsc_name == changes["maintainername"] and (changes["changedby822"] == "" or changes["changedbyname"] == dsc_name):
149 if dsc.has_key("uploaders"):
150 uploaders = string.split(dsc["uploaders"], ",");
153 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
154 uploadernames[name] = "";
155 if uploadernames.has_key(changes["changedbyname"]):
158 # Some group maintained packages (e.g. Debian QA) are never NMU's
159 if self.group_maint.has_key(changes["maintaineremail"]):
164 ######################################################################################################
166 # Ensure that source exists somewhere in the archive for the binary
167 # upload being processed.
169 # (1) exact match => 1.0-3
170 # (2) Bin-only NMU of an MU => 1.0-3.0.1
171 # (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
173 def source_exists (package, source_version):
174 q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
176 # Reduce the query results to a list of version numbers
177 ql = map(lambda x: x[0], q.getresult());
180 if ql.count(source_version):
184 orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
185 if ql.count(orig_source_version):
189 orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
190 if ql.count(orig_source_version):
196 ######################################################################################################
198 # See if a given package is in the override table
200 def in_override_p (package, component, suite, binary_type, file):
203 if binary_type == "": # must be source
208 # Override suite name; used for example with proposed-updates
209 if Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
210 suite = Cnf["Suite::%s::OverrideSuite" % (suite)];
212 # Avoid <undef> on unknown distributions
213 suite_id = db_access.get_suite_id(suite);
216 component_id = db_access.get_component_id(component);
217 type_id = db_access.get_override_type_id(type);
219 # FIXME: nasty non-US speficic hack
220 if string.lower(component[:7]) == "non-us/":
221 component = component[7:];
223 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
224 % (package, suite_id, component_id, type_id));
225 result = q.getresult();
226 # If checking for a source package fall back on the binary override type
227 if type == "dsc" and not result:
228 type_id = db_access.get_override_type_id("deb");
229 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
230 % (package, suite_id, component_id, type_id));
231 result = q.getresult();
233 # Remember the section and priority so we can check them later if appropriate
235 files[file]["override section"] = result[0][0];
236 files[file]["override priority"] = result[0][1];
240 #####################################################################################################################
242 def check_changes(filename):
243 global reject_message, changes, files
245 # Default in case we bail out
246 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
247 changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"];
248 changes["architecture"] = {};
250 # Parse the .changes field into a dictionary
252 changes = utils.parse_changes(filename, 0)
253 except utils.cant_open_exc:
254 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
256 except utils.changes_parse_error_exc, line:
257 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
260 # Parse the Files field from the .changes into another dictionary
262 files = utils.build_file_list(changes, "");
263 except utils.changes_parse_error_exc, line:
264 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line);
265 except utils.nk_format_exc, format:
266 reject_message = reject_message + "Rejected: unknown format '%s' of changes file '%s'.\n" % (format, filename);
269 # Check for mandatory fields
270 for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"):
271 if not changes.has_key(i):
272 reject_message = reject_message + "Rejected: Missing field `%s' in changes file.\n" % (i)
273 return 0 # Avoid <undef> errors during later tests
275 # Override the Distribution: field if appropriate
276 if Options["Override-Distribution"] != "":
277 reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Options["Override-Distribution"])
278 changes["distribution"] = Options["Override-Distribution"]
280 # Split multi-value fields into a lower-level dictionary
281 for i in ("architecture", "distribution", "binary", "closes"):
282 o = changes.get(i, "")
286 for j in string.split(o):
289 # Fix the Maintainer: field to be RFC822 compatible
290 (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
292 # Fix the Changed-By: field to be RFC822 compatible; if it exists.
293 (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
295 # Ensure all the values in Closes: are numbers
296 if changes.has_key("closes"):
297 for i in changes["closes"].keys():
298 if re_isanum.match (i) == None:
299 reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i)
301 # Ensure there _is_ a target distribution
302 if not changes["distribution"].keys():
303 reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n";
305 # Map frozen to unstable if frozen doesn't exist
306 if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"):
307 del changes["distribution"]["frozen"]
308 changes["distribution"]["unstable"] = 1;
309 reject_message = reject_message + "Mapping frozen to unstable.\n"
311 # Map testing to unstable
312 if changes["distribution"].has_key("testing"):
313 if len(changes["distribution"].keys()) > 1:
314 del changes["distribution"]["testing"];
315 reject_message = reject_message + "Warning: Ignoring testing as a target suite.\n";
317 reject_message = reject_message + "Rejected: invalid distribution 'testing'.\n";
319 # Ensure target distributions exist
320 for i in changes["distribution"].keys():
321 if not Cnf.has_key("Suite::%s" % (i)):
322 reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i)
324 # Map unreleased arches from stable to unstable
325 if changes["distribution"].has_key("stable"):
326 for i in changes["architecture"].keys():
327 if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)):
328 reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i)
329 del changes["distribution"]["stable"]
330 changes["distribution"]["unstable"] = 1;
332 # Map arches not being released from frozen to unstable
333 if changes["distribution"].has_key("frozen"):
334 for i in changes["architecture"].keys():
335 if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)):
336 reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i)
337 del changes["distribution"]["frozen"]
338 changes["distribution"]["unstable"] = 1;
340 # Handle uploads to stable
341 if changes["distribution"].has_key("stable"):
342 # If running from within proposed-updates; assume an install to stable
343 if string.find(os.getcwd(), 'proposed-updates') != -1:
344 # Remove non-stable target distributions
345 for dist in changes["distribution"].keys():
347 reject_message = reject_message + "Removing %s from distribution list.\n" % (dist);
348 del changes["distribution"][dist];
349 changes["stable install"] = 1;
350 # If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool
351 file = files.keys()[0];
352 if os.access(file, os.R_OK) == 0:
353 pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]);
354 changes["installing from the pool"] = 1;
356 # Otherwise (normal case) map stable to updates
358 reject_message = reject_message + "Mapping stable to updates.\n";
359 del changes["distribution"]["stable"];
360 changes["distribution"]["proposed-updates"] = 1;
362 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
363 changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"])
364 changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"])
366 if string.find(reject_message, "Rejected:") != -1:
372 global reject_message
374 archive = utils.where_am_i();
376 for file in files.keys():
377 # Check the file is readable
378 if os.access(file,os.R_OK) == 0:
379 if os.path.exists(file):
380 reject_message = reject_message + "Rejected: Can't read `%s'. [permission denied]\n" % (file)
382 reject_message = reject_message + "Rejected: Can't read `%s'. [file not found]\n" % (file)
384 files[file]["type"] = "unreadable";
386 # If it's byhand skip remaining checks
387 if files[file]["section"] == "byhand":
388 files[file]["byhand"] = 1;
389 files[file]["type"] = "byhand";
390 # Checks for a binary package...
391 elif utils.re_isadeb.match(file) != None:
392 files[file]["type"] = "deb";
394 # Extract package information using dpkg-deb
396 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
398 reject_message = reject_message + "Rejected: %s: debExtractControl() raised %s.\n" % (file, sys.exc_type);
399 # Can't continue, none of the checks on control would work.
402 # Check for mandatory fields
403 if control.Find("Package") == None:
404 reject_message = reject_message + "Rejected: %s: No package field in control.\n" % (file)
405 if control.Find("Architecture") == None:
406 reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file)
407 if control.Find("Version") == None:
408 reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file)
410 # Ensure the package name matches the one give in the .changes
411 if not changes["binary"].has_key(control.Find("Package", "")):
412 reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", ""))
414 # Validate the architecture
415 if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))):
416 reject_message = reject_message + "Rejected: Unknown architecture '%s'.\n" % (control.Find("Architecture", ""))
418 # Check the architecture matches the one given in the .changes
419 if not changes["architecture"].has_key(control.Find("Architecture", "")):
420 reject_message = reject_message + "Rejected: %s: control file lists arch as `%s', which isn't in changes file.\n" % (file, control.Find("Architecture", ""))
421 # Check the section & priority match those given in the .changes (non-fatal)
422 if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"):
423 reject_message = reject_message + "Warning: %s control file lists section as `%s', but changes file has `%s'.\n" % (file, control.Find("Section", ""), files[file]["section"])
424 if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"):
425 reject_message = reject_message + "Warning: %s control file lists priority as `%s', but changes file has `%s'.\n" % (file, control.Find("Priority", ""), files[file]["priority"])
427 epochless_version = utils.re_no_epoch.sub('', control.Find("Version", ""))
429 files[file]["package"] = control.Find("Package");
430 files[file]["architecture"] = control.Find("Architecture");
431 files[file]["version"] = control.Find("Version");
432 files[file]["maintainer"] = control.Find("Maintainer", "");
433 if file[-5:] == ".udeb":
434 files[file]["dbtype"] = "udeb";
435 elif file[-4:] == ".deb":
436 files[file]["dbtype"] = "deb";
438 reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file);
439 files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", ""))
440 files[file]["source"] = control.Find("Source", "");
441 if files[file]["source"] == "":
442 files[file]["source"] = files[file]["package"];
443 # Get the source version
444 source = files[file]["source"];
446 if string.find(source, "(") != -1:
447 m = utils.re_extract_src_version.match(source)
449 source_version = m.group(2)
450 if not source_version:
451 source_version = files[file]["version"];
452 files[file]["source package"] = source;
453 files[file]["source version"] = source_version;
455 # Checks for a source package...
457 m = utils.re_issource.match(file)
459 files[file]["package"] = m.group(1)
460 files[file]["version"] = m.group(2)
461 files[file]["type"] = m.group(3)
463 # Ensure the source package name matches the Source filed in the .changes
464 if changes["source"] != files[file]["package"]:
465 reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"])
467 # Ensure the source version matches the version in the .changes file
468 if files[file]["type"] == "orig.tar.gz":
469 changes_version = changes["chopversion2"]
471 changes_version = changes["chopversion"]
472 if changes_version != files[file]["version"]:
473 reject_message = reject_message + "Rejected: %s: should be %s according to changes file.\n" % (file, changes_version)
475 # Ensure the .changes lists source in the Architecture field
476 if not changes["architecture"].has_key("source"):
477 reject_message = reject_message + "Rejected: %s: changes file doesn't list `source' in Architecture field.\n" % (file)
479 # Check the signature of a .dsc file
480 if files[file]["type"] == "dsc":
481 check_signature(file)
483 files[file]["fullname"] = file
484 files[file]["architecture"] = "source";
486 # Not a binary or source package? Assume byhand...
488 files[file]["byhand"] = 1;
489 files[file]["type"] = "byhand";
491 files[file]["oldfiles"] = {}
492 for suite in changes["distribution"].keys():
494 if files[file].has_key("byhand"):
497 if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])):
498 reject_message = reject_message + "Rejected: unknown component `%s' for suite `%s'.\n" % (files[file]["component"], suite)
501 # See if the package is NEW
502 if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
503 files[file]["new"] = 1
505 if files[file]["type"] == "deb":
506 # Find any old binary packages
507 q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
508 % (files[file]["package"], suite, files[file]["architecture"]))
509 oldfiles = q.dictresult()
510 for oldfile in oldfiles:
511 files[file]["oldfiles"][suite] = oldfile
512 # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
513 if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
514 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
515 # Check for existing copies of the file
516 if not changes.has_key("stable install"):
517 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
518 if q.getresult() != []:
519 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file)
521 # Check for existent source
522 # FIXME: this is no longer per suite
523 if changes["architecture"].has_key("source"):
524 source_version = files[file]["source version"];
525 if source_version != changes["version"]:
526 reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]);
528 if not source_exists (files[file]["source package"], source_version):
529 reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file);
531 # Find any old .dsc files
532 elif files[file]["type"] == "dsc":
533 q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
534 % (files[file]["package"], suite))
535 oldfiles = q.dictresult()
536 if len(oldfiles) >= 1:
537 files[file]["oldfiles"][suite] = oldfiles[0]
539 # Validate the component
540 component = files[file]["component"];
541 component_id = db_access.get_component_id(component);
542 if component_id == -1:
543 reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component);
546 # Validate the priority
547 if string.find(files[file]["priority"],'/') != -1:
548 reject_message = reject_message + "Rejected: file '%s' has invalid priority '%s' [contains '/'].\n" % (file, files[file]["priority"]);
550 # Check the md5sum & size against existing files (if any)
551 location = Cnf["Dir::PoolDir"];
552 files[file]["location id"] = db_access.get_location_id (location, component, archive);
554 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]);
555 files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]);
557 reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file)
559 reject_message = reject_message + "Rejected: md5sum and/or size mismatch on existing copy of %s.\n" % (file)
560 files[file]["files id"] = files_id
562 # Check for packages that have moved from one component to another
563 if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
564 files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
567 if string.find(reject_message, "Rejected:") != -1:
572 ###############################################################################
575 global dsc, dsc_files, reject_message, reprocess, orig_tar_id, orig_tar_location, legacy_source_untouchable;
577 for file in files.keys():
578 if files[file]["type"] == "dsc":
580 dsc = utils.parse_changes(file, 1)
581 except utils.cant_open_exc:
582 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (file)
584 except utils.changes_parse_error_exc, line:
585 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (file, line)
587 except utils.invalid_dsc_format_exc, line:
588 reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (file, line)
591 dsc_files = utils.build_file_list(dsc, 1)
592 except utils.no_files_exc:
593 reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n";
595 except utils.changes_parse_error_exc, line:
596 reject_message = reject_message + "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line);
599 # Enforce mandatory fields
600 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
601 if not dsc.has_key(i):
602 reject_message = reject_message + "Rejected: Missing field `%s' in dsc file.\n" % (i)
604 # The dpkg maintainer from hell strikes again! Bumping the
605 # version number of the .dsc breaks extraction by stable's
607 if dsc["format"] != "1.0":
608 reject_message = reject_message + """Rejected: [dpkg-sucks] source package was produced by a broken version
609 of dpkg-dev 1.9.1{3,4}; please rebuild with >= 1.9.15 version
613 # Ensure the version number in the .dsc matches the version number in the .changes
614 epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version"));
615 changes_version = files[file]["version"];
616 if epochless_dsc_version != files[file]["version"]:
617 reject_message = reject_message + "Rejected: version ('%s') in .dsc does not match version ('%s') in .changes\n" % (epochless_dsc_version, changes_version);
619 # Ensure source is newer than existing source in target suites
620 package = dsc.get("source");
621 new_version = dsc.get("version");
622 for suite in changes["distribution"].keys():
623 q = projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
625 ql = map(lambda x: x[0], q.getresult());
626 for old_version in ql:
627 if apt_pkg.VersionCompare(new_version, old_version) != 1:
628 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, old_version, new_version)
630 # Try and find all files mentioned in the .dsc. This has
631 # to work harder to cope with the multiple possible
632 # locations of an .orig.tar.gz.
633 for dsc_file in dsc_files.keys():
634 if files.has_key(dsc_file):
635 actual_md5 = files[dsc_file]["md5sum"];
636 actual_size = int(files[dsc_file]["size"]);
637 found = "%s in incoming" % (dsc_file)
638 # Check the file does not already exist in the archive
639 if not changes.has_key("stable install"):
640 q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
642 # "It has not broken them. It has fixed a
643 # brokenness. Your crappy hack exploited a
644 # bug in the old dinstall.
646 # "(Come on! I thought it was always obvious
647 # that one just doesn't release different
648 # files with the same name and version.)"
649 # -- ajk@ on d-devel@l.d.o
651 if q.getresult() != []:
652 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file)
653 elif dsc_file[-12:] == ".orig.tar.gz":
655 q = projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
659 # Unfortunately, we make get more than one match
660 # here if, for example, the package was in potato
661 # but had a -sa upload in woody. So we need to a)
662 # choose the right one and b) mark all wrong ones
663 # as excluded from the source poolification (to
664 # avoid file overwrites).
666 x = ql[0]; # default to something sane in case we don't match any or have only one
670 old_file = i[0] + i[1];
671 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
672 actual_size = os.stat(old_file)[stat.ST_SIZE];
673 if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
676 legacy_source_untouchable[i[3]] = "";
678 old_file = x[0] + x[1];
679 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
680 actual_size = os.stat(old_file)[stat.ST_SIZE];
683 dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
686 if suite_type == "legacy" or suite_type == "legacy-mixed":
687 orig_tar_location = "legacy";
689 orig_tar_location = x[4];
691 # Not there? Check in Incoming...
692 # [See comment above process_it() for explanation
693 # of why this is necessary...]
694 if os.path.exists(dsc_file):
695 files[dsc_file] = {};
696 files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
697 files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
698 files[dsc_file]["section"] = files[file]["section"];
699 files[dsc_file]["priority"] = files[file]["priority"];
700 files[dsc_file]["component"] = files[file]["component"];
701 files[dsc_file]["type"] = "orig.tar.gz";
705 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file);
708 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming.\n" % (file, dsc_file);
710 if actual_md5 != dsc_files[dsc_file]["md5sum"]:
711 reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file);
712 if actual_size != int(dsc_files[dsc_file]["size"]):
713 reject_message = reject_message + "Rejected: size for %s doesn't match %s.\n" % (found, file);
715 if string.find(reject_message, "Rejected:") != -1:
720 ###############################################################################
722 # Some cunning stunt broke dpkg-source in dpkg 1.8{,.1}; detect the
723 # resulting bad source packages and reject them.
725 # Even more amusingly the fix in 1.8.1.1 didn't actually fix the
726 # problem just changed the symptoms.
729 global dsc, dsc_files, reject_message, reprocess;
731 for filename in files.keys():
732 if files[filename]["type"] == "diff.gz":
733 file = gzip.GzipFile(filename, 'r');
734 for line in file.readlines():
735 if re_bad_diff.search(line):
736 reject_message = reject_message + "Rejected: [dpkg-sucks] source package was produced by a broken version of dpkg-dev 1.8.x; please rebuild with >= 1.8.3 version installed.\n";
739 if string.find(reject_message, "Rejected:") != -1:
744 ###############################################################################
746 def check_md5sums ():
747 global reject_message;
749 for file in files.keys():
751 file_handle = utils.open_file(file);
752 except utils.cant_open_exc:
755 if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
756 reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file);
758 def check_override ():
761 # Only check section & priority on sourceful non-stable installs
762 if not changes["architecture"].has_key("source") or changes.has_key("stable install"):
766 for file in files.keys():
767 if not files[file].has_key("new") and files[file]["type"] == "deb":
768 section = files[file]["section"];
769 override_section = files[file]["override section"];
770 if section != override_section and section != "-":
771 # Ignore this; it's a common mistake and not worth whining about
772 if string.lower(section) == "non-us/main" and string.lower(override_section) == "non-us":
774 summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
775 priority = files[file]["priority"];
776 override_priority = files[file]["override priority"];
777 if priority != override_priority and priority != "-":
778 summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
783 Subst["__SUMMARY__"] = summary;
784 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.override-disparity","r").read());
785 utils.send_mail (mail_message, "")
787 #####################################################################################################################
789 # Set up the per-package template substitution mappings
791 def update_subst (changes_filename):
794 # If katie crashed out in the right place, architecture may still be a string.
795 if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
796 changes["architecture"] = { "Unknown" : "" };
797 # and maintainer822 may not exist.
798 if not changes.has_key("maintainer822"):
799 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
801 Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
802 Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename);
803 Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
805 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
806 if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
807 Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
808 Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
809 Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
811 Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
812 Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
813 Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
815 Subst["__REJECT_MESSAGE__"] = reject_message;
816 Subst["__SOURCE__"] = changes.get("source", "Unknown");
817 Subst["__VERSION__"] = changes.get("version", "Unknown");
819 #####################################################################################################################
821 def action (changes_filename):
822 byhand = summary = new = "";
824 # changes["distribution"] may not exist in corner cases
825 # (e.g. unreadable changes files)
826 if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
827 changes["distribution"] = {};
831 for suite in changes["distribution"].keys():
832 if Cnf.has_key("Suite::%s::Confirm"):
833 confirm.append(suite)
836 for file in files.keys():
837 if files[file].has_key("byhand"):
839 summary = summary + file + " byhand\n"
840 elif files[file].has_key("new"):
842 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
843 if files[file].has_key("othercomponents"):
844 summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
845 if files[file]["type"] == "deb":
846 summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
848 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
849 destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
850 summary = summary + file + "\n to " + destination + "\n"
852 short_summary = summary;
854 # This is for direport's benefit...
855 f = re_fdnic.sub("\n .\n", changes.get("changes",""));
857 if confirm or byhand or new:
858 summary = summary + "Changes: " + f;
860 summary = summary + announce (short_summary, 0)
862 (prompt, answer) = ("", "XXX")
863 if Options["No-Action"] or Options["Automatic"]:
866 if string.find(reject_message, "Rejected") != -1:
868 modified_time = time.time()-os.path.getmtime(changes_filename);
869 except: # i.e. ignore errors like 'file does not exist';
871 if modified_time < 86400:
872 print "SKIP (too new)\n" + reject_message,;
873 prompt = "[S]kip, Manual reject, Quit ?";
875 print "REJECT\n" + reject_message,;
876 prompt = "[R]eject, Manual reject, Skip, Quit ?";
877 if Options["Automatic"]:
880 print "NEW to %s\n%s%s" % (string.join(suites, ", "), reject_message, summary),;
881 prompt = "[S]kip, New ack, Manual reject, Quit ?";
882 if Options["Automatic"] and Options["Ack-New"]:
885 print "BYHAND\n" + reject_message + summary,;
886 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
888 print "CONFIRM to %s\n%s%s" % (string.join(confirm, ", "), reject_message, summary),
889 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
891 print "INSTALL\n" + reject_message + summary,;
892 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
893 if Options["Automatic"]:
896 while string.find(prompt, answer) == -1:
898 answer = utils.our_raw_input()
899 m = re_default_answer.match(prompt)
902 answer = string.upper(answer[:1])
905 reject (changes_filename, "");
907 manual_reject (changes_filename);
909 install (changes_filename, summary, short_summary);
911 acknowledge_new (changes_filename, summary);
915 #####################################################################################################################
917 def install (changes_filename, summary, short_summary):
918 global install_count, install_bytes, Subst;
920 # stable installs are a special case
921 if changes.has_key("stable install"):
922 stable_install (changes_filename, summary, short_summary);
927 Logger.log(["installing changes",changes_filename]);
929 archive = utils.where_am_i();
931 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
932 projectB.query("BEGIN WORK");
934 # Add the .dsc file to the DB
935 for file in files.keys():
936 if files[file]["type"] == "dsc":
937 package = dsc["source"]
938 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
939 maintainer = dsc["maintainer"]
940 maintainer = string.replace(maintainer, "'", "\\'")
941 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
942 filename = files[file]["pool name"] + file;
943 dsc_location_id = files[file]["location id"];
944 if not files[file]["files id"]:
945 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id)
946 projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)"
947 % (package, version, maintainer_id, files[file]["files id"]))
949 for suite in changes["distribution"].keys():
950 suite_id = db_access.get_suite_id(suite);
951 projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
953 # Add the source files to the DB (files and dsc_files)
954 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"]));
955 for dsc_file in dsc_files.keys():
956 filename = files[file]["pool name"] + dsc_file;
957 # If the .orig.tar.gz is already in the pool, it's
958 # files id is stored in dsc_files by check_dsc().
959 files_id = dsc_files[dsc_file].get("files id", None);
961 files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
962 # FIXME: needs to check for -1/-2 and or handle exception
964 files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
965 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id));
967 # Add the .deb files to the DB
968 for file in files.keys():
969 if files[file]["type"] == "deb":
970 package = files[file]["package"]
971 version = files[file]["version"]
972 maintainer = files[file]["maintainer"]
973 maintainer = string.replace(maintainer, "'", "\\'")
974 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
975 architecture = files[file]["architecture"]
976 architecture_id = db_access.get_architecture_id (architecture);
977 type = files[file]["dbtype"];
978 dsc_component = files[file]["component"]
979 source = files[file]["source package"]
980 source_version = files[file]["source version"];
981 filename = files[file]["pool name"] + file;
982 if not files[file]["files id"]:
983 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
984 source_id = db_access.get_source_id (source, source_version);
986 projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, %d, '%s')"
987 % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type));
989 projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, '%s')"
990 % (package, version, maintainer_id, architecture_id, files[file]["files id"], type));
991 for suite in changes["distribution"].keys():
992 suite_id = db_access.get_suite_id(suite);
993 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id));
995 # If the .orig.tar.gz is in a legacy directory we need to poolify
996 # it, so that apt-get source (and anything else that goes by the
997 # "Directory:" field in the Sources.gz file) works.
998 if orig_tar_id != None and orig_tar_location == "legacy":
999 q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id));
1000 qd = q.dictresult();
1002 # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details)
1003 if legacy_source_untouchable.has_key(qid["files_id"]):
1005 # First move the files to the new location
1006 legacy_filename = qid["path"]+qid["filename"];
1007 pool_location = utils.poolify (changes["source"], files[file]["component"]);
1008 pool_filename = pool_location + os.path.basename(qid["filename"]);
1009 destination = Cnf["Dir::PoolDir"] + pool_location
1010 utils.move(legacy_filename, destination);
1011 # Then Update the DB's files table
1012 q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"]));
1014 # If this is a sourceful diff only upload that is moving non-legacy
1015 # cross-component we need to copy the .orig.tar.gz into the new
1016 # component too for the same reasons as above.
1018 if changes["architecture"].has_key("source") and orig_tar_id != None and \
1019 orig_tar_location != "legacy" and orig_tar_location != dsc_location_id:
1020 q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id));
1021 ql = q.getresult()[0];
1022 old_filename = ql[0] + ql[1];
1024 file_md5sum = ql[3];
1025 new_filename = utils.poolify (changes["source"], dsc_component) + os.path.basename(old_filename);
1026 new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1027 if new_files_id == None:
1028 utils.copy(old_filename, Cnf["Dir::PoolDir"] + new_filename);
1029 new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1030 projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id));
1032 # Install the files into the pool
1033 for file in files.keys():
1034 if files[file].has_key("byhand"):
1036 destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file
1037 destdir = os.path.dirname(destination)
1038 utils.move (file, destination)
1039 Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]);
1040 install_bytes = install_bytes + float(files[file]["size"])
1042 # Copy the .changes file across for suite which need it.
1043 for suite in changes["distribution"].keys():
1044 if Cnf.has_key("Suite::%s::CopyChanges" % (suite)):
1045 utils.copy (changes_filename, Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)]);
1047 projectB.query("COMMIT WORK");
1050 utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
1052 utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(changes_filename), sys.exc_type));
1054 install_count = install_count + 1;
1056 if not Options["No-Mail"]:
1057 Subst["__SUITE__"] = "";
1058 Subst["__SUMMARY__"] = summary;
1059 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1060 utils.send_mail (mail_message, "")
1061 announce (short_summary, 1)
1064 #####################################################################################################################
1066 def stable_install (changes_filename, summary, short_summary):
1067 global install_count, install_bytes, Subst;
1069 print "Installing to stable."
1071 archive = utils.where_am_i();
1073 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
1074 projectB.query("BEGIN WORK");
1076 # Add the .dsc file to the DB
1077 for file in files.keys():
1078 if files[file]["type"] == "dsc":
1079 package = dsc["source"]
1080 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
1081 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
1084 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1085 source_id = ql[0][0];
1086 suite_id = db_access.get_suite_id('proposed-updates');
1087 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1088 suite_id = db_access.get_suite_id('stable');
1089 projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id));
1090 install_bytes = install_bytes + float(files[file]["size"])
1092 # Add the .deb files to the DB
1093 for file in files.keys():
1094 if files[file]["type"] == "deb":
1095 package = files[file]["package"]
1096 version = files[file]["version"]
1097 architecture = files[file]["architecture"]
1098 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
1101 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1102 binary_id = ql[0][0];
1103 suite_id = db_access.get_suite_id('proposed-updates');
1104 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1105 suite_id = db_access.get_suite_id('stable');
1106 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id));
1107 install_bytes = install_bytes + float(files[file]["size"])
1109 projectB.query("COMMIT WORK");
1112 utils.move (changes_filename, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(changes_filename));
1114 # Update the Stable ChangeLog file
1116 new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog";
1117 changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog";
1118 if os.path.exists(new_changelog_filename):
1119 os.unlink (new_changelog_filename);
1121 new_changelog = utils.open_file(new_changelog_filename, 'w');
1122 for file in files.keys():
1123 if files[file]["type"] == "deb":
1124 new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file));
1125 elif utils.re_issource.match(file) != None:
1126 new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file));
1128 new_changelog.write("%s\n" % (file));
1129 chop_changes = re_fdnic.sub("\n", changes["changes"]);
1130 new_changelog.write(chop_changes + '\n\n');
1131 if os.access(changelog_filename, os.R_OK) != 0:
1132 changelog = utils.open_file(changelog_filename);
1133 new_changelog.write(changelog.read());
1134 new_changelog.close();
1135 if os.access(changelog_filename, os.R_OK) != 0:
1136 os.unlink(changelog_filename);
1137 utils.move(new_changelog_filename, changelog_filename);
1139 install_count = install_count + 1;
1141 if not Options["No-Mail"]:
1142 Subst["__SUITE__"] = " into stable";
1143 Subst["__SUMMARY__"] = summary;
1144 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1145 utils.send_mail (mail_message, "")
1146 announce (short_summary, 1)
1148 ################################################################################
1150 def reject (changes_filename, manual_reject_mail_filename):
1153 print "Rejecting.\n"
1155 base_changes_filename = os.path.basename(changes_filename);
1156 reason_filename = re_changes.sub("reason", base_changes_filename);
1157 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename);
1159 # Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored)
1161 utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
1163 utils.warn("couldn't reject changes file '%s'. [Got %s]" % (base_changes_filename, sys.exc_type));
1165 if not changes.has_key("stable install"):
1166 for file in files.keys():
1167 if os.path.exists(file):
1169 utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
1171 utils.warn("couldn't reject file '%s'. [Got %s]" % (file, sys.exc_type));
1174 suite_id = db_access.get_suite_id('proposed-updates');
1175 # Remove files from proposed-updates suite
1176 for file in files.keys():
1177 if files[file]["type"] == "dsc":
1178 package = dsc["source"];
1179 version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
1180 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version));
1183 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1184 source_id = ql[0][0];
1185 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1186 elif files[file]["type"] == "deb":
1187 package = files[file]["package"];
1188 version = files[file]["version"];
1189 architecture = files[file]["architecture"];
1190 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture));
1193 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1194 binary_id = ql[0][0];
1195 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1197 # If this is not a manual rejection generate the .reason file and rejection mail message
1198 if manual_reject_mail_filename == "":
1199 if os.path.exists(reject_filename):
1200 os.unlink(reject_filename);
1201 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1202 os.write(fd, reject_message);
1204 Subst["__MANUAL_REJECT_MESSAGE__"] = "";
1205 Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
1206 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
1207 else: # Have a manual rejection file to use
1208 reject_mail_message = ""; # avoid <undef>'s
1210 # Send the rejection mail if appropriate
1211 if not Options["No-Mail"]:
1212 utils.send_mail (reject_mail_message, manual_reject_mail_filename);
1214 Logger.log(["rejected", changes_filename]);
1216 ##################################################################
1218 def manual_reject (changes_filename):
1221 # Build up the rejection email
1222 user_email_address = utils.whoami() + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
1223 manual_reject_message = Options.get("Manual-Reject", "")
1225 Subst["__MANUAL_REJECT_MESSAGE__"] = manual_reject_message;
1226 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
1227 if changes.has_key("stable install"):
1228 template = "katie.stable-rejected";
1230 template = "katie.rejected";
1231 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/"+template,"r").read());
1233 # Write the rejection email out as the <foo>.reason file
1234 reason_filename = re_changes.sub("reason", os.path.basename(changes_filename));
1235 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename)
1236 if os.path.exists(reject_filename):
1237 os.unlink(reject_filename);
1238 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1239 os.write(fd, reject_mail_message);
1242 # If we weren't given one, spawn an editor so the user can add one in
1243 if manual_reject_message == "":
1244 result = os.system("vi +6 %s" % (reject_filename))
1246 utils.fubar("vi invocation failed for `%s'!" % (reject_filename), result);
1248 # Then process it as if it were an automatic rejection
1249 reject (changes_filename, reject_filename)
1251 #####################################################################################################################
1253 def acknowledge_new (changes_filename, summary):
1254 global new_ack_new, Subst;
1256 changes_filename = os.path.basename(changes_filename);
1258 new_ack_new[changes_filename] = 1;
1260 if new_ack_old.has_key(changes_filename):
1261 print "Ack already sent.";
1264 print "Sending new ack.";
1265 if not Options["No-Mail"]:
1266 Subst["__SUMMARY__"] = summary;
1267 new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.new","r").read());
1268 utils.send_mail(new_ack_message,"");
1270 #####################################################################################################################
1272 def announce (short_summary, action):
1275 # Only do announcements for source uploads with a recent dpkg-dev installed
1276 if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
1281 Subst["__SHORT_SUMMARY__"] = short_summary;
1283 for dist in changes["distribution"].keys():
1284 list = Cnf.Find("Suite::%s::Announce" % (dist))
1285 if list == "" or lists_done.has_key(list):
1287 lists_done[list] = 1
1288 summary = summary + "Announcing to %s\n" % (list)
1291 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
1292 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.announce","r").read());
1293 utils.send_mail (mail_message, "")
1295 bugs = changes["closes"].keys()
1297 if not nmu.is_an_nmu(changes, dsc):
1298 summary = summary + "Closing bugs: "
1300 summary = summary + "%s " % (bug)
1302 Subst["__BUG_NUMBER__"] = bug;
1303 if changes["distribution"].has_key("stable"):
1304 Subst["__STABLE_WARNING__"] = """
1305 Note that this package is not part of the released stable Debian
1306 distribution. It may have dependencies on other unreleased software,
1307 or other instabilities. Please take care if you wish to install it.
1308 The update will eventually make its way into the next released Debian
1311 Subst["__STABLE_WARNING__"] = "";
1312 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-close","r").read());
1313 utils.send_mail (mail_message, "")
1315 Logger.log(["closing bugs"]+bugs);
1317 summary = summary + "Setting bugs to severity fixed: "
1318 control_message = ""
1320 summary = summary + "%s " % (bug)
1321 control_message = control_message + "tag %s + fixed\n" % (bug)
1322 if action and control_message != "":
1323 Subst["__CONTROL_MESSAGE__"] = control_message;
1324 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-nmu-fixed","r").read());
1325 utils.send_mail (mail_message, "")
1327 Logger.log(["setting bugs to fixed"]+bugs);
1328 summary = summary + "\n"
1332 ###############################################################################
1334 # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in
1335 # Incoming. -1 will reference the .orig.tar.gz, but -2 will not.
1336 # dsccheckdistrib() can find the .orig.tar.gz but it will not have
1337 # processed it during it's checks of -2. If -1 has been deleted or
1338 # otherwise not checked by da-install, the .orig.tar.gz will not have
1339 # been checked at all. To get round this, we force the .orig.tar.gz
1340 # into the .changes structure and reprocess the .changes file.
1342 def process_it (changes_file):
1343 global reprocess, orig_tar_id, orig_tar_location, changes, dsc, dsc_files, files, reject_message;
1345 # Reset some globals
1352 orig_tar_location = "";
1353 legacy_source_untouchable = {};
1354 reject_message = "";
1356 # Absolutize the filename to avoid the requirement of being in the
1357 # same directory as the .changes file.
1358 changes_file = os.path.abspath(changes_file);
1360 # And since handling of installs to stable munges with the CWD;
1361 # save and restore it.
1365 check_signature (changes_file);
1366 check_changes (changes_file);
1375 traceback.print_exc(file=sys.stdout);
1378 update_subst(changes_file);
1379 action(changes_file);
1384 ###############################################################################
1387 global Cnf, Options, projectB, install_bytes, new_ack_old, Subst, nmu, Logger
1389 changes_files = init();
1394 if Options["Version"]:
1395 print "katie %s" % (katie_version);
1398 # -n/--dry-run invalidates some other options which would involve things happening
1399 if Options["No-Action"]:
1400 Options["Automatic"] = "";
1401 Options["Ack-New"] = "";
1403 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
1405 db_access.init(Cnf, projectB);
1407 # Check that we aren't going to clash with the daily cron job
1409 if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]:
1410 utils.fubar("Archive maintenance in progress. Try again later.");
1412 # Obtain lock if not in no-action mode and initialize the log
1414 if not Options["No-Action"]:
1415 lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT);
1416 fcntl.lockf(lock_fd, FCNTL.F_TLOCK);
1417 Logger = logging.Logger(Cnf, "katie");
1419 if Options["Ack-New"]:
1420 # Read in the list of already-acknowledged NEW packages
1421 if not os.path.exists(Cnf["Dinstall::NewAckList"]):
1422 utils.touch_file(Cnf["Dinstall::NewAckList"]);
1423 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"]);
1425 for line in new_ack_list.readlines():
1426 new_ack_old[line[:-1]] = 1;
1427 new_ack_list.close();
1429 # Initialize the substitution template mapping global
1431 Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
1432 Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
1433 bcc = "X-Katie: %s" % (katie_version);
1434 if Cnf.has_key("Dinstall::Bcc"):
1435 Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
1437 Subst["__BCC__"] = bcc;
1438 Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
1439 Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
1440 Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"];
1442 # Read in the group-maint override file
1445 # Sort the .changes files so that we process sourceful ones first
1446 changes_files.sort(utils.changes_compare);
1448 # Process the changes files
1449 for changes_file in changes_files:
1450 print "\n" + changes_file;
1451 process_it (changes_file);
1455 if install_count > 1:
1457 sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))));
1458 Logger.log(["total",install_count,install_bytes]);
1460 # Write out the list of already-acknowledged NEW packages
1461 if Options["Ack-New"]:
1462 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w')
1463 for i in new_ack_new.keys():
1464 new_ack_list.write(i+'\n')
1465 new_ack_list.close()
1467 if not Options["No-Action"]:
1470 if __name__ == '__main__':