3 # Installs Debian packages
4 # Copyright (C) 2000, 2001 James Troup <james@nocrew.org>
5 # $Id: katie,v 1.68 2002-01-28 18:53:01 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 # Originally based almost entirely on dinstall by Guy Maor <maor@debian.org>
23 ###############################################################################
25 # Cartman: "I'm trying to make the best of a bad situation, I don't
26 # need to hear crap from a bunch of hippy freaks living in
27 # denial. Screw you guys, I'm going home."
29 # Kyle: "But Cartman, we're trying to..."
31 # Cartman: "uhh.. screw you guys... home."
33 ###############################################################################
35 import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback
36 import apt_inst, apt_pkg
37 import utils, db_access, logging
41 ###############################################################################
43 re_isanum = re.compile (r"^\d+$");
44 re_changes = re.compile (r"changes$");
45 re_default_answer = re.compile(r"\[(.*)\]");
46 re_fdnic = re.compile("\n\n");
47 re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null");
48 re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
49 re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
51 #########################################################################################
69 orig_tar_location = "";
70 legacy_source_untouchable = {};
73 katie_version = "$Revision: 1.68 $";
75 ###############################################################################
80 Cnf = utils.get_conf()
82 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
83 ('h',"help","Dinstall::Options::Help"),
84 ('k',"ack-new","Dinstall::Options::Ack-New"),
85 ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
86 ('n',"no-action","Dinstall::Options::No-Action"),
87 ('p',"no-lock", "Dinstall::Options::No-Lock"),
88 ('s',"no-mail", "Dinstall::Options::No-Mail"),
89 ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
90 ('V',"version","Dinstall::Options::Version")];
92 for i in ["automatic", "help", "ack-new", "manual-reject", "no-action",
93 "no-lock", "no-mail", "override-distribution", "version"]:
94 if not Cnf.has_key("Dinstall::Options::%s" % (i)):
95 Cnf["Dinstall::Options::%s" % (i)] = "";
97 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
98 Options = Cnf.SubTree("Dinstall::Options")
100 return changes_files;
102 ###############################################################################
104 def usage (exit_code=0):
105 print """Usage: dinstall [OPTION]... [CHANGES]...
106 -a, --automatic automatic run
107 -h, --help show this help and exit.
108 -k, --ack-new acknowledge new packages !! for cron.daily only !!
109 -m, --manual-reject=MSG manual reject with `msg'
110 -n, --no-action don't do anything
111 -p, --no-lock don't check lockfile !! for cron.daily only !!
112 -s, --no-mail don't send any mail
113 -u, --distribution=DIST override distribution to `dist'
114 -V, --version display the version number and exit"""
117 #########################################################################################
119 def check_signature (filename):
120 global reject_message
122 (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename))
124 reject_message = reject_message + "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output)
128 ######################################################################################################
131 # Read in the group maintainer override file
133 self.group_maint = {};
134 if Cnf.get("Dinstall::GroupOverrideFilename"):
135 filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
136 file = utils.open_file(filename);
137 for line in file.readlines():
138 line = string.strip(utils.re_comments.sub('', line));
140 self.group_maint[line] = 1;
143 def is_an_nmu (self, changes, dsc):
144 (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
145 # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
146 if dsc_name == changes["maintainername"] and (changes["changedby822"] == "" or changes["changedbyname"] == dsc_name):
149 if dsc.has_key("uploaders"):
150 uploaders = string.split(dsc["uploaders"], ",");
153 (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
154 uploadernames[name] = "";
155 if uploadernames.has_key(changes["changedbyname"]):
158 # Some group maintained packages (e.g. Debian QA) are never NMU's
159 if self.group_maint.has_key(changes["maintaineremail"]):
164 ######################################################################################################
166 # Ensure that source exists somewhere in the archive for the binary
167 # upload being processed.
169 # (1) exact match => 1.0-3
170 # (2) Bin-only NMU of an MU => 1.0-3.0.1
171 # (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
173 def source_exists (package, source_version):
174 q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
176 # Reduce the query results to a list of version numbers
177 ql = map(lambda x: x[0], q.getresult());
180 if ql.count(source_version):
184 orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
185 if ql.count(orig_source_version):
189 orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
190 if ql.count(orig_source_version):
196 ######################################################################################################
198 # See if a given package is in the override table
200 def in_override_p (package, component, suite, binary_type, file):
203 if binary_type == "": # must be source
208 # Override suite name; used for example with proposed-updates
209 if Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "":
210 suite = Cnf["Suite::%s::OverrideSuite" % (suite)];
212 # Avoid <undef> on unknown distributions
213 suite_id = db_access.get_suite_id(suite);
216 component_id = db_access.get_component_id(component);
217 type_id = db_access.get_override_type_id(type);
219 # FIXME: nasty non-US speficic hack
220 if string.lower(component[:7]) == "non-us/":
221 component = component[7:];
223 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
224 % (package, suite_id, component_id, type_id));
225 result = q.getresult();
226 # If checking for a source package fall back on the binary override type
227 if type == "dsc" and not result:
228 type_id = db_access.get_override_type_id("deb");
229 q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
230 % (package, suite_id, component_id, type_id));
231 result = q.getresult();
233 # Remember the section and priority so we can check them later if appropriate
235 files[file]["override section"] = result[0][0];
236 files[file]["override priority"] = result[0][1];
240 #####################################################################################################################
242 def check_changes(filename):
243 global reject_message, changes, files
245 # Default in case we bail out
246 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
247 changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"];
248 changes["architecture"] = {};
250 # Parse the .changes field into a dictionary
252 changes = utils.parse_changes(filename, 0)
253 except utils.cant_open_exc:
254 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
256 except utils.changes_parse_error_exc, line:
257 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
260 # Parse the Files field from the .changes into another dictionary
262 files = utils.build_file_list(changes, "");
263 except utils.changes_parse_error_exc, line:
264 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line);
265 except utils.nk_format_exc, format:
266 reject_message = reject_message + "Rejected: unknown format '%s' of changes file '%s'.\n" % (format, filename);
269 # Check for mandatory fields
270 for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"):
271 if not changes.has_key(i):
272 reject_message = reject_message + "Rejected: Missing field `%s' in changes file.\n" % (i)
273 return 0 # Avoid <undef> errors during later tests
275 # Override the Distribution: field if appropriate
276 if Options["Override-Distribution"] != "":
277 reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Options["Override-Distribution"])
278 changes["distribution"] = Options["Override-Distribution"]
280 # Split multi-value fields into a lower-level dictionary
281 for i in ("architecture", "distribution", "binary", "closes"):
282 o = changes.get(i, "")
286 for j in string.split(o):
289 # Fix the Maintainer: field to be RFC822 compatible
290 (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
292 # Fix the Changed-By: field to be RFC822 compatible; if it exists.
293 (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
295 # Ensure all the values in Closes: are numbers
296 if changes.has_key("closes"):
297 for i in changes["closes"].keys():
298 if re_isanum.match (i) == None:
299 reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i)
301 # Ensure there _is_ a target distribution
302 if not changes["distribution"].keys():
303 reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n";
305 # Map frozen to unstable if frozen doesn't exist
306 if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"):
307 del changes["distribution"]["frozen"]
308 changes["distribution"]["unstable"] = 1;
309 reject_message = reject_message + "Mapping frozen to unstable.\n"
311 # Map testing to unstable
312 if changes["distribution"].has_key("testing"):
313 if len(changes["distribution"].keys()) > 1:
314 del changes["distribution"]["testing"];
315 reject_message = reject_message + "Warning: Ignoring testing as a target suite.\n";
317 reject_message = reject_message + "Rejected: invalid distribution 'testing'.\n";
319 # Ensure target distributions exist
320 for i in changes["distribution"].keys():
321 if not Cnf.has_key("Suite::%s" % (i)):
322 reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i)
324 # Map unreleased arches from stable to unstable
325 if changes["distribution"].has_key("stable"):
326 for i in changes["architecture"].keys():
327 if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)):
328 reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i)
329 del changes["distribution"]["stable"]
330 changes["distribution"]["unstable"] = 1;
332 # Map arches not being released from frozen to unstable
333 if changes["distribution"].has_key("frozen"):
334 for i in changes["architecture"].keys():
335 if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)):
336 reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i)
337 del changes["distribution"]["frozen"]
338 changes["distribution"]["unstable"] = 1;
340 # Handle uploads to stable
341 if changes["distribution"].has_key("stable"):
342 # If running from within proposed-updates; assume an install to stable
343 if string.find(os.getcwd(), 'proposed-updates') != -1:
344 # Remove non-stable target distributions
345 for dist in changes["distribution"].keys():
347 reject_message = reject_message + "Removing %s from distribution list.\n" % (dist);
348 del changes["distribution"][dist];
349 changes["stable install"] = 1;
350 # If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool
351 file = files.keys()[0];
352 if os.access(file, os.R_OK) == 0:
353 pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]);
354 changes["installing from the pool"] = 1;
356 # Otherwise (normal case) map stable to updates
358 reject_message = reject_message + "Mapping stable to updates.\n";
359 del changes["distribution"]["stable"];
360 changes["distribution"]["proposed-updates"] = 1;
362 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
363 changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"])
364 changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"])
366 if string.find(reject_message, "Rejected:") != -1:
372 global reject_message
374 archive = utils.where_am_i();
376 for file in files.keys():
377 # Check the file is readable
378 if os.access(file,os.R_OK) == 0:
379 if os.path.exists(file):
380 reject_message = reject_message + "Rejected: Can't read `%s'. [permission denied]\n" % (file)
382 reject_message = reject_message + "Rejected: Can't read `%s'. [file not found]\n" % (file)
384 files[file]["type"] = "unreadable";
386 # If it's byhand skip remaining checks
387 if files[file]["section"] == "byhand":
388 files[file]["byhand"] = 1;
389 files[file]["type"] = "byhand";
390 # Checks for a binary package...
391 elif utils.re_isadeb.match(file) != None:
392 files[file]["type"] = "deb";
394 # Extract package information using dpkg-deb
396 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
398 reject_message = reject_message + "Rejected: %s: debExtractControl() raised %s.\n" % (file, sys.exc_type);
399 # Can't continue, none of the checks on control would work.
402 # Check for mandatory fields
403 if control.Find("Package") == None:
404 reject_message = reject_message + "Rejected: %s: No package field in control.\n" % (file)
405 if control.Find("Architecture") == None:
406 reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file)
407 if control.Find("Version") == None:
408 reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file)
410 # Ensure the package name matches the one give in the .changes
411 if not changes["binary"].has_key(control.Find("Package", "")):
412 reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", ""))
414 # Validate the architecture
415 if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))):
416 reject_message = reject_message + "Rejected: Unknown architecture '%s'.\n" % (control.Find("Architecture", ""))
418 # Check the architecture matches the one given in the .changes
419 if not changes["architecture"].has_key(control.Find("Architecture", "")):
420 reject_message = reject_message + "Rejected: %s: control file lists arch as `%s', which isn't in changes file.\n" % (file, control.Find("Architecture", ""))
421 # Check the section & priority match those given in the .changes (non-fatal)
422 if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"):
423 reject_message = reject_message + "Warning: %s control file lists section as `%s', but changes file has `%s'.\n" % (file, control.Find("Section", ""), files[file]["section"])
424 if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"):
425 reject_message = reject_message + "Warning: %s control file lists priority as `%s', but changes file has `%s'.\n" % (file, control.Find("Priority", ""), files[file]["priority"])
427 epochless_version = utils.re_no_epoch.sub('', control.Find("Version", ""))
429 files[file]["package"] = control.Find("Package");
430 files[file]["architecture"] = control.Find("Architecture");
431 files[file]["version"] = control.Find("Version");
432 files[file]["maintainer"] = control.Find("Maintainer", "");
433 if file[-5:] == ".udeb":
434 files[file]["dbtype"] = "udeb";
435 elif file[-4:] == ".deb":
436 files[file]["dbtype"] = "deb";
438 reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file);
439 files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", ""))
440 files[file]["source"] = control.Find("Source", "");
441 if files[file]["source"] == "":
442 files[file]["source"] = files[file]["package"];
443 # Get the source version
444 source = files[file]["source"];
446 if string.find(source, "(") != -1:
447 m = utils.re_extract_src_version.match(source)
449 source_version = m.group(2)
450 if not source_version:
451 source_version = files[file]["version"];
452 files[file]["source package"] = source;
453 files[file]["source version"] = source_version;
455 # Checks for a source package...
457 m = utils.re_issource.match(file)
459 files[file]["package"] = m.group(1)
460 files[file]["version"] = m.group(2)
461 files[file]["type"] = m.group(3)
463 # Ensure the source package name matches the Source filed in the .changes
464 if changes["source"] != files[file]["package"]:
465 reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"])
467 # Ensure the source version matches the version in the .changes file
468 if files[file]["type"] == "orig.tar.gz":
469 changes_version = changes["chopversion2"]
471 changes_version = changes["chopversion"]
472 if changes_version != files[file]["version"]:
473 reject_message = reject_message + "Rejected: %s: should be %s according to changes file.\n" % (file, changes_version)
475 # Ensure the .changes lists source in the Architecture field
476 if not changes["architecture"].has_key("source"):
477 reject_message = reject_message + "Rejected: %s: changes file doesn't list `source' in Architecture field.\n" % (file)
479 # Check the signature of a .dsc file
480 if files[file]["type"] == "dsc":
481 check_signature(file)
483 files[file]["fullname"] = file
484 files[file]["architecture"] = "source";
486 # Not a binary or source package? Assume byhand...
488 files[file]["byhand"] = 1;
489 files[file]["type"] = "byhand";
491 files[file]["oldfiles"] = {}
492 for suite in changes["distribution"].keys():
494 if files[file].has_key("byhand"):
497 if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])):
498 reject_message = reject_message + "Rejected: unknown component `%s' for suite `%s'.\n" % (files[file]["component"], suite)
501 # See if the package is NEW
502 if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
503 files[file]["new"] = 1
505 if files[file]["type"] == "deb":
506 # Find any old binary packages
507 q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
508 % (files[file]["package"], suite, files[file]["architecture"]))
509 oldfiles = q.dictresult()
510 for oldfile in oldfiles:
511 files[file]["oldfiles"][suite] = oldfile
512 # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
513 if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
514 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
515 # Check for existing copies of the file
516 if not changes.has_key("stable install"):
517 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
518 if q.getresult() != []:
519 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file)
521 # Check for existent source
522 # FIXME: this is no longer per suite
523 if changes["architecture"].has_key("source"):
524 source_version = files[file]["source version"];
525 if source_version != changes["version"]:
526 reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]);
528 if not source_exists (files[file]["source package"], source_version):
529 reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file);
531 # Find any old .dsc files
532 elif files[file]["type"] == "dsc":
533 q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
534 % (files[file]["package"], suite))
535 oldfiles = q.dictresult()
536 if len(oldfiles) >= 1:
537 files[file]["oldfiles"][suite] = oldfiles[0]
539 # Validate the component
540 component = files[file]["component"];
541 component_id = db_access.get_component_id(component);
542 if component_id == -1:
543 reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component);
546 # Validate the priority
547 if string.find(files[file]["priority"],'/') != -1:
548 reject_message = reject_message + "Rejected: file '%s' has invalid priority '%s' [contains '/'].\n" % (file, files[file]["priority"]);
550 # Check the md5sum & size against existing files (if any)
551 location = Cnf["Dir::PoolDir"];
552 files[file]["location id"] = db_access.get_location_id (location, component, archive);
554 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]);
555 files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]);
557 reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file)
559 reject_message = reject_message + "Rejected: md5sum and/or size mismatch on existing copy of %s.\n" % (file)
560 files[file]["files id"] = files_id
562 # Check for packages that have moved from one component to another
563 if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
564 files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
567 if string.find(reject_message, "Rejected:") != -1:
572 ###############################################################################
575 global dsc, dsc_files, reject_message, reprocess, orig_tar_id, orig_tar_location, legacy_source_untouchable;
577 for file in files.keys():
578 if files[file]["type"] == "dsc":
580 dsc = utils.parse_changes(file, 1)
581 except utils.cant_open_exc:
582 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (file)
584 except utils.changes_parse_error_exc, line:
585 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (file, line)
587 except utils.invalid_dsc_format_exc, line:
588 reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (file, line)
591 dsc_files = utils.build_file_list(dsc, 1)
592 except utils.no_files_exc:
593 reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n";
595 except utils.changes_parse_error_exc, line:
596 reject_message = reject_message + "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line);
599 # Enforce mandatory fields
600 for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
601 if not dsc.has_key(i):
602 reject_message = reject_message + "Rejected: Missing field `%s' in dsc file.\n" % (i)
604 # The dpkg maintainer from hell strikes again! Bumping the
605 # version number of the .dsc breaks extraction by stable's
607 if dsc["format"] != "1.0":
608 reject_message = reject_message + """Rejected: [dpkg-sucks] source package was produced by a broken version
609 of dpkg-dev 1.9.1{3,4}; please rebuild with >= 1.9.15 version
613 # Ensure the version number in the .dsc matches the version number in the .changes
614 epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version"));
615 changes_version = files[file]["version"];
616 if epochless_dsc_version != files[file]["version"]:
617 reject_message = reject_message + "Rejected: version ('%s') in .dsc does not match version ('%s') in .changes\n" % (epochless_dsc_version, changes_version);
619 # Ensure source is newer than existing source in target suites
620 package = dsc.get("source");
621 new_version = dsc.get("version");
622 for suite in changes["distribution"].keys():
623 q = projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
625 ql = map(lambda x: x[0], q.getresult());
626 for old_version in ql:
627 if apt_pkg.VersionCompare(new_version, old_version) != 1:
628 reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, old_version, new_version)
630 # Try and find all files mentioned in the .dsc. This has
631 # to work harder to cope with the multiple possible
632 # locations of an .orig.tar.gz.
633 for dsc_file in dsc_files.keys():
634 if files.has_key(dsc_file):
635 actual_md5 = files[dsc_file]["md5sum"];
636 actual_size = int(files[dsc_file]["size"]);
637 found = "%s in incoming" % (dsc_file)
638 # Check the file does not already exist in the archive
639 if not changes.has_key("stable install"):
640 q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
642 # "It has not broken them. It has fixed a
643 # brokenness. Your crappy hack exploited a
644 # bug in the old dinstall.
646 # "(Come on! I thought it was always obvious
647 # that one just doesn't release different
648 # files with the same name and version.)"
649 # -- ajk@ on d-devel@l.d.o
651 if q.getresult() != []:
652 reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file)
653 elif dsc_file[-12:] == ".orig.tar.gz":
655 q = projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
659 # Unfortunately, we make get more than one match
660 # here if, for example, the package was in potato
661 # but had a -sa upload in woody. So we need to a)
662 # choose the right one and b) mark all wrong ones
663 # as excluded from the source poolification (to
664 # avoid file overwrites).
666 x = ql[0]; # default to something sane in case we don't match any or have only one
670 old_file = i[0] + i[1];
671 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
672 actual_size = os.stat(old_file)[stat.ST_SIZE];
673 if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
676 legacy_source_untouchable[i[3]] = "";
678 old_file = x[0] + x[1];
679 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
680 actual_size = os.stat(old_file)[stat.ST_SIZE];
683 dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
686 if suite_type == "legacy" or suite_type == "legacy-mixed":
687 orig_tar_location = "legacy";
689 orig_tar_location = x[4];
691 # Not there? Check in Incoming...
692 # [See comment above process_it() for explanation
693 # of why this is necessary...]
694 if os.path.exists(dsc_file):
695 files[dsc_file] = {};
696 files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
697 files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
698 files[dsc_file]["section"] = files[file]["section"];
699 files[dsc_file]["priority"] = files[file]["priority"];
700 files[dsc_file]["component"] = files[file]["component"];
701 files[dsc_file]["type"] = "orig.tar.gz";
705 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file);
708 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming.\n" % (file, dsc_file);
710 if actual_md5 != dsc_files[dsc_file]["md5sum"]:
711 reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file);
712 if actual_size != int(dsc_files[dsc_file]["size"]):
713 reject_message = reject_message + "Rejected: size for %s doesn't match %s.\n" % (found, file);
715 if string.find(reject_message, "Rejected:") != -1:
720 ###############################################################################
722 # Some cunning stunt broke dpkg-source in dpkg 1.8{,.1}; detect the
723 # resulting bad source packages and reject them.
725 # Even more amusingly the fix in 1.8.1.1 didn't actually fix the
726 # problem just changed the symptoms.
729 global dsc, dsc_files, reject_message, reprocess;
731 for filename in files.keys():
732 if files[filename]["type"] == "diff.gz":
733 file = gzip.GzipFile(filename, 'r');
734 for line in file.readlines():
735 if re_bad_diff.search(line):
736 reject_message = reject_message + "Rejected: [dpkg-sucks] source package was produced by a broken version of dpkg-dev 1.8.x; please rebuild with >= 1.8.3 version installed.\n";
739 if string.find(reject_message, "Rejected:") != -1:
744 ###############################################################################
746 def check_md5sums ():
747 global reject_message;
749 for file in files.keys():
751 file_handle = utils.open_file(file);
752 except utils.cant_open_exc:
755 if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
756 reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file);
758 def check_override ():
761 # Only check section & priority on sourceful non-stable installs
762 if not changes["architecture"].has_key("source") or changes.has_key("stable install"):
766 for file in files.keys():
767 if not files[file].has_key("new") and files[file]["type"] == "deb":
768 section = files[file]["section"];
769 override_section = files[file]["override section"];
770 if section != override_section and section != "-":
771 # Ignore this; it's a common mistake and not worth whining about
772 if string.lower(section) == "non-us/main" and string.lower(override_section) == "non-us":
774 summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
775 priority = files[file]["priority"];
776 override_priority = files[file]["override priority"];
777 if priority != override_priority and priority != "-":
778 summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
783 Subst["__SUMMARY__"] = summary;
784 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.override-disparity","r").read());
785 utils.send_mail (mail_message, "")
787 #####################################################################################################################
789 # Set up the per-package template substitution mappings
791 def update_subst (changes_filename):
794 # If katie crashed out in the right place, architecture may still be a string.
795 if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
796 changes["architecture"] = { "Unknown" : "" };
797 # and maintainer822 may not exist.
798 if not changes.has_key("maintainer822"):
799 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
801 Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
802 Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename);
803 Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
805 # For source uploads the Changed-By field wins; otherwise Maintainer wins.
806 if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
807 Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
808 Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
809 Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
811 Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
812 Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
813 Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
814 if Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
815 Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes.get("source"), Cnf["Dinstall::TrackingServer"])
817 Subst["__REJECT_MESSAGE__"] = reject_message;
818 Subst["__SOURCE__"] = changes.get("source", "Unknown");
819 Subst["__VERSION__"] = changes.get("version", "Unknown");
821 #####################################################################################################################
823 def action (changes_filename):
824 byhand = summary = new = "";
826 # changes["distribution"] may not exist in corner cases
827 # (e.g. unreadable changes files)
828 if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
829 changes["distribution"] = {};
833 for suite in changes["distribution"].keys():
834 if Cnf.has_key("Suite::%s::Confirm"):
835 confirm.append(suite)
838 file_keys = files.keys();
840 for file in file_keys:
841 if files[file].has_key("byhand"):
843 summary = summary + file + " byhand\n"
844 elif files[file].has_key("new"):
846 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
847 if files[file].has_key("othercomponents"):
848 summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
849 if files[file]["type"] == "deb":
850 summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
852 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
853 destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
854 summary = summary + file + "\n to " + destination + "\n"
856 short_summary = summary;
858 # This is for direport's benefit...
859 f = re_fdnic.sub("\n .\n", changes.get("changes",""));
861 if confirm or byhand or new:
862 summary = summary + "Changes: " + f;
864 summary = summary + announce (short_summary, 0)
866 (prompt, answer) = ("", "XXX")
867 if Options["No-Action"] or Options["Automatic"]:
870 if string.find(reject_message, "Rejected") != -1:
872 modified_time = time.time()-os.path.getmtime(changes_filename);
873 except: # i.e. ignore errors like 'file does not exist';
875 if modified_time < 86400:
876 print "SKIP (too new)\n" + reject_message,;
877 prompt = "[S]kip, Manual reject, Quit ?";
879 print "REJECT\n" + reject_message,;
880 prompt = "[R]eject, Manual reject, Skip, Quit ?";
881 if Options["Automatic"]:
884 print "NEW to %s\n%s%s" % (string.join(suites, ", "), reject_message, summary),;
885 prompt = "[S]kip, New ack, Manual reject, Quit ?";
886 if Options["Automatic"] and Options["Ack-New"]:
889 print "BYHAND\n" + reject_message + summary,;
890 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
892 print "CONFIRM to %s\n%s%s" % (string.join(confirm, ", "), reject_message, summary),
893 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
895 print "INSTALL\n" + reject_message + summary,;
896 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
897 if Options["Automatic"]:
900 while string.find(prompt, answer) == -1:
902 answer = utils.our_raw_input()
903 m = re_default_answer.match(prompt)
906 answer = string.upper(answer[:1])
909 reject (changes_filename, "");
911 manual_reject (changes_filename);
913 install (changes_filename, summary, short_summary);
915 acknowledge_new (changes_filename, summary);
919 ###############################################################################
921 def install (changes_filename, summary, short_summary):
922 global install_count, install_bytes, Subst;
924 # stable installs are a special case
925 if changes.has_key("stable install"):
926 stable_install (changes_filename, summary, short_summary);
931 Logger.log(["installing changes",changes_filename]);
933 archive = utils.where_am_i();
935 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
936 projectB.query("BEGIN WORK");
938 # Add the .dsc file to the DB
939 for file in files.keys():
940 if files[file]["type"] == "dsc":
941 package = dsc["source"]
942 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
943 maintainer = dsc["maintainer"]
944 maintainer = string.replace(maintainer, "'", "\\'")
945 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
946 filename = files[file]["pool name"] + file;
947 dsc_location_id = files[file]["location id"];
948 if not files[file]["files id"]:
949 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id)
950 projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)"
951 % (package, version, maintainer_id, files[file]["files id"]))
953 for suite in changes["distribution"].keys():
954 suite_id = db_access.get_suite_id(suite);
955 projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
957 # Add the source files to the DB (files and dsc_files)
958 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"]));
959 for dsc_file in dsc_files.keys():
960 filename = files[file]["pool name"] + dsc_file;
961 # If the .orig.tar.gz is already in the pool, it's
962 # files id is stored in dsc_files by check_dsc().
963 files_id = dsc_files[dsc_file].get("files id", None);
965 files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
966 # FIXME: needs to check for -1/-2 and or handle exception
968 files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
969 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id));
971 # Add the .deb files to the DB
972 for file in files.keys():
973 if files[file]["type"] == "deb":
974 package = files[file]["package"]
975 version = files[file]["version"]
976 maintainer = files[file]["maintainer"]
977 maintainer = string.replace(maintainer, "'", "\\'")
978 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
979 architecture = files[file]["architecture"]
980 architecture_id = db_access.get_architecture_id (architecture);
981 type = files[file]["dbtype"];
982 dsc_component = files[file]["component"]
983 source = files[file]["source package"]
984 source_version = files[file]["source version"];
985 filename = files[file]["pool name"] + file;
986 if not files[file]["files id"]:
987 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
988 source_id = db_access.get_source_id (source, source_version);
990 projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, %d, '%s')"
991 % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type));
993 projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, '%s')"
994 % (package, version, maintainer_id, architecture_id, files[file]["files id"], type));
995 for suite in changes["distribution"].keys():
996 suite_id = db_access.get_suite_id(suite);
997 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id));
999 # If the .orig.tar.gz is in a legacy directory we need to poolify
1000 # it, so that apt-get source (and anything else that goes by the
1001 # "Directory:" field in the Sources.gz file) works.
1002 if orig_tar_id != None and orig_tar_location == "legacy":
1003 q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id));
1004 qd = q.dictresult();
1006 # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details)
1007 if legacy_source_untouchable.has_key(qid["files_id"]):
1009 # First move the files to the new location
1010 legacy_filename = qid["path"]+qid["filename"];
1011 pool_location = utils.poolify (changes["source"], files[file]["component"]);
1012 pool_filename = pool_location + os.path.basename(qid["filename"]);
1013 destination = Cnf["Dir::PoolDir"] + pool_location
1014 utils.move(legacy_filename, destination);
1015 # Then Update the DB's files table
1016 q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"]));
1018 # If this is a sourceful diff only upload that is moving non-legacy
1019 # cross-component we need to copy the .orig.tar.gz into the new
1020 # component too for the same reasons as above.
1022 if changes["architecture"].has_key("source") and orig_tar_id != None and \
1023 orig_tar_location != "legacy" and orig_tar_location != dsc_location_id:
1024 q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id));
1025 ql = q.getresult()[0];
1026 old_filename = ql[0] + ql[1];
1028 file_md5sum = ql[3];
1029 new_filename = utils.poolify (changes["source"], dsc_component) + os.path.basename(old_filename);
1030 new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1031 if new_files_id == None:
1032 utils.copy(old_filename, Cnf["Dir::PoolDir"] + new_filename);
1033 new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
1034 projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id));
1036 # Install the files into the pool
1037 for file in files.keys():
1038 if files[file].has_key("byhand"):
1040 destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file
1041 destdir = os.path.dirname(destination)
1042 utils.move (file, destination)
1043 Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]);
1044 install_bytes = install_bytes + float(files[file]["size"])
1046 # Copy the .changes file across for suite which need it.
1047 for suite in changes["distribution"].keys():
1048 if Cnf.has_key("Suite::%s::CopyChanges" % (suite)):
1049 utils.copy (changes_filename, Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)]);
1051 projectB.query("COMMIT WORK");
1054 utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
1056 utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(changes_filename), sys.exc_type));
1058 install_count = install_count + 1;
1060 if not Options["No-Mail"]:
1061 Subst["__SUITE__"] = "";
1062 Subst["__SUMMARY__"] = summary;
1063 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1064 utils.send_mail (mail_message, "")
1065 announce (short_summary, 1)
1068 ################################################################################
1070 def stable_install (changes_filename, summary, short_summary):
1071 global install_count, install_bytes, Subst;
1073 print "Installing to stable."
1075 archive = utils.where_am_i();
1077 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
1078 projectB.query("BEGIN WORK");
1080 # Add the .dsc file to the DB
1081 for file in files.keys():
1082 if files[file]["type"] == "dsc":
1083 package = dsc["source"]
1084 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
1085 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
1088 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1089 source_id = ql[0][0];
1090 suite_id = db_access.get_suite_id('proposed-updates');
1091 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1092 suite_id = db_access.get_suite_id('stable');
1093 projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id));
1094 install_bytes = install_bytes + float(files[file]["size"])
1096 # Add the .deb files to the DB
1097 for file in files.keys():
1098 if files[file]["type"] == "deb":
1099 package = files[file]["package"]
1100 version = files[file]["version"]
1101 architecture = files[file]["architecture"]
1102 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
1105 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1106 binary_id = ql[0][0];
1107 suite_id = db_access.get_suite_id('proposed-updates');
1108 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1109 suite_id = db_access.get_suite_id('stable');
1110 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id));
1111 install_bytes = install_bytes + float(files[file]["size"])
1113 projectB.query("COMMIT WORK");
1116 utils.move (changes_filename, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(changes_filename));
1118 # Update the Stable ChangeLog file
1120 new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog";
1121 changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog";
1122 if os.path.exists(new_changelog_filename):
1123 os.unlink (new_changelog_filename);
1125 new_changelog = utils.open_file(new_changelog_filename, 'w');
1126 for file in files.keys():
1127 if files[file]["type"] == "deb":
1128 new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file));
1129 elif utils.re_issource.match(file) != None:
1130 new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file));
1132 new_changelog.write("%s\n" % (file));
1133 chop_changes = re_fdnic.sub("\n", changes["changes"]);
1134 new_changelog.write(chop_changes + '\n\n');
1135 if os.access(changelog_filename, os.R_OK) != 0:
1136 changelog = utils.open_file(changelog_filename);
1137 new_changelog.write(changelog.read());
1138 new_changelog.close();
1139 if os.access(changelog_filename, os.R_OK) != 0:
1140 os.unlink(changelog_filename);
1141 utils.move(new_changelog_filename, changelog_filename);
1143 install_count = install_count + 1;
1145 if not Options["No-Mail"]:
1146 Subst["__SUITE__"] = " into stable";
1147 Subst["__SUMMARY__"] = summary;
1148 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
1149 utils.send_mail (mail_message, "")
1150 announce (short_summary, 1)
1152 ################################################################################
1154 def reject (changes_filename, manual_reject_mail_filename):
1157 print "Rejecting.\n"
1159 base_changes_filename = os.path.basename(changes_filename);
1160 reason_filename = re_changes.sub("reason", base_changes_filename);
1161 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename);
1163 # Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored)
1165 utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
1167 utils.warn("couldn't reject changes file '%s'. [Got %s]" % (base_changes_filename, sys.exc_type));
1169 if not changes.has_key("stable install"):
1170 for file in files.keys():
1171 if os.path.exists(file):
1173 utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
1175 utils.warn("couldn't reject file '%s'. [Got %s]" % (file, sys.exc_type));
1178 suite_id = db_access.get_suite_id('proposed-updates');
1179 # Remove files from proposed-updates suite
1180 for file in files.keys():
1181 if files[file]["type"] == "dsc":
1182 package = dsc["source"];
1183 version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
1184 q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version));
1187 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
1188 source_id = ql[0][0];
1189 projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
1190 elif files[file]["type"] == "deb":
1191 package = files[file]["package"];
1192 version = files[file]["version"];
1193 architecture = files[file]["architecture"];
1194 q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture));
1197 utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
1198 binary_id = ql[0][0];
1199 projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
1201 # If this is not a manual rejection generate the .reason file and rejection mail message
1202 if manual_reject_mail_filename == "":
1203 if os.path.exists(reject_filename):
1204 os.unlink(reject_filename);
1205 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1206 os.write(fd, reject_message);
1208 Subst["__MANUAL_REJECT_MESSAGE__"] = "";
1209 Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
1210 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
1211 else: # Have a manual rejection file to use
1212 reject_mail_message = ""; # avoid <undef>'s
1214 # Send the rejection mail if appropriate
1215 if not Options["No-Mail"]:
1216 utils.send_mail (reject_mail_message, manual_reject_mail_filename);
1218 Logger.log(["rejected", changes_filename]);
1220 ##################################################################
1222 def manual_reject (changes_filename):
1225 # Build up the rejection email
1226 user_email_address = utils.whoami() + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
1227 manual_reject_message = Options.get("Manual-Reject", "")
1229 Subst["__MANUAL_REJECT_MESSAGE__"] = manual_reject_message;
1230 Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
1231 if changes.has_key("stable install"):
1232 template = "katie.stable-rejected";
1234 template = "katie.rejected";
1235 reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/"+template,"r").read());
1237 # Write the rejection email out as the <foo>.reason file
1238 reason_filename = re_changes.sub("reason", os.path.basename(changes_filename));
1239 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename)
1240 if os.path.exists(reject_filename):
1241 os.unlink(reject_filename);
1242 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
1243 os.write(fd, reject_mail_message);
1246 # If we weren't given one, spawn an editor so the user can add one in
1247 if manual_reject_message == "":
1248 result = os.system("vi +6 %s" % (reject_filename))
1250 utils.fubar("vi invocation failed for `%s'!" % (reject_filename), result);
1252 # Then process it as if it were an automatic rejection
1253 reject (changes_filename, reject_filename)
1255 #####################################################################################################################
1257 def acknowledge_new (changes_filename, summary):
1258 global new_ack_new, Subst;
1260 changes_filename = os.path.basename(changes_filename);
1262 Logger.log(["new",changes_filename]);
1264 new_ack_new[changes_filename] = 1;
1266 if new_ack_old.has_key(changes_filename):
1267 print "Ack already sent.";
1270 print "Sending new ack.";
1271 if not Options["No-Mail"]:
1272 Subst["__SUMMARY__"] = summary;
1273 new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.new","r").read());
1274 utils.send_mail(new_ack_message,"");
1276 #####################################################################################################################
1278 def announce (short_summary, action):
1281 # Only do announcements for source uploads with a recent dpkg-dev installed
1282 if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
1287 Subst["__SHORT_SUMMARY__"] = short_summary;
1289 for dist in changes["distribution"].keys():
1290 list = Cnf.Find("Suite::%s::Announce" % (dist))
1291 if list == "" or lists_done.has_key(list):
1293 lists_done[list] = 1
1294 summary = summary + "Announcing to %s\n" % (list)
1297 Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
1298 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.announce","r").read());
1299 utils.send_mail (mail_message, "")
1301 bugs = changes["closes"].keys()
1303 if not nmu.is_an_nmu(changes, dsc):
1304 summary = summary + "Closing bugs: "
1306 summary = summary + "%s " % (bug)
1308 Subst["__BUG_NUMBER__"] = bug;
1309 if changes["distribution"].has_key("stable"):
1310 Subst["__STABLE_WARNING__"] = """
1311 Note that this package is not part of the released stable Debian
1312 distribution. It may have dependencies on other unreleased software,
1313 or other instabilities. Please take care if you wish to install it.
1314 The update will eventually make its way into the next released Debian
1317 Subst["__STABLE_WARNING__"] = "";
1318 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-close","r").read());
1319 utils.send_mail (mail_message, "")
1321 Logger.log(["closing bugs"]+bugs);
1323 summary = summary + "Setting bugs to severity fixed: "
1324 control_message = ""
1326 summary = summary + "%s " % (bug)
1327 control_message = control_message + "tag %s + fixed\n" % (bug)
1328 if action and control_message != "":
1329 Subst["__CONTROL_MESSAGE__"] = control_message;
1330 mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-nmu-fixed","r").read());
1331 utils.send_mail (mail_message, "")
1333 Logger.log(["setting bugs to fixed"]+bugs);
1334 summary = summary + "\n"
1338 ###############################################################################
1340 # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in
1341 # Incoming. -1 will reference the .orig.tar.gz, but -2 will not.
1342 # dsccheckdistrib() can find the .orig.tar.gz but it will not have
1343 # processed it during it's checks of -2. If -1 has been deleted or
1344 # otherwise not checked by da-install, the .orig.tar.gz will not have
1345 # been checked at all. To get round this, we force the .orig.tar.gz
1346 # into the .changes structure and reprocess the .changes file.
1348 def process_it (changes_file):
1349 global reprocess, orig_tar_id, orig_tar_location, changes, dsc, dsc_files, files, reject_message;
1351 # Reset some globals
1358 orig_tar_location = "";
1359 legacy_source_untouchable = {};
1360 reject_message = "";
1362 # Absolutize the filename to avoid the requirement of being in the
1363 # same directory as the .changes file.
1364 changes_file = os.path.abspath(changes_file);
1366 # And since handling of installs to stable munges with the CWD;
1367 # save and restore it.
1371 check_signature (changes_file);
1372 check_changes (changes_file);
1381 traceback.print_exc(file=sys.stdout);
1384 update_subst(changes_file);
1385 action(changes_file);
1390 ###############################################################################
1393 global Cnf, Options, projectB, install_bytes, new_ack_old, Subst, nmu, Logger
1395 changes_files = init();
1400 if Options["Version"]:
1401 print "katie %s" % (katie_version);
1404 # -n/--dry-run invalidates some other options which would involve things happening
1405 if Options["No-Action"]:
1406 Options["Automatic"] = "";
1407 Options["Ack-New"] = "";
1409 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
1411 db_access.init(Cnf, projectB);
1413 # Check that we aren't going to clash with the daily cron job
1415 if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]:
1416 utils.fubar("Archive maintenance in progress. Try again later.");
1418 # Obtain lock if not in no-action mode and initialize the log
1420 if not Options["No-Action"]:
1421 lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT);
1422 fcntl.lockf(lock_fd, FCNTL.F_TLOCK);
1423 Logger = logging.Logger(Cnf, "katie");
1425 if Options["Ack-New"]:
1426 # Read in the list of already-acknowledged NEW packages
1427 if not os.path.exists(Cnf["Dinstall::NewAckList"]):
1428 utils.touch_file(Cnf["Dinstall::NewAckList"]);
1429 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"]);
1431 for line in new_ack_list.readlines():
1432 new_ack_old[line[:-1]] = 1;
1433 new_ack_list.close();
1435 # Initialize the substitution template mapping global
1437 Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
1438 Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
1439 bcc = "X-Katie: %s" % (katie_version);
1440 if Cnf.has_key("Dinstall::Bcc"):
1441 Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
1443 Subst["__BCC__"] = bcc;
1444 Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
1445 Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
1446 Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"];
1448 # Read in the group-maint override file
1451 # Sort the .changes files so that we process sourceful ones first
1452 changes_files.sort(utils.changes_compare);
1454 # Process the changes files
1455 for changes_file in changes_files:
1456 print "\n" + changes_file;
1457 process_it (changes_file);
1461 if install_count > 1:
1463 sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))));
1464 Logger.log(["total",install_count,install_bytes]);
1466 # Write out the list of already-acknowledged NEW packages
1467 if Options["Ack-New"]:
1468 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w')
1469 for i in new_ack_new.keys():
1470 new_ack_list.write(i+'\n')
1471 new_ack_list.close()
1473 if not Options["No-Action"]:
1476 if __name__ == '__main__':