3 # Installs Debian packaes
4 # Copyright (C) 2000 James Troup <james@nocrew.org>
5 # $Id: katie,v 1.2 2000-11-24 00:35:45 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 # Based (almost entirely) on dinstall by Guy Maor <maor@debian.org>
23 #########################################################################################
25 # Cartman: "I'm trying to make the best of a bad situation, I don't
26 # need to hear crap from a bunch of hippy freaks living in
27 # denial. Screw you guys, I'm going home."
29 # Kyle: "But Cartman, we're trying to..."
31 # Cartman: "uhh.. screw you guys... home."
33 #########################################################################################
35 import FCNTL, commands, fcntl, getopt, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time
36 import apt_inst, apt_pkg
37 import utils, db_access
39 ###############################################################################
41 re_isanum = re.compile (r'^\d+$');
42 re_isadeb = re.compile (r'.*\.u?deb$');
43 re_issource = re.compile (r'(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)');
44 re_dpackage = re.compile (r'^package:\s*(.*)', re.IGNORECASE);
45 re_darchitecture = re.compile (r'^architecture:\s*(.*)', re.IGNORECASE);
46 re_dversion = re.compile (r'^version:\s*(.*)', re.IGNORECASE);
47 re_dsection = re.compile (r'^section:\s*(.*)', re.IGNORECASE);
48 re_dpriority = re.compile (r'^priority:\s*(.*)', re.IGNORECASE);
49 re_changes = re.compile (r'changes$');
50 re_override_package = re.compile(r'(\S*)\s+.*');
51 re_default_answer = re.compile(r"\[(.*)\]");
52 re_fdnic = re.compile("\n\n");
54 ###############################################################################
57 reject_footer = """If you don't understand why your files were rejected, or if the
58 override file requires editing, reply to this email.
60 Your rejected files are in incoming/REJECT/. (Some may also be in
61 incoming/ if your .changes file was unparsable.) If only some of the
62 files need to repaired, you may move any good files back to incoming/.
63 Please remove any bad files from incoming/REJECT/."""
65 new_ack_footer = """Your package contains new components which requires manual editing of
66 the override file. It is ok otherwise, so please be patient. New
67 packages are usually added to the override file about once a week.
69 You may have gotten the distribution wrong. You'll get warnings above
70 if files already exist in other distributions."""
72 installed_footer = """If the override file requires editing, file a bug on ftp.debian.org.
74 Thank you for your contribution to Debian GNU."""
76 #########################################################################################
94 #########################################################################################
96 def usage (exit_code):
97 print """Usage: dinstall [OPTION]... [CHANGES]...
98 -a, --automatic automatic run
99 -d, --debug=VALUE debug
100 -k, --ack-new acknowledge new packages
101 -m, --manual-reject=MSG manual reject with `msg'
102 -n, --dry-run don't do anything
103 -p, --no-lock don't check lockfile !! for cron.daily only !!
104 -r, --no-version-check override version check
105 -u, --distribution=DIST override distribution to `dist'"""
108 def check_signature (filename):
109 global reject_message
111 (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --load-extension rsaref --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename))
113 reject_message = "Rejected: GPG signature check failed on `%s'.\n%s\n" % (filename, output)
117 #####################################################################################################################
119 def read_override_file (filename, suite, component):
122 file = utils.open_file(filename, 'r');
123 for line in file.readlines():
124 line = string.strip(utils.re_comments.sub('', line))
125 override_package = re_override_package.sub(r'\1', line)
126 if override_package != "":
127 overrides[suite][component][override_package] = 1
131 # See if a given package is in the override file. Caches and only loads override files on demand.
133 def in_override_p (package, component, suite):
136 # FIXME: nasty non-US speficic hack
137 if string.lower(component[:7]) == "non-us/":
138 component = component[7:];
139 if not overrides.has_key(suite) or not overrides[suite].has_key(component):
140 if not overrides.has_key(suite):
141 overrides[suite] = {}
142 if not overrides[suite].has_key(component):
143 overrides[suite][component] = {}
144 if Cnf.has_key("Suite::%s::SingleOverrideFile" % (suite)): # legacy mixed suite (i.e. experimental)
145 override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)];
146 read_override_file (override_filename, suite, component);
148 for src in ("", ".src"):
149 override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.' + component + src;
150 read_override_file (override_filename, suite, component);
152 return overrides[suite][component].get(package, None);
154 #####################################################################################################################
156 def check_changes(filename):
157 global reject_message, changes, files
159 # Parse the .changes field into a dictionary [FIXME - need to trap errors, pass on to reject_message etc.]
161 changes = utils.parse_changes(filename)
162 except utils.cant_open_exc:
163 reject_message = "Rejected: can't read changes file '%s'.\n" % (filename)
165 except utils.changes_parse_error_exc, line:
166 reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
167 changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
170 # Parse the Files field from the .changes into another dictionary [FIXME need to trap errors as above]
171 files = utils.build_file_list(changes, "")
173 # Check for mandatory fields
174 for i in ("source", "binary", "architecture", "version", "distribution","maintainer", "files"):
175 if not changes.has_key(i):
176 reject_message = "Rejected: Missing field `%s' in changes file." % (i)
177 return 0 # Avoid <undef> errors during later tests
179 # Fix the Maintainer: field to be RFC822 compatible
180 (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
182 # Override the Distribution: field if appropriate
183 if Cnf["Dinstall::Options::Override-Distribution"] != "":
184 reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"])
185 changes["distribution"] = Cnf["Dinstall::Options::Override-Distribution"]
187 # Split multi-value fields into a lower-level dictionary
188 for i in ("architecture", "distribution", "binary", "closes"):
189 o = changes.get(i, "")
193 for j in string.split(o):
196 # Ensure all the values in Closes: are numbers
197 if changes.has_key("closes"):
198 for i in changes["closes"].keys():
199 if re_isanum.match (i) == None:
200 reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i)
202 # Map frozen to unstable if frozen doesn't exist
203 if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"):
204 del changes["distribution"]["frozen"]
205 reject_message = reject_message + "Mapping frozen to unstable.\n"
207 # Ensure target distributions exist
208 for i in changes["distribution"].keys():
209 if not Cnf.has_key("Suite::%s" % (i)):
210 reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i)
212 # Map unreleased arches from stable to unstable
213 if changes["distribution"].has_key("stable"):
214 for i in changes["architecture"].keys():
215 if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)):
216 reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i)
217 del changes["distribution"]["stable"]
219 # Map arches not being released from frozen to unstable
220 if changes["distribution"].has_key("frozen"):
221 for i in changes["architecture"].keys():
222 if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)):
223 reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i)
224 del changes["distribution"]["frozen"]
226 # Handle uploads to stable
227 if changes["distribution"].has_key("stable"):
228 # If running from within proposed-updates kill non-stable distributions
229 if string.find(os.getcwd(), 'proposed-updates') != -1:
230 for i in ("frozen", "unstable"):
231 if changes["distributions"].has_key(i):
232 reject_message = reject_message + "Removing %s from distribution list.\n"
233 del changes["distribution"][i]
234 # Otherwise (normal case) map stable to updates
236 reject_message = reject_message + "Mapping stable to updates.\n";
237 del changes["distribution"]["stable"];
238 changes["distribution"]["proposed-updates"] = 1;
240 # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
241 changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"])
242 changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"])
244 if string.find(reject_message, "Rejected:") != -1:
250 global reject_message
252 archive = utils.where_am_i();
254 for file in files.keys():
255 # Check the file is readable
256 if os.access(file,os.R_OK) == 0:
257 reject_message = reject_message + "Rejected: Can't read `%s'.\n" % (file)
258 files[file]["type"] = "unreadable";
260 # If it's byhand skip remaining checks
261 if files[file]["section"] == "byhand":
262 files[file]["byhand"] = 1;
263 files[file]["type"] = "byhand";
264 # Checks for a binary package...
265 elif re_isadeb.match(file) != None:
266 # Extract package information using dpkg-deb
267 control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r")))
269 # Check for mandatory fields
270 if control.Find("Package") == None:
271 reject_message = reject_message + "Rejected: %s: No package field in control.\n" % (file)
272 if control.Find("Architecture") == None:
273 reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file)
274 if control.Find("Version") == None:
275 reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file)
277 # Ensure the package name matches the one give in the .changes
278 if not changes["binary"].has_key(control.Find("Package", "")):
279 reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", ""))
281 # Validate the architecture
282 if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))):
283 reject_message = reject_message + "Rejected: Unknown architecture '%s'.\n" % (control.Find("Architecture", ""))
285 # Check the architecture matches the one given in the .changes
286 if not changes["architecture"].has_key(control.Find("Architecture", "")):
287 reject_message = reject_message + "Rejected: %s: control file lists arch as `%s', which isn't in changes file.\n" % (file, control.Find("Architecture", ""))
288 # Check the section & priority match those given in the .changes (non-fatal)
289 if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"):
290 reject_message = reject_message + "Warning: %s control file lists section as `%s', but changes file has `%s'.\n" % (file, control.Find("Section", ""), files[file]["section"])
291 if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"):
292 reject_message = reject_message + "Warning: %s control file lists priority as `%s', but changes file has `%s'.\n" % (file, control.Find("Priority", ""), files[file]["priority"])
294 epochless_version = utils.re_no_epoch.sub('', control.Find("Version", ""))
296 files[file]["package"] = control.Find("Package");
297 files[file]["architecture"] = control.Find("Architecture");
298 files[file]["version"] = control.Find("Version");
299 files[file]["maintainer"] = control.Find("Maintainer", "");
300 if file[-5:] == ".udeb":
301 files[file]["dbtype"] = "udeb";
302 elif file[-4:] == ".deb":
303 files[file]["dbtype"] = "deb";
305 reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file);
306 files[file]["type"] = "deb";
307 files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", ""))
308 files[file]["source"] = control.Find("Source", "");
309 if files[file]["source"] == "":
310 files[file]["source"] = files[file]["package"];
311 # Checks for a source package...
313 m = re_issource.match(file)
315 files[file]["package"] = m.group(1)
316 files[file]["version"] = m.group(2)
317 files[file]["type"] = m.group(3)
319 # Ensure the source package name matches the Source filed in the .changes
320 if changes["source"] != files[file]["package"]:
321 reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"])
323 # Ensure the source version matches the version in the .changes file
324 if files[file]["type"] == "orig.tar.gz":
325 changes_version = changes["chopversion2"]
327 changes_version = changes["chopversion"]
328 if changes_version != files[file]["version"]:
329 reject_message = reject_message + "Rejected: %s: should be %s according to changes file.\n" % (file, changes_version)
331 # Ensure the .changes lists source in the Architecture field
332 if not changes["architecture"].has_key("source"):
333 reject_message = reject_message + "Rejected: %s: changes file doesn't list `source' in Architecture field.\n" % (file)
335 # Check the signature of a .dsc file
336 if files[file]["type"] == "dsc":
337 check_signature(file)
339 files[file]["fullname"] = file
341 # Not a binary or source package? Assume byhand...
343 files[file]["byhand"] = 1;
344 files[file]["type"] = "byhand";
346 files[file]["oldfiles"] = {}
347 for suite in changes["distribution"].keys():
349 if files[file].has_key("byhand"):
352 if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])):
353 reject_message = reject_message + "Rejected: unknown component `%s' for suite `%s'.\n" % (files[file]["component"], suite)
356 # See if the package is NEW
357 if not in_override_p(files[file]["package"], files[file]["component"], suite):
358 files[file]["new"] = 1
360 # Find any old binary packages
361 if files[file]["type"] == "deb":
362 q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND a.arch_string = '%s' AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
363 % (files[file]["package"], suite, files[file]["architecture"]))
364 oldfiles = q.dictresult()
365 for oldfile in oldfiles:
366 files[file]["oldfiles"][suite] = oldfile
367 # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
368 if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
369 if Cnf["Dinstall::Options::No-Version-Check"]:
370 reject_message = reject_message + "Overriden rejection"
372 reject_message = reject_message + "Rejected"
373 reject_message = reject_message + ": %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
374 # Find any old .dsc files
375 elif files[file]["type"] == "dsc":
376 q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
377 % (files[file]["package"], suite))
378 oldfiles = q.dictresult()
379 if len(oldfiles) >= 1:
380 files[file]["oldfiles"][suite] = oldfiles[0]
382 # Validate the component
383 component = files[file]["component"];
384 component_id = db_access.get_component_id(component);
385 if component_id == -1:
386 reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component);
389 # Check the md5sum & size against existing files (if any)
390 location = Cnf["Dir::PoolDir"];
391 files[file]["location id"] = db_access.get_location_id (location, component, archive);
392 files_id = db_access.get_files_id(component + '/' + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]);
394 reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file)
396 reject_message = reject_message + "Rejected: md5sum and/or size mismatch on existing copy of %s.\n" % (file)
397 files[file]["files id"] = files_id
399 # Check for packages that have moved from one component to another
400 if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
401 files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
404 if string.find(reject_message, "Rejected:") != -1:
409 ###############################################################################
412 global dsc, dsc_files, reject_message, reprocess, orig_tar_id;
414 for file in files.keys():
415 if files[file]["type"] == "dsc":
417 dsc = utils.parse_changes(file)
418 except utils.cant_open_exc:
419 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
421 except utils.changes_parse_error_exc, line:
422 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
425 dsc_files = utils.build_file_list(dsc, 1)
426 except utils.no_files_exc:
427 reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n";
430 # Try and find all files mentioned in the .dsc. This has
431 # to work harder to cope with the multiple possible
432 # locations of an .orig.tar.gz.
433 for dsc_file in dsc_files.keys():
434 if files.has_key(dsc_file):
435 actual_md5 = files[dsc_file]["md5sum"]
436 found = "%s in incoming" % (dsc_file)
437 elif dsc_file[-12:] == ".orig.tar.gz":
439 # See comment above process_it() for explanation...
440 if os.access(dsc_file, os.R_OK) != 0:
441 files[dsc_file] = {};
442 files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
443 files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
444 files[dsc_file]["section"] = files[file]["section"];
445 files[dsc_file]["priority"] = files[file]["priority"];
446 files[dsc_file]["component"] = files[file]["component"];
450 q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (dsc_file));
453 old_file = ql[0][0] + ql[0][1];
454 actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r"));
456 suite_type = ql[0][2];
458 if suite_type == "legacy" or suite_type == "legacy-mixed":
459 orig_tar_id = ql[0][3];
461 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file);
464 reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming." % (file, dsc_file);
466 if actual_md5 != dsc_files[dsc_file]["md5sum"]:
467 reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file)
469 if string.find(reject_message, "Rejected:") != -1:
474 ###############################################################################
476 def check_md5sums ():
477 global reject_message;
479 for file in files.keys():
481 file_handle = utils.open_file(file,"r");
482 except utils.cant_open_exc:
485 if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
486 reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file);
488 #####################################################################################################################
490 def action (changes_filename):
491 byhand = confirm = suites = summary = new = "";
493 # changes["distribution"] may not exist in corner cases
494 # (e.g. unreadable changes files)
495 if not changes.has_key("distribution"):
496 changes["distribution"] = {};
498 for suite in changes["distribution"].keys():
499 if Cnf.has_key("Suite::%s::Confirm"):
500 confirm = confirm + suite + ", "
501 suites = suites + suite + ", "
502 confirm = confirm[:-2]
505 for file in files.keys():
506 if files[file].has_key("byhand"):
508 summary = summary + file + " byhand\n"
509 elif files[file].has_key("new"):
511 summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"])
512 if files[file].has_key("othercomponents"):
513 summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
514 if files[file]["type"] == "deb":
515 summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r")))["Description"] + '\n';
517 files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
518 destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
519 summary = summary + file + "\n to " + destination + "\n"
521 short_summary = summary;
523 # This is for direport's benefit...
524 f = re_fdnic.sub("\n .\n", changes.get("changes",""));
526 if confirm or byhand or new:
527 summary = summary + "Changes: " + f;
529 summary = summary + announce (short_summary, 0)
531 (prompt, answer) = ("", "XXX")
532 if Cnf["Dinstall::Options::No-Action"] or Cnf["Dinstall::Options::Automatic"]:
535 if string.find(reject_message, "Rejected") != -1:
536 if time.time()-os.path.getmtime(changes_filename) < 86400:
537 print "SKIP (too new)\n" + reject_message,;
538 prompt = "[S]kip, Manual reject, Quit ?";
540 print "REJECT\n" + reject_message,;
541 prompt = "[R]eject, Manual reject, Skip, Quit ?";
542 if Cnf["Dinstall::Options::Automatic"]:
545 print "NEW to %s\n%s%s" % (suites, reject_message, summary),;
546 prompt = "[S]kip, New ack, Manual reject, Quit ?";
547 if Cnf["Dinstall::Options::Automatic"] and Cnf["Dinstall::Options::Ack-New"]:
550 print "BYHAND\n" + reject_message + summary,;
551 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
553 print "CONFIRM to %s\n%s%s" % (confirm, reject_message, summary),
554 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
556 print "INSTALL\n" + reject_message + summary,;
557 prompt = "[I]nstall, Manual reject, Skip, Quit ?";
558 if Cnf["Dinstall::Options::Automatic"]:
561 while string.find(prompt, answer) == -1:
563 answer = utils.our_raw_input()
564 m = re_default_answer.match(prompt)
567 answer = string.upper(answer[:1])
570 reject (changes_filename, "");
572 manual_reject (changes_filename);
574 install (changes_filename, summary, short_summary);
576 acknowledge_new (changes_filename, summary);
580 #####################################################################################################################
582 def install (changes_filename, summary, short_summary):
583 global install_count, install_bytes
587 archive = utils.where_am_i();
589 # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
590 projectB.query("BEGIN WORK");
592 # Add the .dsc file to the DB
593 for file in files.keys():
594 if files[file]["type"] == "dsc":
595 package = dsc["source"]
596 version = dsc["version"] # NB: not files[file]["version"], that has no epoch
597 maintainer = dsc["maintainer"]
598 maintainer = string.replace(maintainer, "'", "\\'")
599 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
600 filename = files[file]["pool name"] + file;
601 dsc_location_id = files[file]["location id"];
602 if not files[file]["files id"]:
603 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id)
604 dsc_file_id = files[file]["files id"]
605 projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)"
606 % (package, version, maintainer_id, files[file]["files id"]))
608 for suite in changes["distribution"].keys():
609 suite_id = db_access.get_suite_id(suite);
610 projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
613 # Add the .diff.gz and {.orig,}.tar.gz files to the DB (files and dsc_files)
614 for file in files.keys():
615 if files[file]["type"] == "diff.gz" or files[file]["type"] == "orig.tar.gz" or files[file]["type"] == "tar.gz":
616 if not files[file]["files id"]:
617 filename = files[file]["pool name"] + file;
618 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
619 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"]));
621 # Add the .deb files to the DB
622 for file in files.keys():
623 if files[file]["type"] == "deb":
624 package = files[file]["package"]
625 version = files[file]["version"]
626 maintainer = files[file]["maintainer"]
627 maintainer = string.replace(maintainer, "'", "\\'")
628 maintainer_id = db_access.get_or_set_maintainer_id(maintainer);
629 architecture = files[file]["architecture"]
630 architecture_id = db_access.get_architecture_id (architecture);
631 type = files[file]["dbtype"];
632 component = files[file]["component"]
633 source = files[file]["source"]
635 if string.find(source, "(") != -1:
636 m = utils.re_extract_src_version.match(source)
638 source_version = m.group(2)
639 if not source_version:
640 source_version = version
641 filename = files[file]["pool name"] + file;
642 if not files[file]["files id"]:
643 files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
644 source_id = db_access.get_source_id (source, source_version);
646 projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, %d, '%s')"
647 % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type));
649 projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, '%s')"
650 % (package, version, maintainer_id, architecture_id, files[file]["files id"], type));
651 for suite in changes["distribution"].keys():
652 suite_id = db_access.get_suite_id(suite);
653 projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id));
655 # Install the files into the pool
656 for file in files.keys():
657 if files[file].has_key("byhand"):
659 destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file
660 destdir = os.path.dirname(destination)
661 utils.move (file, destination)
662 install_bytes = install_bytes + float(files[file]["size"])
664 # Copy the .changes file across for suite which need it.
665 for suite in changes["distribution"].keys():
666 if Cnf.has_key("Suties::%s::CopyChanges" % (suite)):
667 destination = Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)] + os.path.basename(changes_filename)
668 copy_file (changes_filename, destination)
670 # If the .orig.tar.gz is in a legacy directory we need to poolify
671 # it, so that apt-get source (and anything else that goes by the
672 # "Directory:" field in the Sources.gz file) works.
673 if orig_tar_id != None:
674 q = projectB.query("SELECT l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location" % (orig_tar_id));
677 # First move the files to the new location
678 legacy_filename = qid["path"]+qid["filename"];
679 pool_location = utils.poolify (files[file]["package"], files[file]["component"]);
680 pool_filename = pool_location + os.path.basename(qid["filename"]);
681 destination = Cnf["Dir::PoolDir"] + pool_location
682 utils.move(legacy_filename, destination);
683 # Update the DB: files table
684 new_files_id = db_access.set_files_id(pool_filename, qid["size"], qid["md5sum"], dsc_location_id);
685 # Update the DB: dsc_files table
686 projectB.query("INSERT INTO dsc_files (source, file) VALUES (%s, %s)" % (qid["source"], new_files_id));
687 # Update the DB: source table
688 if legacy_filename[-4:] == ".dsc":
689 projectB.query("UPDATE source SET file = %s WHERE id = %d" % (new_files_id, qid["source"]));
692 # Remove old data from the DB: dsc_files table
693 projectB.query("DELETE FROM dsc_files WHERE id = %s" % (qid["dsc_files_id"]));
694 # Remove old data from the DB: files table
695 projectB.query("DELETE FROM files WHERE id = %s" % (qid["files_id"]));
697 utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
699 projectB.query("COMMIT WORK");
701 install_count = install_count + 1;
703 if not Cnf["Dinstall::Options::No-Mail"]:
704 mail_message = """Return-Path: %s
707 Bcc: troup@auric.debian.org
708 Subject: %s INSTALLED
714 %s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, reject_message, summary, installed_footer)
715 utils.send_mail (mail_message, "")
716 announce (short_summary, 1)
718 #####################################################################################################################
720 def reject (changes_filename, manual_reject_mail_filename):
723 base_changes_filename = os.path.basename(changes_filename);
724 reason_filename = re_changes.sub("reason", base_changes_filename);
725 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename);
727 # Move the .changes files and it's contents into REJECT/
728 utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
729 for file in files.keys():
730 if os.access(file,os.R_OK) == 0:
731 utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
733 # If this is not a manual rejection generate the .reason file and rejection mail message
734 if manual_reject_mail_filename == "":
735 if os.path.exists(reject_filename):
736 os.unlink(reject_filename);
737 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
738 os.write(fd, reject_message);
740 reject_mail_message = """From: %s
742 Bcc: troup@auric.debian.org
747 %s""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, reject_message, reject_footer);
748 else: # Have a manual rejection file to use
749 reject_mail_message = ""; # avoid <undef>'s
751 # Send the rejection mail if appropriate
752 if not Cnf["Dinstall::Options::No-Mail"]:
753 utils.send_mail (reject_mail_message, manual_reject_mail_filename);
755 ##################################################################
757 def manual_reject (changes_filename):
758 # Build up the rejection email
759 user_email_address = string.replace(string.split(pwd.getpwuid(os.getuid())[4],',')[0], '.', '')
760 user_email_address = user_email_address + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
761 manual_reject_message = Cnf.get("Dinstall::Options::Manual-Reject", "")
763 reject_mail_message = """From: %s
766 Bcc: troup@auric.debian.org
772 %s""" % (user_email_address, Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, manual_reject_message, reject_message, reject_footer)
774 # Write the rejection email out as the <foo>.reason file
775 reason_filename = re_changes.sub("reason", os.path.basename(changes_filename));
776 reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename)
777 if os.path.exists(reject_filename):
778 os.unlink(reject_filename);
779 fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
780 os.write(fd, reject_mail_message);
783 # If we weren't given one, spawn an editor so the user can add one in
784 if manual_reject_message == "":
785 result = os.system("vi +6 %s" % (reject_file))
787 sys.stderr.write ("vi invocation failed for `%s'!" % (reject_file))
790 # Then process it as if it were an automatic rejection
791 reject (changes_filename, reject_filename)
793 #####################################################################################################################
795 def acknowledge_new (changes_filename, summary):
798 new_ack_new[changes_filename] = 1;
800 if new_ack_old.has_key(changes_filename):
801 print "Ack already sent.";
804 print "Sending new ack.";
805 if not Cnf["Dinstall::Options::No-Mail"]:
806 new_ack_message = """Return-Path: %s
809 Bcc: troup@auric.debian.org
813 %s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, summary, new_ack_footer);
814 utils.send_mail(new_ack_message,"");
816 #####################################################################################################################
818 def announce (short_summary, action):
819 # Only do announcements for source uploads with a recent dpkg-dev installed
820 if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
826 for dist in changes["distribution"].keys():
827 list = Cnf["Suite::%s::Announce" % (dist)]
828 if lists_done.has_key(list):
831 summary = summary + "Announcing to %s\n" % (list)
834 mail_message = """Return-Path: %s
837 Bcc: troup@auric.debian.org
838 Subject: Installed %s %s (%s)
844 """ % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], list, changes["source"], changes["version"], string.join(changes["architecture"].keys(), ' ' ),
845 changes["filecontents"], short_summary)
846 utils.send_mail (mail_message, "")
848 (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
849 bugs = changes["closes"].keys()
851 if dsc_name == changes["maintainername"]:
852 summary = summary + "Closing bugs: "
854 summary = summary + "%s " % (bug)
856 mail_message = """Return-Path: %s
858 To: %s-close@bugs.debian.org
859 Bcc: troup@auric.debian.org
860 Subject: Bug#%s: fixed in %s %s
862 We believe that the bug you reported is fixed in the latest version of
863 %s, which has been installed in the Debian FTP archive:
865 %s""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], bug, bug, changes["source"], changes["version"], changes["source"], short_summary)
867 if changes["distribution"].has_key("stable"):
868 mail_message = mail_message + """Note that this package is not part of the released stable Debian
869 distribution. It may have dependencies on other unreleased software,
870 or other instabilities. Please take care if you wish to install it.
871 The update will eventually make its way into the next released Debian
874 mail_message = mail_message + """A summary of the changes between this version and the previous one is
877 Thank you for reporting the bug, which will now be closed. If you
878 have further comments please address them to %s@bugs.debian.org,
879 and the maintainer will reopen the bug report if appropriate.
881 Debian distribution maintenance software
883 %s (supplier of updated %s package)
885 (This message was generated automatically at their request; if you
886 believe that there is a problem with it please contact the archive
887 administrators by mailing ftpmaster@debian.org)
890 %s""" % (bug, changes["maintainer"], changes["source"], changes["filecontents"])
892 utils.send_mail (mail_message, "")
894 summary = summary + "Setting bugs to severity fixed: "
897 summary = summary + "%s " % (bug)
898 control_message = control_message + "severity %s fixed\n" % (bug)
899 if action and control_message != "":
900 mail_message = """Return-Path: %s
902 To: control@bugs.debian.org
903 Bcc: troup@auric.debian.org, %s
904 Subject: Fixed in NMU of %s %s
909 This message was generated automatically in response to a
910 non-maintainer upload. The .changes file follows.
913 """ % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes["maintainer822"], changes["source"], changes["version"], control_message, changes["filecontents"])
914 utils.send_mail (mail_message, "")
915 summary = summary + "\n"
919 ###############################################################################
921 # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in
922 # Incoming. -1 will reference the .orig.tar.gz, but -2 will not.
923 # dsccheckdistrib() can find the .orig.tar.gz but it will not have
924 # processed it during it's checks of -2. If -1 has been deleted or
925 # otherwise not checked by da-install, the .orig.tar.gz will not have
926 # been checked at all. To get round this, we force the .orig.tar.gz
927 # into the .changes structure and reprocess the .changes file.
929 def process_it (changes_file):
930 global reprocess, orig_tar_id;
935 check_signature (changes_file);
936 check_changes (changes_file);
943 action(changes_file);
945 ###############################################################################
948 global Cnf, projectB, reject_message, install_bytes, new_ack_old
952 Cnf = apt_pkg.newConfiguration();
953 apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file());
955 Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
956 ('d',"debug","Dinstall::Options::Debug", "IntVal"),
957 ('h',"help","Dinstall::Options::Help"),
958 ('k',"ack-new","Dinstall::Options::Ack-New"),
959 ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
960 ('n',"no-action","Dinstall::Options::No-Action"),
961 ('p',"no-lock", "Dinstall::Options::No-Lock"),
962 ('r',"no-version-check", "Dinstall::Options::No-Version-Check"),
963 ('s',"no-mail", "Dinstall::Options::No-Mail"),
964 ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
965 ('v',"version","Dinstall::Options::Version")];
967 changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
969 if Cnf["Dinstall::Options::Help"]:
972 if Cnf["Dinstall::Options::Version"]:
973 print "katie version 0.0000000000";
976 postgresql_user = None; # Default == Connect as user running program.
978 # -n/--dry-run invalidates some other options which would involve things happening
979 if Cnf["Dinstall::Options::No-Action"]:
980 Cnf["Dinstall::Options::Automatic"] = ""
981 Cnf["Dinstall::Options::Ack-New"] = ""
982 postgresql_user = Cnf["DB::ROUser"];
984 projectB = pg.connect('projectb', Cnf["DB::Host"], int(Cnf["DB::Port"]), None, None, postgresql_user);
986 db_access.init(Cnf, projectB);
988 # Check that we aren't going to clash with the daily cron job
990 if os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Cnf["Dinstall::Options::No-Lock"]:
991 sys.stderr.write("Archive maintenance in progress. Try again later.\n");
994 # Obtain lock if not in no-action mode
996 if not Cnf["Dinstall::Options::No-Action"]:
997 lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR);
998 fcntl.lockf(lock_fd, FCNTL.F_TLOCK);
1000 # Read in the list of already-acknowledged NEW packages
1001 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'r');
1003 for line in new_ack_list.readlines():
1004 new_ack_old[line[:-1]] = 1;
1005 new_ack_list.close();
1007 # Process the changes files
1008 for changes_file in changes_files:
1010 print "\n" + changes_file;
1011 process_it (changes_file);
1014 if install_bytes > 10000:
1015 install_bytes = install_bytes / 1000;
1016 install_mag = " Kb";
1017 if install_bytes > 10000:
1018 install_bytes = install_bytes / 1000;
1019 install_mag = " Mb";
1022 if install_count > 1:
1024 sys.stderr.write("Installed %d package %s, %d%s.\n" % (install_count, sets, int(install_bytes), install_mag))
1026 # Write out the list of already-acknowledged NEW packages
1027 if Cnf["Dinstall::Options::Ack-New"]:
1028 new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w')
1029 for i in new_ack_new.keys():
1030 new_ack_list.write(i+'\n')
1031 new_ack_list.close()
1034 if __name__ == '__main__':