4 # Copyright (C) 2000, 2001, 2002, 2003 James Troup <james@nocrew.org>
5 # $Id: utils.py,v 1.56 2003-02-21 19:20:00 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 import commands, os, pwd, re, select, socket, shutil, string, sys, tempfile, traceback;
27 ################################################################################
29 re_comments = re.compile(r"\#.*")
30 re_no_epoch = re.compile(r"^\d*\:")
31 re_no_revision = re.compile(r"\-[^-]*$")
32 re_arch_from_filename = re.compile(r"/binary-[^/]+/")
33 re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
34 re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$");
35 re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$");
37 re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)");
38 re_multi_line_field = re.compile(r"^\s(.*)");
39 re_taint_free = re.compile(r"^[-+~\.\w]+$");
41 re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\> \t]+)\>");
43 changes_parse_error_exc = "Can't parse line in .changes file";
44 invalid_dsc_format_exc = "Invalid .dsc file";
45 nk_format_exc = "Unknown Format: in .changes file";
46 no_files_exc = "No Files: field in .dsc file.";
47 cant_open_exc = "Can't read file.";
48 unknown_hostname_exc = "Unknown hostname";
49 cant_overwrite_exc = "Permission denied; can't overwrite existent file."
50 file_exists_exc = "Destination file exists";
51 send_mail_invalid_args_exc = "Both arguments are non-null.";
52 sendmail_failed_exc = "Sendmail invocation failed";
53 tried_too_hard_exc = "Tried too hard to find a free filename.";
55 default_config = "/etc/katie/katie.conf";
56 default_apt_config = "/etc/katie/apt.conf";
58 ######################################################################################
60 def open_file(filename, mode='r'):
62 f = open(filename, mode);
64 raise cant_open_exc, filename
67 ######################################################################################
69 def our_raw_input(prompt=""):
71 sys.stdout.write(prompt);
77 sys.stderr.write('\nUser interrupt (^D).\n');
80 ######################################################################################
84 if c not in string.digits:
88 ######################################################################################
90 def extract_component_from_section(section):
93 if section.find('/') != -1:
94 component = section.split('/')[0];
95 if component.lower() == "non-us" and section.count('/') > 0:
96 s = component + '/' + section.split('/')[1];
97 if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs
100 if section.lower() == "non-us":
101 component = "non-US/main";
103 # non-US prefix is case insensitive
104 if component.lower()[:6] == "non-us":
105 component = "non-US"+component[6:];
107 # Expand default component
109 if Cnf.has_key("Component::%s" % section):
113 elif component == "non-US":
114 component = "non-US/main";
116 return (section, component);
118 ######################################################################################
120 # dsc_whitespace_rules turns on strict format checking to avoid
121 # allowing in source packages which are unextracable by the
122 # inappropriately fragile dpkg-source.
127 # o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
128 # followed by any PGP header data and must end with a blank line.
130 # o The data section must end with a blank line and must be followed by
131 # "-----BEGIN PGP SIGNATURE-----".
133 def parse_changes(filename, dsc_whitespace_rules=0):
134 changes_in = open_file(filename);
137 lines = changes_in.readlines();
140 raise changes_parse_error_exc, "[Empty changes file]";
142 # Reindex by line number so we can easily verify the format of
148 indexed_lines[index] = line[:-1];
150 inside_signature = 0;
152 indices = indexed_lines.keys()
155 while index < max(indices):
157 line = indexed_lines[index];
159 if dsc_whitespace_rules:
161 if index > max(indices):
162 raise invalid_dsc_format_exc, index;
163 line = indexed_lines[index];
164 if not line.startswith("-----BEGIN PGP SIGNATURE"):
165 raise invalid_dsc_format_exc, index;
166 inside_signature = 0;
168 if line.startswith("-----BEGIN PGP SIGNATURE"):
170 if line.startswith("-----BEGIN PGP SIGNED MESSAGE"):
171 if dsc_whitespace_rules:
172 inside_signature = 1;
173 while index < max(indices) and line != "":
175 line = indexed_lines[index];
177 slf = re_single_line_field.match(line);
179 field = slf.groups()[0].lower();
180 changes[field] = slf.groups()[1];
184 changes[field] += '\n';
186 mlf = re_multi_line_field.match(line);
189 raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line);
190 if first == 1 and changes[field] != "":
191 changes[field] += '\n';
193 changes[field] += mlf.groups()[0] + '\n';
197 if dsc_whitespace_rules and inside_signature:
198 raise invalid_dsc_format_exc, index;
201 changes["filecontents"] = "".join(lines);
204 raise changes_parse_error_exc, error;
208 ######################################################################################
210 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
212 def build_file_list(changes, is_a_dsc=0):
214 format = changes.get("format", "")
216 format = float(format)
217 if not is_a_dsc and (format < 1.5 or format > 2.0):
218 raise nk_format_exc, format;
220 # No really, this has happened. Think 0 length .dsc file.
221 if not changes.has_key("files"):
224 for i in changes["files"].split("\n"):
228 section = priority = "";
231 (md5, size, name) = s
233 (md5, size, section, priority, name) = s
235 raise changes_parse_error_exc, i
237 if section == "": section = "-"
238 if priority == "": priority = "-"
240 (section, component) = extract_component_from_section(section);
242 files[name] = { "md5sum" : md5,
245 "priority": priority,
246 "component": component }
250 ######################################################################################
252 # Fix the `Maintainer:' field to be an RFC822 compatible address.
253 # cf. Debian Policy Manual (D.2.4)
255 # 06:28|<Culus> 'The standard sucks, but my tool is supposed to
256 # interoperate with it. I know - I'll fix the suckage
257 # and make things incompatible!'
259 def fix_maintainer (maintainer):
260 m = re_parse_maintainer.match(maintainer);
264 if m != None and len(m.groups()) == 2:
267 if name.find(',') != -1 or name.find('.') != -1:
268 rfc822 = "%s (%s)" % (email, name);
269 return (rfc822, name, email)
271 ######################################################################################
273 # sendmail wrapper, takes _either_ a message string or a file as arguments
274 def send_mail (message, filename):
275 # Sanity check arguments
276 if message != "" and filename != "":
277 raise send_mail_invalid_args_exc;
279 # If we've been passed a string dump it into a temporary file
281 filename = tempfile.mktemp();
282 fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
283 os.write (fd, message);
287 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename));
289 raise sendmail_failed_exc, output;
291 # Clean up any temporary files
293 os.unlink (filename);
295 ######################################################################################
297 def poolify (source, component):
300 # FIXME: this is nasty
301 component = component.lower().replace('non-us/', 'non-US/');
302 if source[:3] == "lib":
303 return component + source[:4] + '/' + source + '/'
305 return component + source[:1] + '/' + source + '/'
307 ######################################################################################
309 def move (src, dest, overwrite = 0, perms = 0664):
310 if os.path.exists(dest) and os.path.isdir(dest):
313 dest_dir = os.path.dirname(dest);
314 if not os.path.exists(dest_dir):
315 umask = os.umask(00000);
316 os.makedirs(dest_dir, 02775);
318 #print "Moving %s to %s..." % (src, dest);
319 if os.path.exists(dest) and os.path.isdir(dest):
320 dest += '/' + os.path.basename(src);
321 # Don't overwrite unless forced to
322 if os.path.exists(dest):
324 raise file_exists_exc;
326 if not os.access(dest, os.W_OK):
327 raise cant_overwrite_exc
328 shutil.copy2(src, dest);
329 os.chmod(dest, perms);
332 def copy (src, dest, overwrite = 0, perms = 0664):
333 if os.path.exists(dest) and os.path.isdir(dest):
336 dest_dir = os.path.dirname(dest);
337 if not os.path.exists(dest_dir):
338 umask = os.umask(00000);
339 os.makedirs(dest_dir, 02775);
341 #print "Copying %s to %s..." % (src, dest);
342 if os.path.exists(dest) and os.path.isdir(dest):
343 dest += '/' + os.path.basename(src);
344 # Don't overwrite unless forced to
345 if os.path.exists(dest):
347 raise file_exists_exc
349 if not os.access(dest, os.W_OK):
350 raise cant_overwrite_exc
351 shutil.copy2(src, dest);
352 os.chmod(dest, perms);
354 ######################################################################################
357 res = socket.gethostbyaddr(socket.gethostname());
358 database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname");
359 if database_hostname:
360 return database_hostname;
364 def which_conf_file ():
365 res = socket.gethostbyaddr(socket.gethostname());
366 if Cnf.get("Config::" + res[0] + "::KatieConfig"):
367 return Cnf["Config::" + res[0] + "::KatieConfig"]
369 return default_config;
371 def which_apt_conf_file ():
372 res = socket.gethostbyaddr(socket.gethostname());
373 if Cnf.get("Config::" + res[0] + "::AptConfig"):
374 return Cnf["Config::" + res[0] + "::AptConfig"]
376 return default_apt_config;
378 ######################################################################################
380 # Escape characters which have meaning to SQL's regex comparison operator ('~')
381 # (woefully incomplete)
384 s = s.replace('+', '\\\\+');
385 s = s.replace('.', '\\\\.');
388 ######################################################################################
390 # Perform a substition of template
391 def TemplateSubst(map, filename):
392 file = open_file(filename);
393 template = file.read();
395 template = template.replace(x,map[x]);
399 ######################################################################################
401 def fubar(msg, exit_code=1):
402 sys.stderr.write("E: %s\n" % (msg));
406 sys.stderr.write("W: %s\n" % (msg));
408 ######################################################################################
410 # Returns the user name with a laughable attempt at rfc822 conformancy
411 # (read: removing stray periods).
413 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '');
415 ######################################################################################
425 return ("%d%s" % (c, t))
427 ################################################################################
429 def cc_fix_changes (changes):
430 o = changes.get("architecture", "")
432 del changes["architecture"]
433 changes["architecture"] = {}
435 changes["architecture"][j] = 1
437 # Sort by source name, source version, 'have source', and then by filename
438 def changes_compare (a, b):
440 a_changes = parse_changes(a);
445 b_changes = parse_changes(b);
449 cc_fix_changes (a_changes);
450 cc_fix_changes (b_changes);
452 # Sort by source name
453 a_source = a_changes.get("source");
454 b_source = b_changes.get("source");
455 q = cmp (a_source, b_source);
459 # Sort by source version
460 a_version = a_changes.get("version");
461 b_version = b_changes.get("version");
462 q = apt_pkg.VersionCompare(a_version, b_version);
466 # Sort by 'have source'
467 a_has_source = a_changes["architecture"].get("source");
468 b_has_source = b_changes["architecture"].get("source");
469 if a_has_source and not b_has_source:
471 elif b_has_source and not a_has_source:
474 # Fall back to sort by filename
477 ################################################################################
479 def find_next_free (dest, too_many=100):
482 while os.path.exists(dest) and extra < too_many:
483 dest = orig_dest + '.' + repr(extra);
485 if extra >= too_many:
486 raise tried_too_hard_exc;
489 ################################################################################
491 def result_join (original, sep = '\t'):
493 for i in xrange(len(original)):
494 if original[i] == None:
497 list.append(original[i]);
498 return sep.join(list);
500 ################################################################################
502 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
504 for line in str.split('\n'):
506 if line or include_blank_lines:
507 out += "%s%s\n" % (prefix, line);
508 # Strip trailing new line
513 ################################################################################
515 def validate_changes_file_arg(file, fatal=1):
519 if file.endswith(".katie"):
520 file = file[:-6]+".changes";
522 if not file.endswith(".changes"):
523 error = "invalid file type; not a changes file";
525 if not os.access(file,os.R_OK):
526 if os.path.exists(file):
527 error = "permission denied";
529 error = "file not found";
533 fubar("%s: %s." % (orig_filename, error));
535 warn("Skipping %s - %s" % (orig_filename, error));
540 ################################################################################
543 return (arch != "source" and arch != "all");
545 ################################################################################
547 def join_with_commas_and(list):
548 if len(list) == 0: return "nothing";
549 if len(list) == 1: return list[0];
550 return ", ".join(list[:-1]) + " and " + list[-1];
552 ################################################################################
557 ################################################################################
559 # Handle -a, -c and -s arguments; returns them as SQL constraints
560 def parse_args(Options):
564 for suite in Options["Suite"].split():
565 suite_id = db_access.get_suite_id(suite);
567 warn("suite '%s' not recognised." % (suite));
569 suite_ids_list.append(suite_id);
571 con_suites = "AND su.id IN (%s)" % ", ".join(map(str, suite_ids_list));
573 fubar("No valid suite given.");
578 if Options["Component"]:
579 component_ids_list = [];
580 for component in Options["Component"].split():
581 component_id = db_access.get_component_id(component);
582 if component_id == -1:
583 warn("component '%s' not recognised." % (component));
585 component_ids_list.append(component_id);
586 if component_ids_list:
587 con_components = "AND c.id IN (%s)" % ", ".join(map(str, component_ids_list));
589 fubar("No valid component given.");
593 # Process architecture
594 con_architectures = "";
595 if Options["Architecture"]:
598 for architecture in Options["Architecture"].split():
599 if architecture == "source":
602 architecture_id = db_access.get_architecture_id(architecture);
603 if architecture_id == -1:
604 warn("architecture '%s' not recognised." % (architecture));
606 arch_ids_list.append(architecture_id);
608 con_architectures = "AND a.id IN (%s)" % ", ".join(map(str, arch_ids_list));
611 fubar("No valid architecture given.");
615 return (con_suites, con_architectures, con_components, check_source);
617 ################################################################################
619 # Inspired(tm) by Bryn Keller's print_exc_plus (See
620 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215)
623 tb = sys.exc_info()[2];
630 frame = frame.f_back;
632 traceback.print_exc();
634 print "\nFrame %s in %s at line %s" % (frame.f_code.co_name,
635 frame.f_code.co_filename,
637 for key, value in frame.f_locals.items():
638 print "\t%20s = " % key,;
642 print "<unable to print>";
644 ################################################################################
646 def try_with_debug(function):
654 ################################################################################
656 # Function for use in sorting lists of architectures.
657 # Sorts normally except that 'source' dominates all others.
659 def arch_compare_sw (a, b):
660 if a == "source" and b == "source":
669 ################################################################################
671 def Dict(**dict): return dict
673 ########################################
675 # Our very own version of commands.getouputstatus(), hacked to support
677 def gpgv_get_status_output(cmd, status_read, status_write):
678 cmd = ['/bin/sh', '-c', cmd];
679 p2cread, p2cwrite = os.pipe();
680 c2pread, c2pwrite = os.pipe();
681 errout, errin = os.pipe();
691 for i in range(3, 256):
692 if i != status_write:
698 os.execvp(cmd[0], cmd);
704 os.dup2(c2pread, c2pwrite);
705 os.dup2(errout, errin);
707 output = status = "";
709 i, o, e = select.select([c2pwrite, errin, status_read], [], []);
712 r = os.read(fd, 8196);
714 more_data.append(fd);
715 if fd == c2pwrite or fd == errin:
717 elif fd == status_read:
720 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd));
722 pid, exit_status = os.waitpid(pid, 0)
724 os.close(status_write);
725 os.close(status_read);
735 return output, status, exit_status;
737 ############################################################
740 def check_signature (filename, reject):
741 """Check the signature of a file and return the fingerprint if the
742 signature is valid or 'None' if it's not. The first argument is the
743 filename whose signature should be checked. The second argument is a
744 reject function and is called when an error is found. The reject()
745 function must allow for two arguments: the first is the error message,
746 the second is an optional prefix string. It is possible that reject()
747 is called more than once during an invocation of check_signature()."""
749 # Ensure the filename contains no shell meta-characters or other badness
750 if not re_taint_free.match(os.path.basename(filename)):
751 reject("!!WARNING!! tainted filename: '%s'." % (filename));
754 # Invoke gpgv on the file
755 status_read, status_write = os.pipe();
756 cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \
757 % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename);
758 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write);
760 # Process the status-fd output
762 bad = internal_error = "";
763 for line in status.split('\n'):
767 split = line.split();
769 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line);
771 (gnupg, keyword) = split[:2];
772 if gnupg != "[GNUPG:]":
773 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg);
776 if keywords.has_key(keyword) and (keyword != "NODATA" and keyword != "SIGEXPIRED"):
777 internal_error += "found duplicate status token ('%s').\n" % (keyword);
780 keywords[keyword] = args;
782 # If we failed to parse the status-fd output, let's just whine and bail now
784 reject("internal error while performing signature check on %s." % (filename));
785 reject(internal_error, "");
786 reject("Please report the above errors to the Archive maintainers by replying to this mail.", "");
789 # Now check for obviously bad things in the processed output
790 if keywords.has_key("SIGEXPIRED"):
791 reject("key used to sign %s has expired." % (filename));
793 if keywords.has_key("KEYREVOKED"):
794 reject("key used to sign %s has been revoked." % (filename));
796 if keywords.has_key("BADSIG"):
797 reject("bad signature on %s." % (filename));
799 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
800 reject("failed to check signature on %s." % (filename));
802 if keywords.has_key("NO_PUBKEY"):
803 reject("key used to sign %s not found in keyring." % (filename));
805 if keywords.has_key("BADARMOR"):
806 reject("ascii armour of signature was corrupt in %s." % (filename));
808 if keywords.has_key("NODATA"):
809 reject("no signature found in %s." % (filename));
815 # Next check gpgv exited with a zero return code
817 reject("gpgv failed while checking %s." % (filename));
819 reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "");
821 reject(prefix_multi_line_string(output, " [GPG output:] "), "");
824 # Sanity check the good stuff we expect
825 if not keywords.has_key("VALIDSIG"):
826 reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename));
829 args = keywords["VALIDSIG"];
831 reject("internal error while checking signature on %s." % (filename));
834 fingerprint = args[0];
835 if not keywords.has_key("GOODSIG"):
836 reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename));
838 if not keywords.has_key("SIG_ID"):
839 reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename));
842 # Finally ensure there's not something we don't recognise
843 known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
844 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
847 for keyword in keywords.keys():
848 if not known_keywords.has_key(keyword):
849 reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename));
857 ################################################################################
861 Cnf = apt_pkg.newConfiguration();
862 apt_pkg.ReadConfigFileISC(Cnf,default_config);
864 if which_conf_file() != default_config:
865 apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
867 ################################################################################