4 # Copyright (C) 2000, 2001, 2002, 2003 James Troup <james@nocrew.org>
5 # $Id: utils.py,v 1.60 2003-11-17 17:59:29 troup Exp $
7 ################################################################################
9 # This program is free software; you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation; either version 2 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
19 # You should have received a copy of the GNU General Public License
20 # along with this program; if not, write to the Free Software
21 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 ################################################################################
25 import commands, os, pwd, re, select, socket, shutil, string, sys, tempfile, traceback;
29 ################################################################################
31 re_comments = re.compile(r"\#.*")
32 re_no_epoch = re.compile(r"^\d*\:")
33 re_no_revision = re.compile(r"\-[^-]*$")
34 re_arch_from_filename = re.compile(r"/binary-[^/]+/")
35 re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
36 re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$");
37 re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$");
39 re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)");
40 re_multi_line_field = re.compile(r"^\s(.*)");
41 re_taint_free = re.compile(r"^[-+~\.\w]+$");
43 re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\> \t]+)\>");
45 changes_parse_error_exc = "Can't parse line in .changes file";
46 invalid_dsc_format_exc = "Invalid .dsc file";
47 nk_format_exc = "Unknown Format: in .changes file";
48 no_files_exc = "No Files: field in .dsc file.";
49 cant_open_exc = "Can't read file.";
50 unknown_hostname_exc = "Unknown hostname";
51 cant_overwrite_exc = "Permission denied; can't overwrite existent file."
52 file_exists_exc = "Destination file exists";
53 sendmail_failed_exc = "Sendmail invocation failed";
54 tried_too_hard_exc = "Tried too hard to find a free filename.";
56 default_config = "/etc/katie/katie.conf";
57 default_apt_config = "/etc/katie/apt.conf";
59 ################################################################################
61 def open_file(filename, mode='r'):
63 f = open(filename, mode);
65 raise cant_open_exc, filename;
68 ################################################################################
70 def our_raw_input(prompt=""):
72 sys.stdout.write(prompt);
78 sys.stderr.write("\nUser interrupt (^D).\n");
81 ################################################################################
85 if c not in string.digits:
89 ################################################################################
91 def extract_component_from_section(section):
94 if section.find('/') != -1:
95 component = section.split('/')[0];
96 if component.lower() == "non-us" and section.count('/') > 0:
97 s = component + '/' + section.split('/')[1];
98 if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs
101 if section.lower() == "non-us":
102 component = "non-US/main";
104 # non-US prefix is case insensitive
105 if component.lower()[:6] == "non-us":
106 component = "non-US"+component[6:];
108 # Expand default component
110 if Cnf.has_key("Component::%s" % section):
114 elif component == "non-US":
115 component = "non-US/main";
117 return (section, component);
119 ################################################################################
121 # Parses a changes file and returns a dictionary where each field is a
122 # key. The mandatory first argument is the filename of the .changes
125 # dsc_whitespace_rules is an optional boolean argument which defaults
126 # to off. If true, it turns on strict format checking to avoid
127 # allowing in source packages which are unextracable by the
128 # inappropriately fragile dpkg-source.
132 # o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
133 # followed by any PGP header data and must end with a blank line.
135 # o The data section must end with a blank line and must be followed by
136 # "-----BEGIN PGP SIGNATURE-----".
138 def parse_changes(filename, dsc_whitespace_rules=0):
142 changes_in = open_file(filename);
143 lines = changes_in.readlines();
146 raise changes_parse_error_exc, "[Empty changes file]";
148 # Reindex by line number so we can easily verify the format of
154 indexed_lines[index] = line[:-1];
156 inside_signature = 0;
158 num_of_lines = len(indexed_lines.keys());
161 while index < num_of_lines:
163 line = indexed_lines[index];
165 if dsc_whitespace_rules:
167 if index > num_of_lines:
168 raise invalid_dsc_format_exc, index;
169 line = indexed_lines[index];
170 if not line.startswith("-----BEGIN PGP SIGNATURE"):
171 raise invalid_dsc_format_exc, index;
172 inside_signature = 0;
176 if line.startswith("-----BEGIN PGP SIGNATURE"):
178 if line.startswith("-----BEGIN PGP SIGNED MESSAGE"):
179 if dsc_whitespace_rules:
180 inside_signature = 1;
181 while index < num_of_lines and line != "":
183 line = indexed_lines[index];
185 slf = re_single_line_field.match(line);
187 field = slf.groups()[0].lower();
188 changes[field] = slf.groups()[1];
192 changes[field] += '\n';
194 mlf = re_multi_line_field.match(line);
197 raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line);
198 if first == 1 and changes[field] != "":
199 changes[field] += '\n';
201 changes[field] += mlf.groups()[0] + '\n';
205 if dsc_whitespace_rules and inside_signature:
206 raise invalid_dsc_format_exc, index;
209 changes["filecontents"] = "".join(lines);
212 raise changes_parse_error_exc, error;
216 ################################################################################
218 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
220 def build_file_list(changes, is_a_dsc=0):
223 # Make sure we have a Files: field to parse...
224 if not changes.has_key("files"):
227 # Make sure we recognise the format of the Files: field
228 format = changes.get("format", "");
230 format = float(format);
231 if not is_a_dsc and (format < 1.5 or format > 2.0):
232 raise nk_format_exc, format;
234 # Parse each entry/line:
235 for i in changes["files"].split('\n'):
239 section = priority = "";
242 (md5, size, name) = s;
244 (md5, size, section, priority, name) = s;
246 raise changes_parse_error_exc, i;
253 (section, component) = extract_component_from_section(section);
255 files[name] = Dict(md5sum=md5, size=size, section=section,
256 priority=priority, component=component);
260 ################################################################################
262 # Fix the `Maintainer:' field to be an RFC822 compatible address.
263 # cf. Debian Policy Manual (D.2.4)
265 # 06:28|<Culus> 'The standard sucks, but my tool is supposed to
266 # interoperate with it. I know - I'll fix the suckage
267 # and make things incompatible!'
269 def fix_maintainer (maintainer):
270 m = re_parse_maintainer.match(maintainer);
274 if m != None and len(m.groups()) == 2:
277 if name.find(',') != -1 or name.find('.') != -1:
278 rfc822 = "%s (%s)" % (email, name);
279 return (rfc822, name, email)
281 ################################################################################
283 # sendmail wrapper, takes _either_ a message string or a file as arguments
284 def send_mail (message, filename=""):
285 # If we've been passed a string dump it into a temporary file
287 filename = tempfile.mktemp();
288 fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
289 os.write (fd, message);
293 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename));
295 raise sendmail_failed_exc, output;
297 # Clean up any temporary files
299 os.unlink (filename);
301 ################################################################################
303 def poolify (source, component):
306 # FIXME: this is nasty
307 component = component.lower().replace("non-us/", "non-US/");
308 if source[:3] == "lib":
309 return component + source[:4] + '/' + source + '/'
311 return component + source[:1] + '/' + source + '/'
313 ################################################################################
315 def move (src, dest, overwrite = 0, perms = 0664):
316 if os.path.exists(dest) and os.path.isdir(dest):
319 dest_dir = os.path.dirname(dest);
320 if not os.path.exists(dest_dir):
321 umask = os.umask(00000);
322 os.makedirs(dest_dir, 02775);
324 #print "Moving %s to %s..." % (src, dest);
325 if os.path.exists(dest) and os.path.isdir(dest):
326 dest += '/' + os.path.basename(src);
327 # Don't overwrite unless forced to
328 if os.path.exists(dest):
330 fubar("Can't move %s to %s - file already exists." % (src, dest));
332 if not os.access(dest, os.W_OK):
333 fubar("Can't move %s to %s - can't write to existing file." % (src, dest));
334 shutil.copy2(src, dest);
335 os.chmod(dest, perms);
338 def copy (src, dest, overwrite = 0, perms = 0664):
339 if os.path.exists(dest) and os.path.isdir(dest):
342 dest_dir = os.path.dirname(dest);
343 if not os.path.exists(dest_dir):
344 umask = os.umask(00000);
345 os.makedirs(dest_dir, 02775);
347 #print "Copying %s to %s..." % (src, dest);
348 if os.path.exists(dest) and os.path.isdir(dest):
349 dest += '/' + os.path.basename(src);
350 # Don't overwrite unless forced to
351 if os.path.exists(dest):
353 raise file_exists_exc
355 if not os.access(dest, os.W_OK):
356 raise cant_overwrite_exc
357 shutil.copy2(src, dest);
358 os.chmod(dest, perms);
360 ################################################################################
363 res = socket.gethostbyaddr(socket.gethostname());
364 database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname");
365 if database_hostname:
366 return database_hostname;
370 def which_conf_file ():
371 res = socket.gethostbyaddr(socket.gethostname());
372 if Cnf.get("Config::" + res[0] + "::KatieConfig"):
373 return Cnf["Config::" + res[0] + "::KatieConfig"]
375 return default_config;
377 def which_apt_conf_file ():
378 res = socket.gethostbyaddr(socket.gethostname());
379 if Cnf.get("Config::" + res[0] + "::AptConfig"):
380 return Cnf["Config::" + res[0] + "::AptConfig"]
382 return default_apt_config;
384 ################################################################################
386 # Escape characters which have meaning to SQL's regex comparison operator ('~')
387 # (woefully incomplete)
390 s = s.replace('+', '\\\\+');
391 s = s.replace('.', '\\\\.');
394 ################################################################################
396 # Perform a substition of template
397 def TemplateSubst(map, filename):
398 file = open_file(filename);
399 template = file.read();
401 template = template.replace(x,map[x]);
405 ################################################################################
407 def fubar(msg, exit_code=1):
408 sys.stderr.write("E: %s\n" % (msg));
412 sys.stderr.write("W: %s\n" % (msg));
414 ################################################################################
416 # Returns the user name with a laughable attempt at rfc822 conformancy
417 # (read: removing stray periods).
419 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '');
421 ################################################################################
431 return ("%d%s" % (c, t))
433 ################################################################################
435 def cc_fix_changes (changes):
436 o = changes.get("architecture", "");
438 del changes["architecture"];
439 changes["architecture"] = {};
441 changes["architecture"][j] = 1;
443 # Sort by source name, source version, 'have source', and then by filename
444 def changes_compare (a, b):
446 a_changes = parse_changes(a);
451 b_changes = parse_changes(b);
455 cc_fix_changes (a_changes);
456 cc_fix_changes (b_changes);
458 # Sort by source name
459 a_source = a_changes.get("source");
460 b_source = b_changes.get("source");
461 q = cmp (a_source, b_source);
465 # Sort by source version
466 a_version = a_changes.get("version");
467 b_version = b_changes.get("version");
468 q = apt_pkg.VersionCompare(a_version, b_version);
472 # Sort by 'have source'
473 a_has_source = a_changes["architecture"].get("source");
474 b_has_source = b_changes["architecture"].get("source");
475 if a_has_source and not b_has_source:
477 elif b_has_source and not a_has_source:
480 # Fall back to sort by filename
483 ################################################################################
485 def find_next_free (dest, too_many=100):
488 while os.path.exists(dest) and extra < too_many:
489 dest = orig_dest + '.' + repr(extra);
491 if extra >= too_many:
492 raise tried_too_hard_exc;
495 ################################################################################
497 def result_join (original, sep = '\t'):
499 for i in xrange(len(original)):
500 if original[i] == None:
503 list.append(original[i]);
504 return sep.join(list);
506 ################################################################################
508 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
510 for line in str.split('\n'):
512 if line or include_blank_lines:
513 out += "%s%s\n" % (prefix, line);
514 # Strip trailing new line
519 ################################################################################
521 def validate_changes_file_arg(file, fatal=1):
525 if file.endswith(".katie"):
526 file = file[:-6]+".changes";
528 if not file.endswith(".changes"):
529 error = "invalid file type; not a changes file";
531 if not os.access(file,os.R_OK):
532 if os.path.exists(file):
533 error = "permission denied";
535 error = "file not found";
539 fubar("%s: %s." % (orig_filename, error));
541 warn("Skipping %s - %s" % (orig_filename, error));
546 ################################################################################
549 return (arch != "source" and arch != "all");
551 ################################################################################
553 def join_with_commas_and(list):
554 if len(list) == 0: return "nothing";
555 if len(list) == 1: return list[0];
556 return ", ".join(list[:-1]) + " and " + list[-1];
558 ################################################################################
563 ################################################################################
565 # Handle -a, -c and -s arguments; returns them as SQL constraints
566 def parse_args(Options):
570 for suite in split_args(Options["Suite"]):
571 suite_id = db_access.get_suite_id(suite);
573 warn("suite '%s' not recognised." % (suite));
575 suite_ids_list.append(suite_id);
577 con_suites = "AND su.id IN (%s)" % ", ".join(map(str, suite_ids_list));
579 fubar("No valid suite given.");
584 if Options["Component"]:
585 component_ids_list = [];
586 for component in split_args(Options["Component"]):
587 component_id = db_access.get_component_id(component);
588 if component_id == -1:
589 warn("component '%s' not recognised." % (component));
591 component_ids_list.append(component_id);
592 if component_ids_list:
593 con_components = "AND c.id IN (%s)" % ", ".join(map(str, component_ids_list));
595 fubar("No valid component given.");
599 # Process architecture
600 con_architectures = "";
601 if Options["Architecture"]:
604 for architecture in split_args(Options["Architecture"]):
605 if architecture == "source":
608 architecture_id = db_access.get_architecture_id(architecture);
609 if architecture_id == -1:
610 warn("architecture '%s' not recognised." % (architecture));
612 arch_ids_list.append(architecture_id);
614 con_architectures = "AND a.id IN (%s)" % ", ".join(map(str, arch_ids_list));
617 fubar("No valid architecture given.");
621 return (con_suites, con_architectures, con_components, check_source);
623 ################################################################################
625 # Inspired(tm) by Bryn Keller's print_exc_plus (See
626 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215)
629 tb = sys.exc_info()[2];
636 frame = frame.f_back;
638 traceback.print_exc();
640 print "\nFrame %s in %s at line %s" % (frame.f_code.co_name,
641 frame.f_code.co_filename,
643 for key, value in frame.f_locals.items():
644 print "\t%20s = " % key,;
648 print "<unable to print>";
650 ################################################################################
652 def try_with_debug(function):
660 ################################################################################
662 # Function for use in sorting lists of architectures.
663 # Sorts normally except that 'source' dominates all others.
665 def arch_compare_sw (a, b):
666 if a == "source" and b == "source":
675 ################################################################################
677 # Split command line arguments which can be separated by either commas
678 # or whitespace. If dwim is set, it will complain about string ending
679 # in comma since this usually means someone did 'madison -a i386, m68k
680 # foo' or something and the inevitable confusion resulting from 'm68k'
681 # being treated as an argument is undesirable.
683 def split_args (s, dwim=1):
684 if s.find(",") == -1:
687 if s[-1:] == "," and dwim:
688 fubar("split_args: found trailing comma, spurious space maybe?");
691 ################################################################################
693 def Dict(**dict): return dict
695 ########################################
697 # Our very own version of commands.getouputstatus(), hacked to support
699 def gpgv_get_status_output(cmd, status_read, status_write):
700 cmd = ['/bin/sh', '-c', cmd];
701 p2cread, p2cwrite = os.pipe();
702 c2pread, c2pwrite = os.pipe();
703 errout, errin = os.pipe();
713 for i in range(3, 256):
714 if i != status_write:
720 os.execvp(cmd[0], cmd);
726 os.dup2(c2pread, c2pwrite);
727 os.dup2(errout, errin);
729 output = status = "";
731 i, o, e = select.select([c2pwrite, errin, status_read], [], []);
734 r = os.read(fd, 8196);
736 more_data.append(fd);
737 if fd == c2pwrite or fd == errin:
739 elif fd == status_read:
742 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd));
744 pid, exit_status = os.waitpid(pid, 0)
746 os.close(status_write);
747 os.close(status_read);
757 return output, status, exit_status;
759 ############################################################
762 def check_signature (filename, reject):
763 """Check the signature of a file and return the fingerprint if the
764 signature is valid or 'None' if it's not. The first argument is the
765 filename whose signature should be checked. The second argument is a
766 reject function and is called when an error is found. The reject()
767 function must allow for two arguments: the first is the error message,
768 the second is an optional prefix string. It's possible for reject()
769 to be called more than once during an invocation of check_signature()."""
771 # Ensure the filename contains no shell meta-characters or other badness
772 if not re_taint_free.match(os.path.basename(filename)):
773 reject("!!WARNING!! tainted filename: '%s'." % (filename));
776 # Invoke gpgv on the file
777 status_read, status_write = os.pipe();
778 cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \
779 % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename);
780 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write);
782 # Process the status-fd output
784 bad = internal_error = "";
785 for line in status.split('\n'):
789 split = line.split();
791 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line);
793 (gnupg, keyword) = split[:2];
794 if gnupg != "[GNUPG:]":
795 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg);
798 if keywords.has_key(keyword) and (keyword != "NODATA" and keyword != "SIGEXPIRED"):
799 internal_error += "found duplicate status token ('%s').\n" % (keyword);
802 keywords[keyword] = args;
804 # If we failed to parse the status-fd output, let's just whine and bail now
806 reject("internal error while performing signature check on %s." % (filename));
807 reject(internal_error, "");
808 reject("Please report the above errors to the Archive maintainers by replying to this mail.", "");
811 # Now check for obviously bad things in the processed output
812 if keywords.has_key("SIGEXPIRED"):
813 reject("The key used to sign %s has expired." % (filename));
815 if keywords.has_key("KEYREVOKED"):
816 reject("The key used to sign %s has been revoked." % (filename));
818 if keywords.has_key("BADSIG"):
819 reject("bad signature on %s." % (filename));
821 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
822 reject("failed to check signature on %s." % (filename));
824 if keywords.has_key("NO_PUBKEY"):
825 args = keywords["NO_PUBKEY"];
828 reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, filename));
830 if keywords.has_key("BADARMOR"):
831 reject("ASCII armour of signature was corrupt in %s." % (filename));
833 if keywords.has_key("NODATA"):
834 reject("no signature found in %s." % (filename));
840 # Next check gpgv exited with a zero return code
842 reject("gpgv failed while checking %s." % (filename));
844 reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "");
846 reject(prefix_multi_line_string(output, " [GPG output:] "), "");
849 # Sanity check the good stuff we expect
850 if not keywords.has_key("VALIDSIG"):
851 reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename));
854 args = keywords["VALIDSIG"];
856 reject("internal error while checking signature on %s." % (filename));
859 fingerprint = args[0];
860 if not keywords.has_key("GOODSIG"):
861 reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename));
863 if not keywords.has_key("SIG_ID"):
864 reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename));
867 # Finally ensure there's not something we don't recognise
868 known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
869 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
872 for keyword in keywords.keys():
873 if not known_keywords.has_key(keyword):
874 reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename));
882 ################################################################################
884 # Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603
886 def wrap(paragraph, max_length, prefix=""):
890 words = paragraph.split();
893 word_size = len(word);
894 if word_size > max_length:
896 s += line + '\n' + prefix;
897 s += word + '\n' + prefix;
900 new_length = len(line) + word_size + 1;
901 if new_length > max_length:
902 s += line + '\n' + prefix;
915 ################################################################################
917 # Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
918 # Returns fixed 'src'
919 def clean_symlink (src, dest, root):
920 src = src.replace(root, '', 1);
921 dest = dest.replace(root, '', 1);
922 dest = os.path.dirname(dest);
923 new_src = '../' * len(dest.split('/'));
924 return new_src + src;
926 ################################################################################
930 Cnf = apt_pkg.newConfiguration();
931 apt_pkg.ReadConfigFileISC(Cnf,default_config);
933 if which_conf_file() != default_config:
934 apt_pkg.ReadConfigFileISC(Cnf,which_conf_file());
936 ################################################################################