4 # Copyright (C) 2000, 2001, 2002, 2003 James Troup <james@nocrew.org>
5 # $Id: utils.py,v 1.57 2003-03-14 19:05:13 troup Exp $
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 import commands, os, pwd, re, select, socket, shutil, string, sys, tempfile, traceback;
27 ################################################################################
29 re_comments = re.compile(r"\#.*")
30 re_no_epoch = re.compile(r"^\d*\:")
31 re_no_revision = re.compile(r"\-[^-]*$")
32 re_arch_from_filename = re.compile(r"/binary-[^/]+/")
33 re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
34 re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$");
35 re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$");
37 re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)");
38 re_multi_line_field = re.compile(r"^\s(.*)");
39 re_taint_free = re.compile(r"^[-+~\.\w]+$");
41 re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\> \t]+)\>");
43 changes_parse_error_exc = "Can't parse line in .changes file";
44 invalid_dsc_format_exc = "Invalid .dsc file";
45 nk_format_exc = "Unknown Format: in .changes file";
46 no_files_exc = "No Files: field in .dsc file.";
47 cant_open_exc = "Can't read file.";
48 unknown_hostname_exc = "Unknown hostname";
49 cant_overwrite_exc = "Permission denied; can't overwrite existent file."
50 file_exists_exc = "Destination file exists";
51 send_mail_invalid_args_exc = "Both arguments are non-null.";
52 sendmail_failed_exc = "Sendmail invocation failed";
53 tried_too_hard_exc = "Tried too hard to find a free filename.";
55 default_config = "/etc/katie/katie.conf";
56 default_apt_config = "/etc/katie/apt.conf";
58 ######################################################################################
60 def open_file(filename, mode='r'):
62 f = open(filename, mode);
64 raise cant_open_exc, filename
67 ######################################################################################
69 def our_raw_input(prompt=""):
71 sys.stdout.write(prompt);
77 sys.stderr.write('\nUser interrupt (^D).\n');
80 ######################################################################################
84 if c not in string.digits:
88 ######################################################################################
90 def extract_component_from_section(section):
93 if section.find('/') != -1:
94 component = section.split('/')[0];
95 if component.lower() == "non-us" and section.count('/') > 0:
96 s = component + '/' + section.split('/')[1];
97 if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs
100 if section.lower() == "non-us":
101 component = "non-US/main";
103 # non-US prefix is case insensitive
104 if component.lower()[:6] == "non-us":
105 component = "non-US"+component[6:];
107 # Expand default component
109 if Cnf.has_key("Component::%s" % section):
113 elif component == "non-US":
114 component = "non-US/main";
116 return (section, component);
118 ######################################################################################
120 # dsc_whitespace_rules turns on strict format checking to avoid
121 # allowing in source packages which are unextracable by the
122 # inappropriately fragile dpkg-source.
127 # o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
128 # followed by any PGP header data and must end with a blank line.
130 # o The data section must end with a blank line and must be followed by
131 # "-----BEGIN PGP SIGNATURE-----".
133 def parse_changes(filename, dsc_whitespace_rules=0):
134 changes_in = open_file(filename);
137 lines = changes_in.readlines();
140 raise changes_parse_error_exc, "[Empty changes file]";
142 # Reindex by line number so we can easily verify the format of
148 indexed_lines[index] = line[:-1];
150 inside_signature = 0;
152 indices = indexed_lines.keys()
155 while index < max(indices):
157 line = indexed_lines[index];
159 if dsc_whitespace_rules:
161 if index > max(indices):
162 raise invalid_dsc_format_exc, index;
163 line = indexed_lines[index];
164 if not line.startswith("-----BEGIN PGP SIGNATURE"):
165 raise invalid_dsc_format_exc, index;
166 inside_signature = 0;
168 if line.startswith("-----BEGIN PGP SIGNATURE"):
170 if line.startswith("-----BEGIN PGP SIGNED MESSAGE"):
171 if dsc_whitespace_rules:
172 inside_signature = 1;
173 while index < max(indices) and line != "":
175 line = indexed_lines[index];
177 slf = re_single_line_field.match(line);
179 field = slf.groups()[0].lower();
180 changes[field] = slf.groups()[1];
184 changes[field] += '\n';
186 mlf = re_multi_line_field.match(line);
189 raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line);
190 if first == 1 and changes[field] != "":
191 changes[field] += '\n';
193 changes[field] += mlf.groups()[0] + '\n';
197 if dsc_whitespace_rules and inside_signature:
198 raise invalid_dsc_format_exc, index;
201 changes["filecontents"] = "".join(lines);
204 raise changes_parse_error_exc, error;
208 ######################################################################################
210 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
212 def build_file_list(changes, is_a_dsc=0):
214 format = changes.get("format", "")
216 format = float(format)
217 if not is_a_dsc and (format < 1.5 or format > 2.0):
218 raise nk_format_exc, format;
220 # No really, this has happened. Think 0 length .dsc file.
221 if not changes.has_key("files"):
224 for i in changes["files"].split("\n"):
228 section = priority = "";
231 (md5, size, name) = s
233 (md5, size, section, priority, name) = s
235 raise changes_parse_error_exc, i
237 if section == "": section = "-"
238 if priority == "": priority = "-"
240 (section, component) = extract_component_from_section(section);
242 files[name] = { "md5sum" : md5,
245 "priority": priority,
246 "component": component }
250 ######################################################################################
252 # Fix the `Maintainer:' field to be an RFC822 compatible address.
253 # cf. Debian Policy Manual (D.2.4)
255 # 06:28|<Culus> 'The standard sucks, but my tool is supposed to
256 # interoperate with it. I know - I'll fix the suckage
257 # and make things incompatible!'
259 def fix_maintainer (maintainer):
260 m = re_parse_maintainer.match(maintainer);
264 if m != None and len(m.groups()) == 2:
267 if name.find(',') != -1 or name.find('.') != -1:
268 rfc822 = "%s (%s)" % (email, name);
269 return (rfc822, name, email)
271 ######################################################################################
273 # sendmail wrapper, takes _either_ a message string or a file as arguments
274 def send_mail (message, filename=""):
275 # Sanity check arguments
276 if message != "" and filename != "":
277 raise send_mail_invalid_args_exc;
279 # If we've been passed a string dump it into a temporary file
281 filename = tempfile.mktemp();
282 fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
283 os.write (fd, message);
287 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename));
289 raise sendmail_failed_exc, output;
291 # Clean up any temporary files
293 os.unlink (filename);
295 ######################################################################################
297 def poolify (source, component):
300 # FIXME: this is nasty
301 component = component.lower().replace('non-us/', 'non-US/');
302 if source[:3] == "lib":
303 return component + source[:4] + '/' + source + '/'
305 return component + source[:1] + '/' + source + '/'
307 ######################################################################################
309 def move (src, dest, overwrite = 0, perms = 0664):
310 if os.path.exists(dest) and os.path.isdir(dest):
313 dest_dir = os.path.dirname(dest);
314 if not os.path.exists(dest_dir):
315 umask = os.umask(00000);
316 os.makedirs(dest_dir, 02775);
318 #print "Moving %s to %s..." % (src, dest);
319 if os.path.exists(dest) and os.path.isdir(dest):
320 dest += '/' + os.path.basename(src);
321 # Don't overwrite unless forced to
322 if os.path.exists(dest):
324 raise file_exists_exc;
326 if not os.access(dest, os.W_OK):
327 raise cant_overwrite_exc
328 shutil.copy2(src, dest);
329 os.chmod(dest, perms);
332 def copy (src, dest, overwrite = 0, perms = 0664):
333 if os.path.exists(dest) and os.path.isdir(dest):
336 dest_dir = os.path.dirname(dest);
337 if not os.path.exists(dest_dir):
338 umask = os.umask(00000);
339 os.makedirs(dest_dir, 02775);
341 #print "Copying %s to %s..." % (src, dest);
342 if os.path.exists(dest) and os.path.isdir(dest):
343 dest += '/' + os.path.basename(src);
344 # Don't overwrite unless forced to
345 if os.path.exists(dest):
347 raise file_exists_exc
349 if not os.access(dest, os.W_OK):
350 raise cant_overwrite_exc
351 shutil.copy2(src, dest);
352 os.chmod(dest, perms);
354 ######################################################################################
357 res = socket.gethostbyaddr(socket.gethostname());
358 database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname");
359 if database_hostname:
360 return database_hostname;
364 def which_conf_file ():
365 res = socket.gethostbyaddr(socket.gethostname());
366 if Cnf.get("Config::" + res[0] + "::KatieConfig"):
367 return Cnf["Config::" + res[0] + "::KatieConfig"]
369 return default_config;
371 def which_apt_conf_file ():
372 res = socket.gethostbyaddr(socket.gethostname());
373 if Cnf.get("Config::" + res[0] + "::AptConfig"):
374 return Cnf["Config::" + res[0] + "::AptConfig"]
376 return default_apt_config;
378 ######################################################################################
380 # Escape characters which have meaning to SQL's regex comparison operator ('~')
381 # (woefully incomplete)
384 s = s.replace('+', '\\\\+');
385 s = s.replace('.', '\\\\.');
388 ######################################################################################
390 # Perform a substition of template
391 def TemplateSubst(map, filename):
392 file = open_file(filename);
393 template = file.read();
395 template = template.replace(x,map[x]);
399 ######################################################################################
401 def fubar(msg, exit_code=1):
402 sys.stderr.write("E: %s\n" % (msg));
406 sys.stderr.write("W: %s\n" % (msg));
408 ######################################################################################
410 # Returns the user name with a laughable attempt at rfc822 conformancy
411 # (read: removing stray periods).
413 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '');
415 ######################################################################################
425 return ("%d%s" % (c, t))
427 ################################################################################
429 def cc_fix_changes (changes):
430 o = changes.get("architecture", "")
432 del changes["architecture"]
433 changes["architecture"] = {}
435 changes["architecture"][j] = 1
437 # Sort by source name, source version, 'have source', and then by filename
438 def changes_compare (a, b):
440 a_changes = parse_changes(a);
445 b_changes = parse_changes(b);
449 cc_fix_changes (a_changes);
450 cc_fix_changes (b_changes);
452 # Sort by source name
453 a_source = a_changes.get("source");
454 b_source = b_changes.get("source");
455 q = cmp (a_source, b_source);
459 # Sort by source version
460 a_version = a_changes.get("version");
461 b_version = b_changes.get("version");
462 q = apt_pkg.VersionCompare(a_version, b_version);
466 # Sort by 'have source'
467 a_has_source = a_changes["architecture"].get("source");
468 b_has_source = b_changes["architecture"].get("source");
469 if a_has_source and not b_has_source:
471 elif b_has_source and not a_has_source:
474 # Fall back to sort by filename
477 ################################################################################
479 def find_next_free (dest, too_many=100):
482 while os.path.exists(dest) and extra < too_many:
483 dest = orig_dest + '.' + repr(extra);
485 if extra >= too_many:
486 raise tried_too_hard_exc;
489 ################################################################################
491 def result_join (original, sep = '\t'):
493 for i in xrange(len(original)):
494 if original[i] == None:
497 list.append(original[i]);
498 return sep.join(list);
500 ################################################################################
502 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
504 for line in str.split('\n'):
506 if line or include_blank_lines:
507 out += "%s%s\n" % (prefix, line);
508 # Strip trailing new line
513 ################################################################################
515 def validate_changes_file_arg(file, fatal=1):
519 if file.endswith(".katie"):
520 file = file[:-6]+".changes";
522 if not file.endswith(".changes"):
523 error = "invalid file type; not a changes file";
525 if not os.access(file,os.R_OK):
526 if os.path.exists(file):
527 error = "permission denied";
529 error = "file not found";
533 fubar("%s: %s." % (orig_filename, error));
535 warn("Skipping %s - %s" % (orig_filename, error));
540 ################################################################################
543 return (arch != "source" and arch != "all");
545 ################################################################################
547 def join_with_commas_and(list):
548 if len(list) == 0: return "nothing";
549 if len(list) == 1: return list[0];
550 return ", ".join(list[:-1]) + " and " + list[-1];
552 ################################################################################
557 ################################################################################
559 # Handle -a, -c and -s arguments; returns them as SQL constraints
560 def parse_args(Options):
564 for suite in split_args(Options["Suite"]):
565 suite_id = db_access.get_suite_id(suite);
567 warn("suite '%s' not recognised." % (suite));
569 suite_ids_list.append(suite_id);
571 con_suites = "AND su.id IN (%s)" % ", ".join(map(str, suite_ids_list));
573 fubar("No valid suite given.");
578 if Options["Component"]:
579 component_ids_list = [];
580 for component in split_args(Options["Component"]):
581 component_id = db_access.get_component_id(component);
582 if component_id == -1:
583 warn("component '%s' not recognised." % (component));
585 component_ids_list.append(component_id);
586 if component_ids_list:
587 con_components = "AND c.id IN (%s)" % ", ".join(map(str, component_ids_list));
589 fubar("No valid component given.");
593 # Process architecture
594 con_architectures = "";
595 if Options["Architecture"]:
598 for architecture in split_args(Options["Architecture"]):
599 if architecture == "source":
602 architecture_id = db_access.get_architecture_id(architecture);
603 if architecture_id == -1:
604 warn("architecture '%s' not recognised." % (architecture));
606 arch_ids_list.append(architecture_id);
608 con_architectures = "AND a.id IN (%s)" % ", ".join(map(str, arch_ids_list));
611 fubar("No valid architecture given.");
615 return (con_suites, con_architectures, con_components, check_source);
617 ################################################################################
619 # Inspired(tm) by Bryn Keller's print_exc_plus (See
620 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215)
623 tb = sys.exc_info()[2];
630 frame = frame.f_back;
632 traceback.print_exc();
634 print "\nFrame %s in %s at line %s" % (frame.f_code.co_name,
635 frame.f_code.co_filename,
637 for key, value in frame.f_locals.items():
638 print "\t%20s = " % key,;
642 print "<unable to print>";
644 ################################################################################
646 def try_with_debug(function):
654 ################################################################################
656 # Function for use in sorting lists of architectures.
657 # Sorts normally except that 'source' dominates all others.
659 def arch_compare_sw (a, b):
660 if a == "source" and b == "source":
669 ################################################################################
671 # Split command line arguments which can be separated by either commas
672 # or whitespace. If dwim is set, it will complain about string ending
673 # in comma since this usually means someone did 'madison -a i386, m68k
674 # foo' or something and the inevitable confusion resulting from 'm68k'
675 # being treated as an argument is undesirable.
677 def split_args (s, dwim=1):
678 if s.find(",") == -1:
681 if s[-1:] == "," and dwim:
682 fubar("split_args: found trailing comma, spurious space maybe?");
685 ################################################################################
687 def Dict(**dict): return dict
689 ########################################
691 # Our very own version of commands.getouputstatus(), hacked to support
693 def gpgv_get_status_output(cmd, status_read, status_write):
694 cmd = ['/bin/sh', '-c', cmd];
695 p2cread, p2cwrite = os.pipe();
696 c2pread, c2pwrite = os.pipe();
697 errout, errin = os.pipe();
707 for i in range(3, 256):
708 if i != status_write:
714 os.execvp(cmd[0], cmd);
720 os.dup2(c2pread, c2pwrite);
721 os.dup2(errout, errin);
723 output = status = "";
725 i, o, e = select.select([c2pwrite, errin, status_read], [], []);
728 r = os.read(fd, 8196);
730 more_data.append(fd);
731 if fd == c2pwrite or fd == errin:
733 elif fd == status_read:
736 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd));
738 pid, exit_status = os.waitpid(pid, 0)
740 os.close(status_write);
741 os.close(status_read);
751 return output, status, exit_status;
753 ############################################################
756 def check_signature (filename, reject):
757 """Check the signature of a file and return the fingerprint if the
758 signature is valid or 'None' if it's not. The first argument is the
759 filename whose signature should be checked. The second argument is a
760 reject function and is called when an error is found. The reject()
761 function must allow for two arguments: the first is the error message,
762 the second is an optional prefix string. It is possible that reject()
763 is called more than once during an invocation of check_signature()."""
765 # Ensure the filename contains no shell meta-characters or other badness
766 if not re_taint_free.match(os.path.basename(filename)):
767 reject("!!WARNING!! tainted filename: '%s'." % (filename));
770 # Invoke gpgv on the file
771 status_read, status_write = os.pipe();
772 cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \
773 % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename);
774 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write);
776 # Process the status-fd output
778 bad = internal_error = "";
779 for line in status.split('\n'):
783 split = line.split();
785 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line);
787 (gnupg, keyword) = split[:2];
788 if gnupg != "[GNUPG:]":
789 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg);
792 if keywords.has_key(keyword) and (keyword != "NODATA" and keyword != "SIGEXPIRED"):
793 internal_error += "found duplicate status token ('%s').\n" % (keyword);
796 keywords[keyword] = args;
798 # If we failed to parse the status-fd output, let's just whine and bail now
800 reject("internal error while performing signature check on %s." % (filename));
801 reject(internal_error, "");
802 reject("Please report the above errors to the Archive maintainers by replying to this mail.", "");
805 # Now check for obviously bad things in the processed output
806 if keywords.has_key("SIGEXPIRED"):
807 reject("key used to sign %s has expired." % (filename));
809 if keywords.has_key("KEYREVOKED"):
810 reject("key used to sign %s has been revoked." % (filename));
812 if keywords.has_key("BADSIG"):
813 reject("bad signature on %s." % (filename));
815 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
816 reject("failed to check signature on %s." % (filename));
818 if keywords.has_key("NO_PUBKEY"):
819 reject("key used to sign %s not found in keyring." % (filename));
821 if keywords.has_key("BADARMOR"):
822 reject("ascii armour of signature was corrupt in %s." % (filename));
824 if keywords.has_key("NODATA"):
825 reject("no signature found in %s." % (filename));
831 # Next check gpgv exited with a zero return code
833 reject("gpgv failed while checking %s." % (filename));
835 reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "");
837 reject(prefix_multi_line_string(output, " [GPG output:] "), "");
840 # Sanity check the good stuff we expect
841 if not keywords.has_key("VALIDSIG"):
842 reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename));
845 args = keywords["VALIDSIG"];
847 reject("internal error while checking signature on %s." % (filename));
850 fingerprint = args[0];
851 if not keywords.has_key("GOODSIG"):
852 reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename));
854 if not keywords.has_key("SIG_ID"):
855 reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename));
858 # Finally ensure there's not something we don't recognise
859 known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
860 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
863 for keyword in keywords.keys():
864 if not known_keywords.has_key(keyword):
865 reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename));
873 ################################################################################
877 Cnf = apt_pkg.newConfiguration();
878 apt_pkg.ReadConfigFileISC(Cnf,default_config);
880 if which_conf_file() != default_config:
881 apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
883 ################################################################################