2 # vim:set et ts=4 sw=4:
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @license: GNU General Public License version 2 or later
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
39 import email as modemail
42 from dbconn import DBConn, get_architecture, get_component, get_suite
43 from dak_exceptions import *
44 from textutils import fix_maintainer
45 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
46 re_multi_line_field, re_srchasver, re_taint_free, \
47 re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource
49 from formats import parse_format, validate_changes_format
50 from srcformats import get_format_from_string
51 from collections import defaultdict
53 ################################################################################
55 default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
56 default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used
58 alias_cache = None #: Cache for email alias checks
59 key_uid_email_cache = {} #: Cache for email addresses from gpg key uids
61 # (hashname, function, earliest_changes_version)
62 known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
63 ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
65 # Monkeypatch commands.getstatusoutput as it may not return the correct exit
66 # code in lenny's Python. This also affects commands.getoutput and
68 def dak_getstatusoutput(cmd):
69 pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
70 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
72 output = "".join(pipe.stdout.readlines())
74 if output[-1:] == '\n':
82 commands.getstatusoutput = dak_getstatusoutput
84 ################################################################################
87 """ Escape html chars """
88 return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
90 ################################################################################
92 def open_file(filename, mode='r'):
94 Open C{file}, return fileobject.
96 @type filename: string
97 @param filename: path/filename to open
100 @param mode: open mode
103 @return: open fileobject
105 @raise CantOpenError: If IOError is raised by open, reraise it as CantOpenError.
109 f = open(filename, mode)
111 raise CantOpenError, filename
114 ################################################################################
116 def our_raw_input(prompt=""):
118 sys.stdout.write(prompt)
124 sys.stderr.write("\nUser interrupt (^D).\n")
127 ################################################################################
129 def extract_component_from_section(section):
132 if section.find('/') != -1:
133 component = section.split('/')[0]
135 # Expand default component
137 if Cnf.has_key("Component::%s" % section):
142 return (section, component)
144 ################################################################################
146 def parse_deb822(contents, signing_rules=0):
150 # Split the lines in the input, keeping the linebreaks.
151 lines = contents.splitlines(True)
154 raise ParseChangesError, "[Empty changes file]"
156 # Reindex by line number so we can easily verify the format of
162 indexed_lines[index] = line[:-1]
166 num_of_lines = len(indexed_lines.keys())
169 while index < num_of_lines:
171 line = indexed_lines[index]
173 if signing_rules == 1:
175 if index > num_of_lines:
176 raise InvalidDscError, index
177 line = indexed_lines[index]
178 if not line.startswith("-----BEGIN PGP SIGNATURE"):
179 raise InvalidDscError, index
184 if line.startswith("-----BEGIN PGP SIGNATURE"):
186 if line.startswith("-----BEGIN PGP SIGNED MESSAGE"):
188 if signing_rules == 1:
189 while index < num_of_lines and line != "":
191 line = indexed_lines[index]
193 # If we're not inside the signed data, don't process anything
194 if signing_rules >= 0 and not inside_signature:
196 slf = re_single_line_field.match(line)
198 field = slf.groups()[0].lower()
199 changes[field] = slf.groups()[1]
203 changes[field] += '\n'
205 mlf = re_multi_line_field.match(line)
208 raise ParseChangesError, "'%s'\n [Multi-line field continuing on from nothing?]" % (line)
209 if first == 1 and changes[field] != "":
210 changes[field] += '\n'
212 changes[field] += mlf.groups()[0] + '\n'
216 if signing_rules == 1 and inside_signature:
217 raise InvalidDscError, index
219 changes["filecontents"] = "".join(lines)
221 if changes.has_key("source"):
222 # Strip the source version in brackets from the source field,
223 # put it in the "source-version" field instead.
224 srcver = re_srchasver.search(changes["source"])
226 changes["source"] = srcver.group(1)
227 changes["source-version"] = srcver.group(2)
230 raise ParseChangesError, error
234 ################################################################################
236 def parse_changes(filename, signing_rules=0):
238 Parses a changes file and returns a dictionary where each field is a
239 key. The mandatory first argument is the filename of the .changes
242 signing_rules is an optional argument:
244 - If signing_rules == -1, no signature is required.
245 - If signing_rules == 0 (the default), a signature is required.
246 - If signing_rules == 1, it turns on the same strict format checking
249 The rules for (signing_rules == 1)-mode are:
251 - The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
252 followed by any PGP header data and must end with a blank line.
254 - The data section must end with a blank line and must be followed by
255 "-----BEGIN PGP SIGNATURE-----".
258 changes_in = open_file(filename)
259 content = changes_in.read()
262 unicode(content, 'utf-8')
264 raise ChangesUnicodeError, "Changes file not proper utf-8"
265 return parse_deb822(content, signing_rules)
267 ################################################################################
269 def hash_key(hashname):
270 return '%ssum' % hashname
272 ################################################################################
274 def create_hash(where, files, hashname, hashfunc):
276 create_hash extends the passed files dict with the given hash by
277 iterating over all files on disk and passing them to the hashing
282 for f in files.keys():
284 file_handle = open_file(f)
285 except CantOpenError:
286 rejmsg.append("Could not open file %s for checksumming" % (f))
289 files[f][hash_key(hashname)] = hashfunc(file_handle)
294 ################################################################################
296 def check_hash(where, files, hashname, hashfunc):
298 check_hash checks the given hash in the files dict against the actual
299 files on disk. The hash values need to be present consistently in
300 all file entries. It does not modify its input in any way.
304 for f in files.keys():
308 file_handle = open_file(f)
310 # Check for the hash entry, to not trigger a KeyError.
311 if not files[f].has_key(hash_key(hashname)):
312 rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,
316 # Actually check the hash for correctness.
317 if hashfunc(file_handle) != files[f][hash_key(hashname)]:
318 rejmsg.append("%s: %s check failed in %s" % (f, hashname,
320 except CantOpenError:
321 # TODO: This happens when the file is in the pool.
322 # warn("Cannot open file %s" % f)
329 ################################################################################
331 def check_size(where, files):
333 check_size checks the file sizes in the passed files dict against the
338 for f in files.keys():
343 # TODO: This happens when the file is in the pool.
347 actual_size = entry[stat.ST_SIZE]
348 size = int(files[f]["size"])
349 if size != actual_size:
350 rejmsg.append("%s: actual file size (%s) does not match size (%s) in %s"
351 % (f, actual_size, size, where))
354 ################################################################################
356 def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
358 Verify that the files listed in the Files field of the .dsc are
359 those expected given the announced Format.
361 @type dsc_filename: string
362 @param dsc_filename: path of .dsc file
365 @param dsc: the content of the .dsc parsed by C{parse_changes()}
367 @type dsc_files: dict
368 @param dsc_files: the file list returned by C{build_file_list()}
371 @return: all errors detected
375 # Parse the file if needed
377 dsc = parse_changes(dsc_filename, signing_rules=1);
379 if dsc_files is None:
380 dsc_files = build_file_list(dsc, is_a_dsc=1)
382 # Ensure .dsc lists proper set of source files according to the format
384 has = defaultdict(lambda: 0)
387 (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')),
388 (r'diff.gz', ('debian_diff',)),
389 (r'tar.gz', ('native_tar_gz', 'native_tar')),
390 (r'debian\.tar\.(gz|bz2)', ('debian_tar',)),
391 (r'orig\.tar\.(gz|bz2)', ('orig_tar',)),
392 (r'tar\.(gz|bz2)', ('native_tar',)),
393 (r'orig-.+\.tar\.(gz|bz2)', ('more_orig_tar',)),
396 for f in dsc_files.keys():
397 m = re_issource.match(f)
399 rejmsg.append("%s: %s in Files field not recognised as source."
403 # Populate 'has' dictionary by resolving keys in lookup table
405 for regex, keys in ftype_lookup:
406 if re.match(regex, m.group(3)):
412 # File does not match anything in lookup table; reject
414 reject("%s: unexpected source file '%s'" % (dsc_filename, f))
416 # Check for multiple files
417 for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'):
418 if has[file_type] > 1:
419 rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type))
421 # Source format specific tests
423 format = get_format_from_string(dsc['format'])
425 '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has)
428 except UnknownFormatError:
429 # Not an error here for now
434 ################################################################################
436 def check_hash_fields(what, manifest):
438 check_hash_fields ensures that there are no checksum fields in the
439 given dict that we do not know about.
443 hashes = map(lambda x: x[0], known_hashes)
444 for field in manifest:
445 if field.startswith("checksums-"):
446 hashname = field.split("-",1)[1]
447 if hashname not in hashes:
448 rejmsg.append("Unsupported checksum field for %s "\
449 "in %s" % (hashname, what))
452 ################################################################################
454 def _ensure_changes_hash(changes, format, version, files, hashname, hashfunc):
455 if format >= version:
456 # The version should contain the specified hash.
459 # Import hashes from the changes
460 rejmsg = parse_checksums(".changes", files, changes, hashname)
464 # We need to calculate the hash because it can't possibly
467 return func(".changes", files, hashname, hashfunc)
469 # We could add the orig which might be in the pool to the files dict to
470 # access the checksums easily.
472 def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
474 ensure_dsc_hashes' task is to ensure that each and every *present* hash
475 in the dsc is correct, i.e. identical to the changes file and if necessary
476 the pool. The latter task is delegated to check_hash.
480 if not dsc.has_key('Checksums-%s' % (hashname,)):
482 # Import hashes from the dsc
483 parse_checksums(".dsc", dsc_files, dsc, hashname)
485 rejmsg.extend(check_hash(".dsc", dsc_files, hashname, hashfunc))
488 ################################################################################
490 def parse_checksums(where, files, manifest, hashname):
492 field = 'checksums-%s' % hashname
493 if not field in manifest:
495 for line in manifest[field].split('\n'):
498 clist = line.strip().split(' ')
500 checksum, size, checkfile = clist
502 rejmsg.append("Cannot parse checksum line [%s]" % (line))
504 if not files.has_key(checkfile):
505 # TODO: check for the file's entry in the original files dict, not
506 # the one modified by (auto)byhand and other weird stuff
507 # rejmsg.append("%s: not present in files but in checksums-%s in %s" %
508 # (file, hashname, where))
510 if not files[checkfile]["size"] == size:
511 rejmsg.append("%s: size differs for files and checksums-%s entry "\
512 "in %s" % (checkfile, hashname, where))
514 files[checkfile][hash_key(hashname)] = checksum
515 for f in files.keys():
516 if not files[f].has_key(hash_key(hashname)):
517 rejmsg.append("%s: no entry in checksums-%s in %s" % (checkfile,
521 ################################################################################
523 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
525 def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
528 # Make sure we have a Files: field to parse...
529 if not changes.has_key(field):
530 raise NoFilesFieldError
532 # Validate .changes Format: field
534 validate_changes_format(parse_format(changes['format']), field)
536 includes_section = (not is_a_dsc) and field == "files"
538 # Parse each entry/line:
539 for i in changes[field].split('\n'):
543 section = priority = ""
546 (md5, size, section, priority, name) = s
548 (md5, size, name) = s
550 raise ParseChangesError, i
557 (section, component) = extract_component_from_section(section)
559 files[name] = dict(size=size, section=section,
560 priority=priority, component=component)
561 files[name][hashname] = md5
565 ################################################################################
567 def send_mail (message, filename=""):
568 """sendmail wrapper, takes _either_ a message string or a file as arguments"""
570 # If we've been passed a string dump it into a temporary file
572 (fd, filename) = tempfile.mkstemp()
573 os.write (fd, message)
576 if Cnf.has_key("Dinstall::MailWhiteList") and \
577 Cnf["Dinstall::MailWhiteList"] != "":
578 message_in = open_file(filename)
579 message_raw = modemail.message_from_file(message_in)
583 whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
585 for line in whitelist_in:
586 if not re_whitespace_comment.match(line):
587 if re_re_mark.match(line):
588 whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
590 whitelist.append(re.compile(re.escape(line.strip())))
595 fields = ["To", "Bcc", "Cc"]
598 value = message_raw.get(field, None)
601 for item in value.split(","):
602 (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip())
608 if not mail_whitelisted:
609 print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
613 # Doesn't have any mail in whitelist so remove the header
615 del message_raw[field]
617 message_raw.replace_header(field, ', '.join(match))
619 # Change message fields in order if we don't have a To header
620 if not message_raw.has_key("To"):
623 if message_raw.has_key(field):
624 message_raw[fields[-1]] = message_raw[field]
625 del message_raw[field]
628 # Clean up any temporary files
629 # and return, as we removed all recipients.
631 os.unlink (filename);
634 fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0700);
635 os.write (fd, message_raw.as_string(True));
639 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
641 raise SendmailFailedError, output
643 # Clean up any temporary files
647 ################################################################################
649 def poolify (source, component):
652 if source[:3] == "lib":
653 return component + source[:4] + '/' + source + '/'
655 return component + source[:1] + '/' + source + '/'
657 ################################################################################
659 def move (src, dest, overwrite = 0, perms = 0664):
660 if os.path.exists(dest) and os.path.isdir(dest):
663 dest_dir = os.path.dirname(dest)
664 if not os.path.exists(dest_dir):
665 umask = os.umask(00000)
666 os.makedirs(dest_dir, 02775)
668 #print "Moving %s to %s..." % (src, dest)
669 if os.path.exists(dest) and os.path.isdir(dest):
670 dest += '/' + os.path.basename(src)
671 # Don't overwrite unless forced to
672 if os.path.exists(dest):
674 fubar("Can't move %s to %s - file already exists." % (src, dest))
676 if not os.access(dest, os.W_OK):
677 fubar("Can't move %s to %s - can't write to existing file." % (src, dest))
678 shutil.copy2(src, dest)
679 os.chmod(dest, perms)
682 def copy (src, dest, overwrite = 0, perms = 0664):
683 if os.path.exists(dest) and os.path.isdir(dest):
686 dest_dir = os.path.dirname(dest)
687 if not os.path.exists(dest_dir):
688 umask = os.umask(00000)
689 os.makedirs(dest_dir, 02775)
691 #print "Copying %s to %s..." % (src, dest)
692 if os.path.exists(dest) and os.path.isdir(dest):
693 dest += '/' + os.path.basename(src)
694 # Don't overwrite unless forced to
695 if os.path.exists(dest):
697 raise FileExistsError
699 if not os.access(dest, os.W_OK):
700 raise CantOverwriteError
701 shutil.copy2(src, dest)
702 os.chmod(dest, perms)
704 ################################################################################
707 res = socket.gethostbyaddr(socket.gethostname())
708 database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname")
709 if database_hostname:
710 return database_hostname
714 def which_conf_file ():
715 if os.getenv('DAK_CONFIG'):
716 return os.getenv('DAK_CONFIG')
718 res = socket.gethostbyaddr(socket.gethostname())
719 # In case we allow local config files per user, try if one exists
720 if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
721 homedir = os.getenv("HOME")
722 confpath = os.path.join(homedir, "/etc/dak.conf")
723 if os.path.exists(confpath):
724 apt_pkg.ReadConfigFileISC(Cnf,default_config)
726 # We are still in here, so there is no local config file or we do
727 # not allow local files. Do the normal stuff.
728 if Cnf.get("Config::" + res[0] + "::DakConfig"):
729 return Cnf["Config::" + res[0] + "::DakConfig"]
731 return default_config
733 def which_apt_conf_file ():
734 res = socket.gethostbyaddr(socket.gethostname())
735 # In case we allow local config files per user, try if one exists
736 if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
737 homedir = os.getenv("HOME")
738 confpath = os.path.join(homedir, "/etc/dak.conf")
739 if os.path.exists(confpath):
740 apt_pkg.ReadConfigFileISC(Cnf,default_config)
742 if Cnf.get("Config::" + res[0] + "::AptConfig"):
743 return Cnf["Config::" + res[0] + "::AptConfig"]
745 return default_apt_config
747 def which_alias_file():
748 hostname = socket.gethostbyaddr(socket.gethostname())[0]
749 aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
750 if os.path.exists(aliasfn):
755 ################################################################################
757 def TemplateSubst(subst_map, filename):
758 """ Perform a substition of template """
759 templatefile = open_file(filename)
760 template = templatefile.read()
761 for k, v in subst_map.iteritems():
762 template = template.replace(k, str(v))
766 ################################################################################
768 def fubar(msg, exit_code=1):
769 sys.stderr.write("E: %s\n" % (msg))
773 sys.stderr.write("W: %s\n" % (msg))
775 ################################################################################
777 # Returns the user name with a laughable attempt at rfc822 conformancy
778 # (read: removing stray periods).
780 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '')
783 return pwd.getpwuid(os.getuid())[0]
785 ################################################################################
795 return ("%d%s" % (c, t))
797 ################################################################################
799 def cc_fix_changes (changes):
800 o = changes.get("architecture", "")
802 del changes["architecture"]
803 changes["architecture"] = {}
805 changes["architecture"][j] = 1
807 def changes_compare (a, b):
808 """ Sort by source name, source version, 'have source', and then by filename """
810 a_changes = parse_changes(a)
815 b_changes = parse_changes(b)
819 cc_fix_changes (a_changes)
820 cc_fix_changes (b_changes)
822 # Sort by source name
823 a_source = a_changes.get("source")
824 b_source = b_changes.get("source")
825 q = cmp (a_source, b_source)
829 # Sort by source version
830 a_version = a_changes.get("version", "0")
831 b_version = b_changes.get("version", "0")
832 q = apt_pkg.VersionCompare(a_version, b_version)
836 # Sort by 'have source'
837 a_has_source = a_changes["architecture"].get("source")
838 b_has_source = b_changes["architecture"].get("source")
839 if a_has_source and not b_has_source:
841 elif b_has_source and not a_has_source:
844 # Fall back to sort by filename
847 ################################################################################
849 def find_next_free (dest, too_many=100):
852 while os.path.exists(dest) and extra < too_many:
853 dest = orig_dest + '.' + repr(extra)
855 if extra >= too_many:
856 raise NoFreeFilenameError
859 ################################################################################
861 def result_join (original, sep = '\t'):
863 for i in xrange(len(original)):
864 if original[i] == None:
865 resultlist.append("")
867 resultlist.append(original[i])
868 return sep.join(resultlist)
870 ################################################################################
872 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
874 for line in str.split('\n'):
876 if line or include_blank_lines:
877 out += "%s%s\n" % (prefix, line)
878 # Strip trailing new line
883 ################################################################################
885 def validate_changes_file_arg(filename, require_changes=1):
887 'filename' is either a .changes or .dak file. If 'filename' is a
888 .dak file, it's changed to be the corresponding .changes file. The
889 function then checks if the .changes file a) exists and b) is
890 readable and returns the .changes filename if so. If there's a
891 problem, the next action depends on the option 'require_changes'
894 - If 'require_changes' == -1, errors are ignored and the .changes
895 filename is returned.
896 - If 'require_changes' == 0, a warning is given and 'None' is returned.
897 - If 'require_changes' == 1, a fatal error is raised.
902 orig_filename = filename
903 if filename.endswith(".dak"):
904 filename = filename[:-4]+".changes"
906 if not filename.endswith(".changes"):
907 error = "invalid file type; not a changes file"
909 if not os.access(filename,os.R_OK):
910 if os.path.exists(filename):
911 error = "permission denied"
913 error = "file not found"
916 if require_changes == 1:
917 fubar("%s: %s." % (orig_filename, error))
918 elif require_changes == 0:
919 warn("Skipping %s - %s" % (orig_filename, error))
921 else: # We only care about the .dak file
926 ################################################################################
929 return (arch != "source" and arch != "all")
931 ################################################################################
933 def join_with_commas_and(list):
934 if len(list) == 0: return "nothing"
935 if len(list) == 1: return list[0]
936 return ", ".join(list[:-1]) + " and " + list[-1]
938 ################################################################################
943 (pkg, version, constraint) = atom
945 pp_dep = "%s (%s %s)" % (pkg, constraint, version)
948 pp_deps.append(pp_dep)
949 return " |".join(pp_deps)
951 ################################################################################
956 ################################################################################
958 def parse_args(Options):
959 """ Handle -a, -c and -s arguments; returns them as SQL constraints """
960 # XXX: This should go away and everything which calls it be converted
961 # to use SQLA properly. For now, we'll just fix it not to use
962 # the old Pg interface though
963 session = DBConn().session()
967 for suitename in split_args(Options["Suite"]):
968 suite = get_suite(suitename, session=session)
969 if suite.suite_id is None:
970 warn("suite '%s' not recognised." % (suite.suite_name))
972 suite_ids_list.append(suite.suite_id)
974 con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
976 fubar("No valid suite given.")
981 if Options["Component"]:
982 component_ids_list = []
983 for componentname in split_args(Options["Component"]):
984 component = get_component(componentname, session=session)
985 if component is None:
986 warn("component '%s' not recognised." % (componentname))
988 component_ids_list.append(component.component_id)
989 if component_ids_list:
990 con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
992 fubar("No valid component given.")
996 # Process architecture
997 con_architectures = ""
999 if Options["Architecture"]:
1001 for archname in split_args(Options["Architecture"]):
1002 if archname == "source":
1005 arch = get_architecture(archname, session=session)
1007 warn("architecture '%s' not recognised." % (archname))
1009 arch_ids_list.append(arch.arch_id)
1011 con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
1013 if not check_source:
1014 fubar("No valid architecture given.")
1018 return (con_suites, con_architectures, con_components, check_source)
1020 ################################################################################
1022 # Inspired(tm) by Bryn Keller's print_exc_plus (See
1023 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215)
1026 tb = sys.exc_info()[2]
1033 frame = frame.f_back
1035 traceback.print_exc()
1037 print "\nFrame %s in %s at line %s" % (frame.f_code.co_name,
1038 frame.f_code.co_filename,
1040 for key, value in frame.f_locals.items():
1041 print "\t%20s = " % key,
1045 print "<unable to print>"
1047 ################################################################################
1049 def try_with_debug(function):
1057 ################################################################################
1059 def arch_compare_sw (a, b):
1061 Function for use in sorting lists of architectures.
1063 Sorts normally except that 'source' dominates all others.
1066 if a == "source" and b == "source":
1075 ################################################################################
1077 def split_args (s, dwim=1):
1079 Split command line arguments which can be separated by either commas
1080 or whitespace. If dwim is set, it will complain about string ending
1081 in comma since this usually means someone did 'dak ls -a i386, m68k
1082 foo' or something and the inevitable confusion resulting from 'm68k'
1083 being treated as an argument is undesirable.
1086 if s.find(",") == -1:
1089 if s[-1:] == "," and dwim:
1090 fubar("split_args: found trailing comma, spurious space maybe?")
1093 ################################################################################
1095 def gpgv_get_status_output(cmd, status_read, status_write):
1097 Our very own version of commands.getouputstatus(), hacked to support
1101 cmd = ['/bin/sh', '-c', cmd]
1102 p2cread, p2cwrite = os.pipe()
1103 c2pread, c2pwrite = os.pipe()
1104 errout, errin = os.pipe()
1114 for i in range(3, 256):
1115 if i != status_write:
1121 os.execvp(cmd[0], cmd)
1127 os.dup2(c2pread, c2pwrite)
1128 os.dup2(errout, errin)
1130 output = status = ""
1132 i, o, e = select.select([c2pwrite, errin, status_read], [], [])
1135 r = os.read(fd, 8196)
1137 more_data.append(fd)
1138 if fd == c2pwrite or fd == errin:
1140 elif fd == status_read:
1143 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd))
1145 pid, exit_status = os.waitpid(pid, 0)
1147 os.close(status_write)
1148 os.close(status_read)
1158 return output, status, exit_status
1160 ################################################################################
1162 def process_gpgv_output(status):
1163 # Process the status-fd output
1166 for line in status.split('\n'):
1170 split = line.split()
1172 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
1174 (gnupg, keyword) = split[:2]
1175 if gnupg != "[GNUPG:]":
1176 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
1179 if keywords.has_key(keyword) and keyword not in [ "NODATA", "SIGEXPIRED", "KEYEXPIRED" ]:
1180 internal_error += "found duplicate status token ('%s').\n" % (keyword)
1183 keywords[keyword] = args
1185 return (keywords, internal_error)
1187 ################################################################################
1189 def retrieve_key (filename, keyserver=None, keyring=None):
1191 Retrieve the key that signed 'filename' from 'keyserver' and
1192 add it to 'keyring'. Returns nothing on success, or an error message
1196 # Defaults for keyserver and keyring
1198 keyserver = Cnf["Dinstall::KeyServer"]
1200 keyring = Cnf.ValueList("Dinstall::GPGKeyring")[0]
1202 # Ensure the filename contains no shell meta-characters or other badness
1203 if not re_taint_free.match(filename):
1204 return "%s: tainted filename" % (filename)
1206 # Invoke gpgv on the file
1207 status_read, status_write = os.pipe()
1208 cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
1209 (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
1211 # Process the status-fd output
1212 (keywords, internal_error) = process_gpgv_output(status)
1214 return internal_error
1216 if not keywords.has_key("NO_PUBKEY"):
1217 return "didn't find expected NO_PUBKEY in gpgv status-fd output"
1219 fingerprint = keywords["NO_PUBKEY"][0]
1220 # XXX - gpg sucks. You can't use --secret-keyring=/dev/null as
1221 # it'll try to create a lockfile in /dev. A better solution might
1222 # be a tempfile or something.
1223 cmd = "gpg --no-default-keyring --secret-keyring=%s --no-options" \
1224 % (Cnf["Dinstall::SigningKeyring"])
1225 cmd += " --keyring %s --keyserver %s --recv-key %s" \
1226 % (keyring, keyserver, fingerprint)
1227 (result, output) = commands.getstatusoutput(cmd)
1229 return "'%s' failed with exit code %s" % (cmd, result)
1233 ################################################################################
1235 def gpg_keyring_args(keyrings=None):
1237 keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
1239 return " ".join(["--keyring %s" % x for x in keyrings])
1241 ################################################################################
1243 def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None):
1245 Check the signature of a file and return the fingerprint if the
1246 signature is valid or 'None' if it's not. The first argument is the
1247 filename whose signature should be checked. The second argument is a
1248 reject function and is called when an error is found. The reject()
1249 function must allow for two arguments: the first is the error message,
1250 the second is an optional prefix string. It's possible for reject()
1251 to be called more than once during an invocation of check_signature().
1252 The third argument is optional and is the name of the files the
1253 detached signature applies to. The fourth argument is optional and is
1254 a *list* of keyrings to use. 'autofetch' can either be None, True or
1255 False. If None, the default behaviour specified in the config will be
1261 # Ensure the filename contains no shell meta-characters or other badness
1262 if not re_taint_free.match(sig_filename):
1263 rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
1264 return (None, rejects)
1266 if data_filename and not re_taint_free.match(data_filename):
1267 rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
1268 return (None, rejects)
1271 keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
1273 # Autofetch the signing key if that's enabled
1274 if autofetch == None:
1275 autofetch = Cnf.get("Dinstall::KeyAutoFetch")
1277 error_msg = retrieve_key(sig_filename)
1279 rejects.append(error_msg)
1280 return (None, rejects)
1282 # Build the command line
1283 status_read, status_write = os.pipe()
1284 cmd = "gpgv --status-fd %s %s %s %s" % (
1285 status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
1287 # Invoke gpgv on the file
1288 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
1290 # Process the status-fd output
1291 (keywords, internal_error) = process_gpgv_output(status)
1293 # If we failed to parse the status-fd output, let's just whine and bail now
1295 rejects.append("internal error while performing signature check on %s." % (sig_filename))
1296 rejects.append(internal_error, "")
1297 rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
1298 return (None, rejects)
1300 # Now check for obviously bad things in the processed output
1301 if keywords.has_key("KEYREVOKED"):
1302 rejects.append("The key used to sign %s has been revoked." % (sig_filename))
1303 if keywords.has_key("BADSIG"):
1304 rejects.append("bad signature on %s." % (sig_filename))
1305 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
1306 rejects.append("failed to check signature on %s." % (sig_filename))
1307 if keywords.has_key("NO_PUBKEY"):
1308 args = keywords["NO_PUBKEY"]
1311 rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
1312 if keywords.has_key("BADARMOR"):
1313 rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))
1314 if keywords.has_key("NODATA"):
1315 rejects.append("no signature found in %s." % (sig_filename))
1316 if keywords.has_key("EXPKEYSIG"):
1317 args = keywords["EXPKEYSIG"]
1320 rejects.append("Signature made by expired key 0x%s" % (key))
1321 if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
1322 args = keywords["KEYEXPIRED"]
1326 if timestamp.count("T") == 0:
1328 expiredate = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
1330 expiredate = "unknown (%s)" % (timestamp)
1332 expiredate = timestamp
1333 rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
1335 if len(rejects) > 0:
1336 return (None, rejects)
1338 # Next check gpgv exited with a zero return code
1340 rejects.append("gpgv failed while checking %s." % (sig_filename))
1342 rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
1344 rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "")
1345 return (None, rejects)
1347 # Sanity check the good stuff we expect
1348 if not keywords.has_key("VALIDSIG"):
1349 rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
1351 args = keywords["VALIDSIG"]
1353 rejects.append("internal error while checking signature on %s." % (sig_filename))
1355 fingerprint = args[0]
1356 if not keywords.has_key("GOODSIG"):
1357 rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
1358 if not keywords.has_key("SIG_ID"):
1359 rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
1361 # Finally ensure there's not something we don't recognise
1362 known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
1363 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
1364 NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="")
1366 for keyword in keywords.keys():
1367 if not known_keywords.has_key(keyword):
1368 rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
1370 if len(rejects) > 0:
1371 return (None, rejects)
1373 return (fingerprint, [])
1375 ################################################################################
1377 def gpg_get_key_addresses(fingerprint):
1378 """retreive email addresses from gpg key uids for a given fingerprint"""
1379 addresses = key_uid_email_cache.get(fingerprint)
1380 if addresses != None:
1383 cmd = "gpg --no-default-keyring %s --fingerprint %s" \
1384 % (gpg_keyring_args(), fingerprint)
1385 (result, output) = commands.getstatusoutput(cmd)
1387 for l in output.split('\n'):
1388 m = re_gpg_uid.match(l)
1390 addresses.add(m.group(1))
1391 key_uid_email_cache[fingerprint] = addresses
1394 ################################################################################
1396 # Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603
1398 def wrap(paragraph, max_length, prefix=""):
1402 words = paragraph.split()
1405 word_size = len(word)
1406 if word_size > max_length:
1408 s += line + '\n' + prefix
1409 s += word + '\n' + prefix
1412 new_length = len(line) + word_size + 1
1413 if new_length > max_length:
1414 s += line + '\n' + prefix
1427 ################################################################################
1429 def clean_symlink (src, dest, root):
1431 Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
1434 src = src.replace(root, '', 1)
1435 dest = dest.replace(root, '', 1)
1436 dest = os.path.dirname(dest)
1437 new_src = '../' * len(dest.split('/'))
1438 return new_src + src
1440 ################################################################################
1442 def temp_filename(directory=None, prefix="dak", suffix=""):
1444 Return a secure and unique filename by pre-creating it.
1445 If 'directory' is non-null, it will be the directory the file is pre-created in.
1446 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1447 If 'suffix' is non-null, the filename will end with it.
1449 Returns a pair (fd, name).
1452 return tempfile.mkstemp(suffix, prefix, directory)
1454 ################################################################################
1456 def temp_dirname(parent=None, prefix="dak", suffix=""):
1458 Return a secure and unique directory by pre-creating it.
1459 If 'parent' is non-null, it will be the directory the directory is pre-created in.
1460 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1461 If 'suffix' is non-null, the filename will end with it.
1463 Returns a pathname to the new directory
1466 return tempfile.mkdtemp(suffix, prefix, parent)
1468 ################################################################################
1470 def is_email_alias(email):
1471 """ checks if the user part of the email is listed in the alias file """
1473 if alias_cache == None:
1474 aliasfn = which_alias_file()
1477 for l in open(aliasfn):
1478 alias_cache.add(l.split(':')[0])
1479 uid = email.split('@')[0]
1480 return uid in alias_cache
1482 ################################################################################
1484 def get_changes_files(from_dir):
1486 Takes a directory and lists all .changes files in it (as well as chdir'ing
1487 to the directory; this is due to broken behaviour on the part of p-u/p-a
1488 when you're not in the right place)
1490 Returns a list of filenames
1493 # Much of the rest of p-u/p-a depends on being in the right place
1495 changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')]
1497 fubar("Failed to read list from directory %s (%s)" % (from_dir, e))
1499 return changes_files
1501 ################################################################################
1505 Cnf = apt_pkg.newConfiguration()
1506 if not os.getenv("DAK_TEST"):
1507 apt_pkg.ReadConfigFileISC(Cnf,default_config)
1509 if which_conf_file() != default_config:
1510 apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())