2 # vim:set et ts=4 sw=4:
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @license: GNU General Public License version 2 or later
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
41 import email as modemail
44 from dbconn import DBConn, get_architecture, get_component, get_suite, \
45 get_override_type, Keyring, session_wrapper, \
46 get_active_keyring_paths, get_primary_keyring_path, \
47 get_suite_architectures, get_or_set_metadatakey, DBSource
48 from sqlalchemy import desc
49 from dak_exceptions import *
50 from gpg import SignedFile
51 from textutils import fix_maintainer
52 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
53 re_multi_line_field, re_srchasver, re_taint_free, \
54 re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \
55 re_is_orig_source, re_build_dep_arch
57 from formats import parse_format, validate_changes_format
58 from srcformats import get_format_from_string
59 from collections import defaultdict
61 ################################################################################
63 default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
64 default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used
66 alias_cache = None #: Cache for email alias checks
67 key_uid_email_cache = {} #: Cache for email addresses from gpg key uids
69 # (hashname, function, earliest_changes_version)
70 known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
71 ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
73 # Monkeypatch commands.getstatusoutput as it may not return the correct exit
74 # code in lenny's Python. This also affects commands.getoutput and
76 def dak_getstatusoutput(cmd):
77 pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
78 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
80 output = pipe.stdout.read()
84 if output[-1:] == '\n':
92 commands.getstatusoutput = dak_getstatusoutput
94 ################################################################################
97 """ Escape html chars """
98 return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
100 ################################################################################
102 def open_file(filename, mode='r'):
104 Open C{file}, return fileobject.
106 @type filename: string
107 @param filename: path/filename to open
110 @param mode: open mode
113 @return: open fileobject
115 @raise CantOpenError: If IOError is raised by open, reraise it as CantOpenError.
119 f = open(filename, mode)
121 raise CantOpenError(filename)
124 ################################################################################
126 def our_raw_input(prompt=""):
130 sys.stdout.write(prompt)
139 sys.stderr.write("\nUser interrupt (^D).\n")
142 ################################################################################
144 def extract_component_from_section(section, session=None):
147 if section.find('/') != -1:
148 component = section.split('/')[0]
150 # Expand default component
152 comp = get_component(section, session)
156 component = comp.component_name
158 return (section, component)
160 ################################################################################
162 def parse_deb822(armored_contents, signing_rules=0, keyrings=None, session=None):
163 require_signature = True
166 require_signature = False
168 signed_file = SignedFile(armored_contents, keyrings=keyrings, require_signature=require_signature)
169 contents = signed_file.contents
174 # Split the lines in the input, keeping the linebreaks.
175 lines = contents.splitlines(True)
178 raise ParseChangesError("[Empty changes file]")
180 # Reindex by line number so we can easily verify the format of
186 indexed_lines[index] = line[:-1]
188 num_of_lines = len(indexed_lines.keys())
191 while index < num_of_lines:
193 line = indexed_lines[index]
194 if line == "" and signing_rules == 1:
195 if index != num_of_lines:
196 raise InvalidDscError(index)
198 slf = re_single_line_field.match(line)
200 field = slf.groups()[0].lower()
201 changes[field] = slf.groups()[1]
205 changes[field] += '\n'
207 mlf = re_multi_line_field.match(line)
210 raise ParseChangesError("'%s'\n [Multi-line field continuing on from nothing?]" % (line))
211 if first == 1 and changes[field] != "":
212 changes[field] += '\n'
214 changes[field] += mlf.groups()[0] + '\n'
218 changes["filecontents"] = armored_contents
220 if changes.has_key("source"):
221 # Strip the source version in brackets from the source field,
222 # put it in the "source-version" field instead.
223 srcver = re_srchasver.search(changes["source"])
225 changes["source"] = srcver.group(1)
226 changes["source-version"] = srcver.group(2)
229 raise ParseChangesError(error)
233 ################################################################################
235 def parse_changes(filename, signing_rules=0, dsc_file=0, keyrings=None):
237 Parses a changes file and returns a dictionary where each field is a
238 key. The mandatory first argument is the filename of the .changes
241 signing_rules is an optional argument:
243 - If signing_rules == -1, no signature is required.
244 - If signing_rules == 0 (the default), a signature is required.
245 - If signing_rules == 1, it turns on the same strict format checking
248 The rules for (signing_rules == 1)-mode are:
250 - The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
251 followed by any PGP header data and must end with a blank line.
253 - The data section must end with a blank line and must be followed by
254 "-----BEGIN PGP SIGNATURE-----".
257 changes_in = open_file(filename)
258 content = changes_in.read()
261 unicode(content, 'utf-8')
263 raise ChangesUnicodeError("Changes file not proper utf-8")
264 changes = parse_deb822(content, signing_rules, keyrings=keyrings)
268 # Finally ensure that everything needed for .changes is there
269 must_keywords = ('Format', 'Date', 'Source', 'Binary', 'Architecture', 'Version',
270 'Distribution', 'Maintainer', 'Description', 'Changes', 'Files')
273 for keyword in must_keywords:
274 if not changes.has_key(keyword.lower()):
275 missingfields.append(keyword)
277 if len(missingfields):
278 raise ParseChangesError("Missing mandantory field(s) in changes file (policy 5.5): %s" % (missingfields))
282 ################################################################################
284 def hash_key(hashname):
285 return '%ssum' % hashname
287 ################################################################################
289 def create_hash(where, files, hashname, hashfunc):
291 create_hash extends the passed files dict with the given hash by
292 iterating over all files on disk and passing them to the hashing
297 for f in files.keys():
299 file_handle = open_file(f)
300 except CantOpenError:
301 rejmsg.append("Could not open file %s for checksumming" % (f))
304 files[f][hash_key(hashname)] = hashfunc(file_handle)
309 ################################################################################
311 def check_hash(where, files, hashname, hashfunc):
313 check_hash checks the given hash in the files dict against the actual
314 files on disk. The hash values need to be present consistently in
315 all file entries. It does not modify its input in any way.
319 for f in files.keys():
323 file_handle = open_file(f)
325 # Check for the hash entry, to not trigger a KeyError.
326 if not files[f].has_key(hash_key(hashname)):
327 rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,
331 # Actually check the hash for correctness.
332 if hashfunc(file_handle) != files[f][hash_key(hashname)]:
333 rejmsg.append("%s: %s check failed in %s" % (f, hashname,
335 except CantOpenError:
336 # TODO: This happens when the file is in the pool.
337 # warn("Cannot open file %s" % f)
344 ################################################################################
346 def check_size(where, files):
348 check_size checks the file sizes in the passed files dict against the
353 for f in files.keys():
356 except OSError as exc:
358 # TODO: This happens when the file is in the pool.
362 actual_size = entry[stat.ST_SIZE]
363 size = int(files[f]["size"])
364 if size != actual_size:
365 rejmsg.append("%s: actual file size (%s) does not match size (%s) in %s"
366 % (f, actual_size, size, where))
369 ################################################################################
371 def check_dsc_files(dsc_filename, dsc, dsc_files):
373 Verify that the files listed in the Files field of the .dsc are
374 those expected given the announced Format.
376 @type dsc_filename: string
377 @param dsc_filename: path of .dsc file
380 @param dsc: the content of the .dsc parsed by C{parse_changes()}
382 @type dsc_files: dict
383 @param dsc_files: the file list returned by C{build_file_list()}
386 @return: all errors detected
390 # Ensure .dsc lists proper set of source files according to the format
392 has = defaultdict(lambda: 0)
395 (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')),
396 (r'diff.gz', ('debian_diff',)),
397 (r'tar.gz', ('native_tar_gz', 'native_tar')),
398 (r'debian\.tar\.(gz|bz2|xz)', ('debian_tar',)),
399 (r'orig\.tar\.(gz|bz2|xz)', ('orig_tar',)),
400 (r'tar\.(gz|bz2|xz)', ('native_tar',)),
401 (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
405 m = re_issource.match(f)
407 rejmsg.append("%s: %s in Files field not recognised as source."
411 # Populate 'has' dictionary by resolving keys in lookup table
413 for regex, keys in ftype_lookup:
414 if re.match(regex, m.group(3)):
420 # File does not match anything in lookup table; reject
422 reject("%s: unexpected source file '%s'" % (dsc_filename, f))
424 # Check for multiple files
425 for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'):
426 if has[file_type] > 1:
427 rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type))
429 # Source format specific tests
431 format = get_format_from_string(dsc['format'])
433 '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has)
436 except UnknownFormatError:
437 # Not an error here for now
442 ################################################################################
444 def check_hash_fields(what, manifest):
446 check_hash_fields ensures that there are no checksum fields in the
447 given dict that we do not know about.
451 hashes = map(lambda x: x[0], known_hashes)
452 for field in manifest:
453 if field.startswith("checksums-"):
454 hashname = field.split("-",1)[1]
455 if hashname not in hashes:
456 rejmsg.append("Unsupported checksum field for %s "\
457 "in %s" % (hashname, what))
460 ################################################################################
462 def _ensure_changes_hash(changes, format, version, files, hashname, hashfunc):
463 if format >= version:
464 # The version should contain the specified hash.
467 # Import hashes from the changes
468 rejmsg = parse_checksums(".changes", files, changes, hashname)
472 # We need to calculate the hash because it can't possibly
475 return func(".changes", files, hashname, hashfunc)
477 # We could add the orig which might be in the pool to the files dict to
478 # access the checksums easily.
480 def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
482 ensure_dsc_hashes' task is to ensure that each and every *present* hash
483 in the dsc is correct, i.e. identical to the changes file and if necessary
484 the pool. The latter task is delegated to check_hash.
488 if not dsc.has_key('Checksums-%s' % (hashname,)):
490 # Import hashes from the dsc
491 parse_checksums(".dsc", dsc_files, dsc, hashname)
493 rejmsg.extend(check_hash(".dsc", dsc_files, hashname, hashfunc))
496 ################################################################################
498 def parse_checksums(where, files, manifest, hashname):
500 field = 'checksums-%s' % hashname
501 if not field in manifest:
503 for line in manifest[field].split('\n'):
506 clist = line.strip().split(' ')
508 checksum, size, checkfile = clist
510 rejmsg.append("Cannot parse checksum line [%s]" % (line))
512 if not files.has_key(checkfile):
513 # TODO: check for the file's entry in the original files dict, not
514 # the one modified by (auto)byhand and other weird stuff
515 # rejmsg.append("%s: not present in files but in checksums-%s in %s" %
516 # (file, hashname, where))
518 if not files[checkfile]["size"] == size:
519 rejmsg.append("%s: size differs for files and checksums-%s entry "\
520 "in %s" % (checkfile, hashname, where))
522 files[checkfile][hash_key(hashname)] = checksum
523 for f in files.keys():
524 if not files[f].has_key(hash_key(hashname)):
525 rejmsg.append("%s: no entry in checksums-%s in %s" % (f, hashname, where))
528 ################################################################################
530 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
532 def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
535 # Make sure we have a Files: field to parse...
536 if not changes.has_key(field):
537 raise NoFilesFieldError
539 # Validate .changes Format: field
541 validate_changes_format(parse_format(changes['format']), field)
543 includes_section = (not is_a_dsc) and field == "files"
545 # Parse each entry/line:
546 for i in changes[field].split('\n'):
550 section = priority = ""
553 (md5, size, section, priority, name) = s
555 (md5, size, name) = s
557 raise ParseChangesError(i)
564 (section, component) = extract_component_from_section(section)
566 files[name] = dict(size=size, section=section,
567 priority=priority, component=component)
568 files[name][hashname] = md5
572 ################################################################################
574 # see http://bugs.debian.org/619131
575 def build_package_list(dsc, session = None):
576 if not dsc.has_key("package-list"):
581 for line in dsc["package-list"].split("\n"):
585 fields = line.split()
587 package_type = fields[1]
588 (section, component) = extract_component_from_section(fields[2])
591 # Validate type if we have a session
592 if session and get_override_type(package_type, session) is None:
593 # Maybe just warn and ignore? exit(1) might be a bit hard...
594 utils.fubar("invalid type (%s) in Package-List." % (package_type))
596 if name not in packages or packages[name]["type"] == "dsc":
597 packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[])
601 ################################################################################
603 def send_mail (message, filename=""):
604 """sendmail wrapper, takes _either_ a message string or a file as arguments"""
606 maildir = Cnf.get('Dir::Mail')
608 path = os.path.join(maildir, datetime.datetime.now().isoformat())
609 path = find_next_free(path)
614 # Check whether we're supposed to be sending mail
615 if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]:
618 # If we've been passed a string dump it into a temporary file
620 (fd, filename) = tempfile.mkstemp()
621 os.write (fd, message)
624 if Cnf.has_key("Dinstall::MailWhiteList") and \
625 Cnf["Dinstall::MailWhiteList"] != "":
626 message_in = open_file(filename)
627 message_raw = modemail.message_from_file(message_in)
631 whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
633 for line in whitelist_in:
634 if not re_whitespace_comment.match(line):
635 if re_re_mark.match(line):
636 whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
638 whitelist.append(re.compile(re.escape(line.strip())))
643 fields = ["To", "Bcc", "Cc"]
646 value = message_raw.get(field, None)
649 for item in value.split(","):
650 (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip())
656 if not mail_whitelisted:
657 print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
661 # Doesn't have any mail in whitelist so remove the header
663 del message_raw[field]
665 message_raw.replace_header(field, ', '.join(match))
667 # Change message fields in order if we don't have a To header
668 if not message_raw.has_key("To"):
671 if message_raw.has_key(field):
672 message_raw[fields[-1]] = message_raw[field]
673 del message_raw[field]
676 # Clean up any temporary files
677 # and return, as we removed all recipients.
679 os.unlink (filename);
682 fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0o700);
683 os.write (fd, message_raw.as_string(True));
687 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
689 raise SendmailFailedError(output)
691 # Clean up any temporary files
695 ################################################################################
697 def poolify (source, component=None):
698 if source[:3] == "lib":
699 return source[:4] + '/' + source + '/'
701 return source[:1] + '/' + source + '/'
703 ################################################################################
705 def move (src, dest, overwrite = 0, perms = 0o664):
706 if os.path.exists(dest) and os.path.isdir(dest):
709 dest_dir = os.path.dirname(dest)
710 if not os.path.exists(dest_dir):
711 umask = os.umask(00000)
712 os.makedirs(dest_dir, 0o2775)
714 #print "Moving %s to %s..." % (src, dest)
715 if os.path.exists(dest) and os.path.isdir(dest):
716 dest += '/' + os.path.basename(src)
717 # Don't overwrite unless forced to
718 if os.path.exists(dest):
720 fubar("Can't move %s to %s - file already exists." % (src, dest))
722 if not os.access(dest, os.W_OK):
723 fubar("Can't move %s to %s - can't write to existing file." % (src, dest))
724 shutil.copy2(src, dest)
725 os.chmod(dest, perms)
728 def copy (src, dest, overwrite = 0, perms = 0o664):
729 if os.path.exists(dest) and os.path.isdir(dest):
732 dest_dir = os.path.dirname(dest)
733 if not os.path.exists(dest_dir):
734 umask = os.umask(00000)
735 os.makedirs(dest_dir, 0o2775)
737 #print "Copying %s to %s..." % (src, dest)
738 if os.path.exists(dest) and os.path.isdir(dest):
739 dest += '/' + os.path.basename(src)
740 # Don't overwrite unless forced to
741 if os.path.exists(dest):
743 raise FileExistsError
745 if not os.access(dest, os.W_OK):
746 raise CantOverwriteError
747 shutil.copy2(src, dest)
748 os.chmod(dest, perms)
750 ################################################################################
753 res = socket.getfqdn()
754 database_hostname = Cnf.get("Config::" + res + "::DatabaseHostname")
755 if database_hostname:
756 return database_hostname
760 def which_conf_file ():
761 if os.getenv('DAK_CONFIG'):
762 return os.getenv('DAK_CONFIG')
764 res = socket.getfqdn()
765 # In case we allow local config files per user, try if one exists
766 if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
767 homedir = os.getenv("HOME")
768 confpath = os.path.join(homedir, "/etc/dak.conf")
769 if os.path.exists(confpath):
770 apt_pkg.ReadConfigFileISC(Cnf,confpath)
772 # We are still in here, so there is no local config file or we do
773 # not allow local files. Do the normal stuff.
774 if Cnf.get("Config::" + res + "::DakConfig"):
775 return Cnf["Config::" + res + "::DakConfig"]
777 return default_config
779 def which_apt_conf_file ():
780 res = socket.getfqdn()
781 # In case we allow local config files per user, try if one exists
782 if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
783 homedir = os.getenv("HOME")
784 confpath = os.path.join(homedir, "/etc/dak.conf")
785 if os.path.exists(confpath):
786 apt_pkg.ReadConfigFileISC(Cnf,default_config)
788 if Cnf.get("Config::" + res + "::AptConfig"):
789 return Cnf["Config::" + res + "::AptConfig"]
791 return default_apt_config
793 def which_alias_file():
794 hostname = socket.getfqdn()
795 aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
796 if os.path.exists(aliasfn):
801 ################################################################################
803 def TemplateSubst(subst_map, filename):
804 """ Perform a substition of template """
805 templatefile = open_file(filename)
806 template = templatefile.read()
807 for k, v in subst_map.iteritems():
808 template = template.replace(k, str(v))
812 ################################################################################
814 def fubar(msg, exit_code=1):
815 sys.stderr.write("E: %s\n" % (msg))
819 sys.stderr.write("W: %s\n" % (msg))
821 ################################################################################
823 # Returns the user name with a laughable attempt at rfc822 conformancy
824 # (read: removing stray periods).
826 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '')
829 return pwd.getpwuid(os.getuid())[0]
831 ################################################################################
841 return ("%d%s" % (c, t))
843 ################################################################################
845 def cc_fix_changes (changes):
846 o = changes.get("architecture", "")
848 del changes["architecture"]
849 changes["architecture"] = {}
851 changes["architecture"][j] = 1
853 def changes_compare (a, b):
854 """ Sort by source name, source version, 'have source', and then by filename """
856 a_changes = parse_changes(a)
861 b_changes = parse_changes(b)
865 cc_fix_changes (a_changes)
866 cc_fix_changes (b_changes)
868 # Sort by source name
869 a_source = a_changes.get("source")
870 b_source = b_changes.get("source")
871 q = cmp (a_source, b_source)
875 # Sort by source version
876 a_version = a_changes.get("version", "0")
877 b_version = b_changes.get("version", "0")
878 q = apt_pkg.version_compare(a_version, b_version)
882 # Sort by 'have source'
883 a_has_source = a_changes["architecture"].get("source")
884 b_has_source = b_changes["architecture"].get("source")
885 if a_has_source and not b_has_source:
887 elif b_has_source and not a_has_source:
890 # Fall back to sort by filename
893 ################################################################################
895 def find_next_free (dest, too_many=100):
898 while os.path.exists(dest) and extra < too_many:
899 dest = orig_dest + '.' + repr(extra)
901 if extra >= too_many:
902 raise NoFreeFilenameError
905 ################################################################################
907 def result_join (original, sep = '\t'):
909 for i in xrange(len(original)):
910 if original[i] == None:
911 resultlist.append("")
913 resultlist.append(original[i])
914 return sep.join(resultlist)
916 ################################################################################
918 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
920 for line in str.split('\n'):
922 if line or include_blank_lines:
923 out += "%s%s\n" % (prefix, line)
924 # Strip trailing new line
929 ################################################################################
931 def validate_changes_file_arg(filename, require_changes=1):
933 'filename' is either a .changes or .dak file. If 'filename' is a
934 .dak file, it's changed to be the corresponding .changes file. The
935 function then checks if the .changes file a) exists and b) is
936 readable and returns the .changes filename if so. If there's a
937 problem, the next action depends on the option 'require_changes'
940 - If 'require_changes' == -1, errors are ignored and the .changes
941 filename is returned.
942 - If 'require_changes' == 0, a warning is given and 'None' is returned.
943 - If 'require_changes' == 1, a fatal error is raised.
948 orig_filename = filename
949 if filename.endswith(".dak"):
950 filename = filename[:-4]+".changes"
952 if not filename.endswith(".changes"):
953 error = "invalid file type; not a changes file"
955 if not os.access(filename,os.R_OK):
956 if os.path.exists(filename):
957 error = "permission denied"
959 error = "file not found"
962 if require_changes == 1:
963 fubar("%s: %s." % (orig_filename, error))
964 elif require_changes == 0:
965 warn("Skipping %s - %s" % (orig_filename, error))
967 else: # We only care about the .dak file
972 ################################################################################
975 return (arch != "source" and arch != "all")
977 ################################################################################
979 def join_with_commas_and(list):
980 if len(list) == 0: return "nothing"
981 if len(list) == 1: return list[0]
982 return ", ".join(list[:-1]) + " and " + list[-1]
984 ################################################################################
989 (pkg, version, constraint) = atom
991 pp_dep = "%s (%s %s)" % (pkg, constraint, version)
994 pp_deps.append(pp_dep)
995 return " |".join(pp_deps)
997 ################################################################################
1002 ################################################################################
1004 def parse_args(Options):
1005 """ Handle -a, -c and -s arguments; returns them as SQL constraints """
1006 # XXX: This should go away and everything which calls it be converted
1007 # to use SQLA properly. For now, we'll just fix it not to use
1008 # the old Pg interface though
1009 session = DBConn().session()
1011 if Options["Suite"]:
1013 for suitename in split_args(Options["Suite"]):
1014 suite = get_suite(suitename, session=session)
1015 if not suite or suite.suite_id is None:
1016 warn("suite '%s' not recognised." % (suite and suite.suite_name or suitename))
1018 suite_ids_list.append(suite.suite_id)
1020 con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
1022 fubar("No valid suite given.")
1027 if Options["Component"]:
1028 component_ids_list = []
1029 for componentname in split_args(Options["Component"]):
1030 component = get_component(componentname, session=session)
1031 if component is None:
1032 warn("component '%s' not recognised." % (componentname))
1034 component_ids_list.append(component.component_id)
1035 if component_ids_list:
1036 con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
1038 fubar("No valid component given.")
1042 # Process architecture
1043 con_architectures = ""
1045 if Options["Architecture"]:
1047 for archname in split_args(Options["Architecture"]):
1048 if archname == "source":
1051 arch = get_architecture(archname, session=session)
1053 warn("architecture '%s' not recognised." % (archname))
1055 arch_ids_list.append(arch.arch_id)
1057 con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
1059 if not check_source:
1060 fubar("No valid architecture given.")
1064 return (con_suites, con_architectures, con_components, check_source)
1066 ################################################################################
1068 def arch_compare_sw (a, b):
1070 Function for use in sorting lists of architectures.
1072 Sorts normally except that 'source' dominates all others.
1075 if a == "source" and b == "source":
1084 ################################################################################
1086 def split_args (s, dwim=1):
1088 Split command line arguments which can be separated by either commas
1089 or whitespace. If dwim is set, it will complain about string ending
1090 in comma since this usually means someone did 'dak ls -a i386, m68k
1091 foo' or something and the inevitable confusion resulting from 'm68k'
1092 being treated as an argument is undesirable.
1095 if s.find(",") == -1:
1098 if s[-1:] == "," and dwim:
1099 fubar("split_args: found trailing comma, spurious space maybe?")
1102 ################################################################################
1104 def gpgv_get_status_output(cmd, status_read, status_write):
1106 Our very own version of commands.getouputstatus(), hacked to support
1110 cmd = ['/bin/sh', '-c', cmd]
1111 p2cread, p2cwrite = os.pipe()
1112 c2pread, c2pwrite = os.pipe()
1113 errout, errin = os.pipe()
1123 for i in range(3, 256):
1124 if i != status_write:
1130 os.execvp(cmd[0], cmd)
1136 os.dup2(c2pread, c2pwrite)
1137 os.dup2(errout, errin)
1139 output = status = ""
1141 i, o, e = select.select([c2pwrite, errin, status_read], [], [])
1144 r = os.read(fd, 8196)
1146 more_data.append(fd)
1147 if fd == c2pwrite or fd == errin:
1149 elif fd == status_read:
1152 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd))
1154 pid, exit_status = os.waitpid(pid, 0)
1156 os.close(status_write)
1157 os.close(status_read)
1167 return output, status, exit_status
1169 ################################################################################
1171 def process_gpgv_output(status):
1172 # Process the status-fd output
1175 for line in status.split('\n'):
1179 split = line.split()
1181 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
1183 (gnupg, keyword) = split[:2]
1184 if gnupg != "[GNUPG:]":
1185 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
1188 if keywords.has_key(keyword) and keyword not in [ "NODATA", "SIGEXPIRED", "KEYEXPIRED" ]:
1189 internal_error += "found duplicate status token ('%s').\n" % (keyword)
1192 keywords[keyword] = args
1194 return (keywords, internal_error)
1196 ################################################################################
1198 def retrieve_key (filename, keyserver=None, keyring=None):
1200 Retrieve the key that signed 'filename' from 'keyserver' and
1201 add it to 'keyring'. Returns nothing on success, or an error message
1205 # Defaults for keyserver and keyring
1207 keyserver = Cnf["Dinstall::KeyServer"]
1209 keyring = get_primary_keyring_path()
1211 # Ensure the filename contains no shell meta-characters or other badness
1212 if not re_taint_free.match(filename):
1213 return "%s: tainted filename" % (filename)
1215 # Invoke gpgv on the file
1216 status_read, status_write = os.pipe()
1217 cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
1218 (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
1220 # Process the status-fd output
1221 (keywords, internal_error) = process_gpgv_output(status)
1223 return internal_error
1225 if not keywords.has_key("NO_PUBKEY"):
1226 return "didn't find expected NO_PUBKEY in gpgv status-fd output"
1228 fingerprint = keywords["NO_PUBKEY"][0]
1229 # XXX - gpg sucks. You can't use --secret-keyring=/dev/null as
1230 # it'll try to create a lockfile in /dev. A better solution might
1231 # be a tempfile or something.
1232 cmd = "gpg --no-default-keyring --secret-keyring=%s --no-options" \
1233 % (Cnf["Dinstall::SigningKeyring"])
1234 cmd += " --keyring %s --keyserver %s --recv-key %s" \
1235 % (keyring, keyserver, fingerprint)
1236 (result, output) = commands.getstatusoutput(cmd)
1238 return "'%s' failed with exit code %s" % (cmd, result)
1242 ################################################################################
1244 def gpg_keyring_args(keyrings=None):
1246 keyrings = get_active_keyring_paths()
1248 return " ".join(["--keyring %s" % x for x in keyrings])
1250 ################################################################################
1252 def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None, session=None):
1254 Check the signature of a file and return the fingerprint if the
1255 signature is valid or 'None' if it's not. The first argument is the
1256 filename whose signature should be checked. The second argument is a
1257 reject function and is called when an error is found. The reject()
1258 function must allow for two arguments: the first is the error message,
1259 the second is an optional prefix string. It's possible for reject()
1260 to be called more than once during an invocation of check_signature().
1261 The third argument is optional and is the name of the files the
1262 detached signature applies to. The fourth argument is optional and is
1263 a *list* of keyrings to use. 'autofetch' can either be None, True or
1264 False. If None, the default behaviour specified in the config will be
1270 # Ensure the filename contains no shell meta-characters or other badness
1271 if not re_taint_free.match(sig_filename):
1272 rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
1273 return (None, rejects)
1275 if data_filename and not re_taint_free.match(data_filename):
1276 rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
1277 return (None, rejects)
1280 keyrings = [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).all() ]
1282 # Autofetch the signing key if that's enabled
1283 if autofetch == None:
1284 autofetch = Cnf.get("Dinstall::KeyAutoFetch")
1286 error_msg = retrieve_key(sig_filename)
1288 rejects.append(error_msg)
1289 return (None, rejects)
1291 # Build the command line
1292 status_read, status_write = os.pipe()
1293 cmd = "gpgv --status-fd %s %s %s %s" % (
1294 status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
1296 # Invoke gpgv on the file
1297 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
1299 # Process the status-fd output
1300 (keywords, internal_error) = process_gpgv_output(status)
1302 # If we failed to parse the status-fd output, let's just whine and bail now
1304 rejects.append("internal error while performing signature check on %s." % (sig_filename))
1305 rejects.append(internal_error, "")
1306 rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
1307 return (None, rejects)
1309 # Now check for obviously bad things in the processed output
1310 if keywords.has_key("KEYREVOKED"):
1311 rejects.append("The key used to sign %s has been revoked." % (sig_filename))
1312 if keywords.has_key("BADSIG"):
1313 rejects.append("bad signature on %s." % (sig_filename))
1314 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
1315 rejects.append("failed to check signature on %s." % (sig_filename))
1316 if keywords.has_key("NO_PUBKEY"):
1317 args = keywords["NO_PUBKEY"]
1320 rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
1321 if keywords.has_key("BADARMOR"):
1322 rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))
1323 if keywords.has_key("NODATA"):
1324 rejects.append("no signature found in %s." % (sig_filename))
1325 if keywords.has_key("EXPKEYSIG"):
1326 args = keywords["EXPKEYSIG"]
1329 rejects.append("Signature made by expired key 0x%s" % (key))
1330 if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
1331 args = keywords["KEYEXPIRED"]
1335 if timestamp.count("T") == 0:
1337 expiredate = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
1339 expiredate = "unknown (%s)" % (timestamp)
1341 expiredate = timestamp
1342 rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
1344 if len(rejects) > 0:
1345 return (None, rejects)
1347 # Next check gpgv exited with a zero return code
1349 rejects.append("gpgv failed while checking %s." % (sig_filename))
1351 rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "))
1353 rejects.append(prefix_multi_line_string(output, " [GPG output:] "))
1354 return (None, rejects)
1356 # Sanity check the good stuff we expect
1357 if not keywords.has_key("VALIDSIG"):
1358 rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
1360 args = keywords["VALIDSIG"]
1362 rejects.append("internal error while checking signature on %s." % (sig_filename))
1364 fingerprint = args[0]
1365 if not keywords.has_key("GOODSIG"):
1366 rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
1367 if not keywords.has_key("SIG_ID"):
1368 rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
1370 # Finally ensure there's not something we don't recognise
1371 known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
1372 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
1373 NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="")
1375 for keyword in keywords.keys():
1376 if not known_keywords.has_key(keyword):
1377 rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
1379 if len(rejects) > 0:
1380 return (None, rejects)
1382 return (fingerprint, [])
1384 ################################################################################
1386 def gpg_get_key_addresses(fingerprint):
1387 """retreive email addresses from gpg key uids for a given fingerprint"""
1388 addresses = key_uid_email_cache.get(fingerprint)
1389 if addresses != None:
1392 cmd = "gpg --no-default-keyring %s --fingerprint %s" \
1393 % (gpg_keyring_args(), fingerprint)
1394 (result, output) = commands.getstatusoutput(cmd)
1396 for l in output.split('\n'):
1397 m = re_gpg_uid.match(l)
1399 addresses.append(m.group(1))
1400 key_uid_email_cache[fingerprint] = addresses
1403 ################################################################################
1405 def clean_symlink (src, dest, root):
1407 Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
1410 src = src.replace(root, '', 1)
1411 dest = dest.replace(root, '', 1)
1412 dest = os.path.dirname(dest)
1413 new_src = '../' * len(dest.split('/'))
1414 return new_src + src
1416 ################################################################################
1418 def temp_filename(directory=None, prefix="dak", suffix=""):
1420 Return a secure and unique filename by pre-creating it.
1421 If 'directory' is non-null, it will be the directory the file is pre-created in.
1422 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1423 If 'suffix' is non-null, the filename will end with it.
1425 Returns a pair (fd, name).
1428 return tempfile.mkstemp(suffix, prefix, directory)
1430 ################################################################################
1432 def temp_dirname(parent=None, prefix="dak", suffix=""):
1434 Return a secure and unique directory by pre-creating it.
1435 If 'parent' is non-null, it will be the directory the directory is pre-created in.
1436 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1437 If 'suffix' is non-null, the filename will end with it.
1439 Returns a pathname to the new directory
1442 return tempfile.mkdtemp(suffix, prefix, parent)
1444 ################################################################################
1446 def is_email_alias(email):
1447 """ checks if the user part of the email is listed in the alias file """
1449 if alias_cache == None:
1450 aliasfn = which_alias_file()
1453 for l in open(aliasfn):
1454 alias_cache.add(l.split(':')[0])
1455 uid = email.split('@')[0]
1456 return uid in alias_cache
1458 ################################################################################
1460 def get_changes_files(from_dir):
1462 Takes a directory and lists all .changes files in it (as well as chdir'ing
1463 to the directory; this is due to broken behaviour on the part of p-u/p-a
1464 when you're not in the right place)
1466 Returns a list of filenames
1469 # Much of the rest of p-u/p-a depends on being in the right place
1471 changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')]
1472 except OSError as e:
1473 fubar("Failed to read list from directory %s (%s)" % (from_dir, e))
1475 return changes_files
1477 ################################################################################
1481 Cnf = apt_pkg.Configuration()
1482 if not os.getenv("DAK_TEST"):
1483 apt_pkg.read_config_file_isc(Cnf,default_config)
1485 if which_conf_file() != default_config:
1486 apt_pkg.read_config_file_isc(Cnf,which_conf_file())
1488 ################################################################################
1490 def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"):
1492 Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm
1493 Well, actually it parsed a local copy, but let's document the source
1496 returns a dict associating source package name with a list of open wnpp
1497 bugs (Yes, there might be more than one)
1503 lines = f.readlines()
1504 except IOError as e:
1505 print "Warning: Couldn't open %s; don't know about WNPP bugs, so won't close any." % file
1510 splited_line = line.split(": ", 1)
1511 if len(splited_line) > 1:
1512 wnpp[splited_line[0]] = splited_line[1].split("|")
1514 for source in wnpp.keys():
1516 for wnpp_bug in wnpp[source]:
1517 bug_no = re.search("(\d)+", wnpp_bug).group()
1523 ################################################################################
1525 def get_packages_from_ftp(root, suite, component, architecture):
1527 Returns an object containing apt_pkg-parseable data collected by
1528 aggregating Packages.gz files gathered for each architecture.
1531 @param root: path to ftp archive root directory
1534 @param suite: suite to extract files from
1536 @type component: string
1537 @param component: component to extract files from
1539 @type architecture: string
1540 @param architecture: architecture to extract files from
1543 @return: apt_pkg class containing package data
1545 filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture)
1546 (fd, temp_file) = temp_filename()
1547 (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_file))
1549 fubar("Gunzip invocation failed!\n%s\n" % (output), result)
1550 filename = "%s/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % (root, suite, component, architecture)
1551 if os.path.exists(filename):
1552 (result, output) = commands.getstatusoutput("gunzip -c %s >> %s" % (filename, temp_file))
1554 fubar("Gunzip invocation failed!\n%s\n" % (output), result)
1555 packages = open_file(temp_file)
1556 Packages = apt_pkg.ParseTagFile(packages)
1557 os.unlink(temp_file)
1560 ################################################################################
1562 def deb_extract_control(fh):
1563 """extract DEBIAN/control from a binary package"""
1564 return apt_inst.DebFile(fh).control.extractdata("control")
1566 ################################################################################
1568 def mail_addresses_for_upload(maintainer, changed_by, fingerprint):
1569 """mail addresses to contact for an upload
1571 @type maintainer: str
1572 @param maintainer: Maintainer field of the .changes file
1574 @type changed_by: str
1575 @param changed_by: Changed-By field of the .changes file
1577 @type fingerprint: str
1578 @param fingerprint: fingerprint of the key used to sign the upload
1581 @return: list of RFC 2047-encoded mail addresses to contact regarding
1584 addresses = [maintainer]
1585 if changed_by != maintainer:
1586 addresses.append(changed_by)
1588 fpr_addresses = gpg_get_key_addresses(fingerprint)
1589 if len(fpr_addresses) > 0 and fix_maintainer(changed_by)[3] not in fpr_addresses and fix_maintainer(maintainer)[3] not in fpr_addresses:
1590 addresses.append(fpr_addresses[0])
1592 encoded_addresses = [ fix_maintainer(e)[1] for e in addresses ]
1593 return encoded_addresses
1595 ################################################################################
1597 def call_editor(text="", suffix=".txt"):
1598 """run editor and return the result as a string
1601 @param text: initial text
1604 @param suffix: extension for temporary file
1607 @return: string with the edited text
1609 editor = os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi'))
1610 tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
1614 subprocess.check_call([editor, tmp.name])
1615 return open(tmp.name, 'r').read()
1619 ################################################################################
1621 def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
1622 dbsuite = get_suite(suite, session)
1627 all_arches = set(arches)
1629 all_arches = set([x.arch_string for x in get_suite_architectures(suite)])
1630 all_arches -= set(["source", "all"])
1631 metakey_d = get_or_set_metadatakey("Depends", session)
1632 metakey_p = get_or_set_metadatakey("Provides", session)
1634 'suite_id': dbsuite.suite_id,
1635 'metakey_d_id': metakey_d.key_id,
1636 'metakey_p_id': metakey_p.key_id,
1638 for architecture in all_arches | set(['all']):
1641 virtual_packages = {}
1642 params['arch_id'] = get_architecture(architecture, session).arch_id
1645 SELECT b.id, b.package, s.source, c.name as component,
1646 (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
1647 (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
1649 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
1650 JOIN source s ON b.source = s.id
1651 JOIN files f ON b.file = f.id
1652 JOIN location l ON f.location = l.id
1653 JOIN component c ON l.component = c.id
1654 WHERE b.architecture = :arch_id'''
1655 query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
1656 from_statement(statement).params(params)
1657 for binary_id, package, source, component, depends, provides in query:
1658 sources[package] = source
1659 p2c[package] = component
1660 if depends is not None:
1661 deps[package] = depends
1662 # Maintain a counter for each virtual package. If a
1663 # Provides: exists, set the counter to 0 and count all
1664 # provides by a package not in the list for removal.
1665 # If the counter stays 0 at the end, we know that only
1666 # the to-be-removed packages provided this virtual
1668 if provides is not None:
1669 for virtual_pkg in provides.split(","):
1670 virtual_pkg = virtual_pkg.strip()
1671 if virtual_pkg == package: continue
1672 if not virtual_packages.has_key(virtual_pkg):
1673 virtual_packages[virtual_pkg] = 0
1674 if package not in removals:
1675 virtual_packages[virtual_pkg] += 1
1677 # If a virtual package is only provided by the to-be-removed
1678 # packages, treat the virtual package as to-be-removed too.
1679 for virtual_pkg in virtual_packages.keys():
1680 if virtual_packages[virtual_pkg] == 0:
1681 removals.append(virtual_pkg)
1683 # Check binary dependencies (Depends)
1684 for package in deps.keys():
1685 if package in removals: continue
1688 parsed_dep += apt_pkg.ParseDepends(deps[package])
1689 except ValueError as e:
1690 print "Error for package %s: %s" % (package, e)
1691 for dep in parsed_dep:
1692 # Check for partial breakage. If a package has a ORed
1693 # dependency, there is only a dependency problem if all
1694 # packages in the ORed depends will be removed.
1696 for dep_package, _, _ in dep:
1697 if dep_package in removals:
1699 if unsat == len(dep):
1700 component = p2c[package]
1701 source = sources[package]
1702 if component != "main":
1703 source = "%s/%s" % (source, component)
1704 all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture)
1709 print " - broken Depends:"
1711 print "# Broken Depends:"
1712 for source, bindict in sorted(all_broken.items()):
1714 for binary, arches in sorted(bindict.items()):
1715 if arches == all_arches or 'all' in arches:
1716 lines.append(binary)
1718 lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
1720 print ' %s: %s' % (source, lines[0])
1722 print '%s: %s' % (source, lines[0])
1723 for line in lines[1:]:
1725 print ' ' + ' ' * (len(source) + 2) + line
1727 print ' ' * (len(source) + 2) + line
1731 # Check source dependencies (Build-Depends and Build-Depends-Indep)
1733 metakey_bd = get_or_set_metadatakey("Build-Depends", session)
1734 metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
1736 'suite_id': dbsuite.suite_id,
1737 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
1740 SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep
1742 JOIN source_metadata sm ON s.id = sm.src_id
1744 (SELECT source FROM src_associations
1745 WHERE suite = :suite_id)
1746 AND sm.key_id in :metakey_ids
1747 GROUP BY s.id, s.source'''
1748 query = session.query('id', 'source', 'build_dep').from_statement(statement). \
1750 for source_id, source, build_dep in query:
1751 if source in removals: continue
1753 if build_dep is not None:
1754 # Remove [arch] information since we want to see breakage on all arches
1755 build_dep = re_build_dep_arch.sub("", build_dep)
1757 parsed_dep += apt_pkg.ParseDepends(build_dep)
1758 except ValueError as e:
1759 print "Error for source %s: %s" % (source, e)
1760 for dep in parsed_dep:
1762 for dep_package, _, _ in dep:
1763 if dep_package in removals:
1765 if unsat == len(dep):
1766 component = DBSource.get(source_id, session).get_component_name()
1767 if component != "main":
1768 source = "%s/%s" % (source, component)
1769 all_broken.setdefault(source, set()).add(pp_deps(dep))
1774 print " - broken Build-Depends:"
1776 print "# Broken Build-Depends:"
1777 for source, bdeps in sorted(all_broken.items()):
1778 bdeps = sorted(bdeps)
1780 print ' %s: %s' % (source, bdeps[0])
1782 print '%s: %s' % (source, bdeps[0])
1783 for bdep in bdeps[1:]:
1785 print ' ' + ' ' * (len(source) + 2) + bdep
1787 print ' ' * (len(source) + 2) + bdep