2 # vim:set et ts=4 sw=4:
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
8 @license: GNU General Public License version 2 or later
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
41 import email as modemail
44 from dbconn import DBConn, get_architecture, get_component, get_suite, \
45 get_override_type, Keyring, session_wrapper, \
46 get_active_keyring_paths, get_primary_keyring_path, \
47 get_suite_architectures, get_or_set_metadatakey, DBSource
48 from sqlalchemy import desc
49 from dak_exceptions import *
50 from gpg import SignedFile
51 from textutils import fix_maintainer
52 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
53 re_multi_line_field, re_srchasver, re_taint_free, \
54 re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \
55 re_is_orig_source, re_build_dep_arch
57 from formats import parse_format, validate_changes_format
58 from srcformats import get_format_from_string
59 from collections import defaultdict
61 ################################################################################
63 default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
64 default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used
66 alias_cache = None #: Cache for email alias checks
67 key_uid_email_cache = {} #: Cache for email addresses from gpg key uids
69 # (hashname, function, earliest_changes_version)
70 known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
71 ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
73 # Monkeypatch commands.getstatusoutput as it may not return the correct exit
74 # code in lenny's Python. This also affects commands.getoutput and
76 def dak_getstatusoutput(cmd):
77 pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
78 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
80 output = pipe.stdout.read()
84 if output[-1:] == '\n':
92 commands.getstatusoutput = dak_getstatusoutput
94 ################################################################################
97 """ Escape html chars """
98 return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
100 ################################################################################
102 def open_file(filename, mode='r'):
104 Open C{file}, return fileobject.
106 @type filename: string
107 @param filename: path/filename to open
110 @param mode: open mode
113 @return: open fileobject
115 @raise CantOpenError: If IOError is raised by open, reraise it as CantOpenError.
119 f = open(filename, mode)
121 raise CantOpenError(filename)
124 ################################################################################
126 def our_raw_input(prompt=""):
130 sys.stdout.write(prompt)
139 sys.stderr.write("\nUser interrupt (^D).\n")
142 ################################################################################
144 def extract_component_from_section(section, session=None):
147 if section.find('/') != -1:
148 component = section.split('/')[0]
150 # Expand default component
152 comp = get_component(section, session)
156 component = comp.component_name
158 return (section, component)
160 ################################################################################
162 def parse_deb822(armored_contents, signing_rules=0, keyrings=None, session=None):
163 require_signature = True
166 require_signature = False
168 signed_file = SignedFile(armored_contents, keyrings=keyrings, require_signature=require_signature)
169 contents = signed_file.contents
174 # Split the lines in the input, keeping the linebreaks.
175 lines = contents.splitlines(True)
178 raise ParseChangesError("[Empty changes file]")
180 # Reindex by line number so we can easily verify the format of
186 indexed_lines[index] = line[:-1]
188 num_of_lines = len(indexed_lines.keys())
191 while index < num_of_lines:
193 line = indexed_lines[index]
194 if line == "" and signing_rules == 1:
195 if index != num_of_lines:
196 raise InvalidDscError(index)
198 slf = re_single_line_field.match(line)
200 field = slf.groups()[0].lower()
201 changes[field] = slf.groups()[1]
205 changes[field] += '\n'
207 mlf = re_multi_line_field.match(line)
210 raise ParseChangesError("'%s'\n [Multi-line field continuing on from nothing?]" % (line))
211 if first == 1 and changes[field] != "":
212 changes[field] += '\n'
214 changes[field] += mlf.groups()[0] + '\n'
218 changes["filecontents"] = armored_contents
220 if changes.has_key("source"):
221 # Strip the source version in brackets from the source field,
222 # put it in the "source-version" field instead.
223 srcver = re_srchasver.search(changes["source"])
225 changes["source"] = srcver.group(1)
226 changes["source-version"] = srcver.group(2)
229 raise ParseChangesError(error)
233 ################################################################################
235 def parse_changes(filename, signing_rules=0, dsc_file=0, keyrings=None):
237 Parses a changes file and returns a dictionary where each field is a
238 key. The mandatory first argument is the filename of the .changes
241 signing_rules is an optional argument:
243 - If signing_rules == -1, no signature is required.
244 - If signing_rules == 0 (the default), a signature is required.
245 - If signing_rules == 1, it turns on the same strict format checking
248 The rules for (signing_rules == 1)-mode are:
250 - The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
251 followed by any PGP header data and must end with a blank line.
253 - The data section must end with a blank line and must be followed by
254 "-----BEGIN PGP SIGNATURE-----".
257 changes_in = open_file(filename)
258 content = changes_in.read()
261 unicode(content, 'utf-8')
263 raise ChangesUnicodeError("Changes file not proper utf-8")
264 changes = parse_deb822(content, signing_rules, keyrings=keyrings)
268 # Finally ensure that everything needed for .changes is there
269 must_keywords = ('Format', 'Date', 'Source', 'Binary', 'Architecture', 'Version',
270 'Distribution', 'Maintainer', 'Description', 'Changes', 'Files')
273 for keyword in must_keywords:
274 if not changes.has_key(keyword.lower()):
275 missingfields.append(keyword)
277 if len(missingfields):
278 raise ParseChangesError("Missing mandantory field(s) in changes file (policy 5.5): %s" % (missingfields))
282 ################################################################################
284 def hash_key(hashname):
285 return '%ssum' % hashname
287 ################################################################################
289 def create_hash(where, files, hashname, hashfunc):
291 create_hash extends the passed files dict with the given hash by
292 iterating over all files on disk and passing them to the hashing
297 for f in files.keys():
299 file_handle = open_file(f)
300 except CantOpenError:
301 rejmsg.append("Could not open file %s for checksumming" % (f))
304 files[f][hash_key(hashname)] = hashfunc(file_handle)
309 ################################################################################
311 def check_hash(where, files, hashname, hashfunc):
313 check_hash checks the given hash in the files dict against the actual
314 files on disk. The hash values need to be present consistently in
315 all file entries. It does not modify its input in any way.
319 for f in files.keys():
323 file_handle = open_file(f)
325 # Check for the hash entry, to not trigger a KeyError.
326 if not files[f].has_key(hash_key(hashname)):
327 rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,
331 # Actually check the hash for correctness.
332 if hashfunc(file_handle) != files[f][hash_key(hashname)]:
333 rejmsg.append("%s: %s check failed in %s" % (f, hashname,
335 except CantOpenError:
336 # TODO: This happens when the file is in the pool.
337 # warn("Cannot open file %s" % f)
344 ################################################################################
346 def check_size(where, files):
348 check_size checks the file sizes in the passed files dict against the
353 for f in files.keys():
356 except OSError as exc:
358 # TODO: This happens when the file is in the pool.
362 actual_size = entry[stat.ST_SIZE]
363 size = int(files[f]["size"])
364 if size != actual_size:
365 rejmsg.append("%s: actual file size (%s) does not match size (%s) in %s"
366 % (f, actual_size, size, where))
369 ################################################################################
371 def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
373 Verify that the files listed in the Files field of the .dsc are
374 those expected given the announced Format.
376 @type dsc_filename: string
377 @param dsc_filename: path of .dsc file
380 @param dsc: the content of the .dsc parsed by C{parse_changes()}
382 @type dsc_files: dict
383 @param dsc_files: the file list returned by C{build_file_list()}
386 @return: all errors detected
390 # Parse the file if needed
392 dsc = parse_changes(dsc_filename, signing_rules=1, dsc_file=1);
394 if dsc_files is None:
395 dsc_files = build_file_list(dsc, is_a_dsc=1)
397 # Ensure .dsc lists proper set of source files according to the format
399 has = defaultdict(lambda: 0)
402 (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')),
403 (r'diff.gz', ('debian_diff',)),
404 (r'tar.gz', ('native_tar_gz', 'native_tar')),
405 (r'debian\.tar\.(gz|bz2|xz)', ('debian_tar',)),
406 (r'orig\.tar\.(gz|bz2|xz)', ('orig_tar',)),
407 (r'tar\.(gz|bz2|xz)', ('native_tar',)),
408 (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
411 for f in dsc_files.keys():
412 m = re_issource.match(f)
414 rejmsg.append("%s: %s in Files field not recognised as source."
418 # Populate 'has' dictionary by resolving keys in lookup table
420 for regex, keys in ftype_lookup:
421 if re.match(regex, m.group(3)):
427 # File does not match anything in lookup table; reject
429 reject("%s: unexpected source file '%s'" % (dsc_filename, f))
431 # Check for multiple files
432 for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'):
433 if has[file_type] > 1:
434 rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type))
436 # Source format specific tests
438 format = get_format_from_string(dsc['format'])
440 '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has)
443 except UnknownFormatError:
444 # Not an error here for now
449 ################################################################################
451 def check_hash_fields(what, manifest):
453 check_hash_fields ensures that there are no checksum fields in the
454 given dict that we do not know about.
458 hashes = map(lambda x: x[0], known_hashes)
459 for field in manifest:
460 if field.startswith("checksums-"):
461 hashname = field.split("-",1)[1]
462 if hashname not in hashes:
463 rejmsg.append("Unsupported checksum field for %s "\
464 "in %s" % (hashname, what))
467 ################################################################################
469 def _ensure_changes_hash(changes, format, version, files, hashname, hashfunc):
470 if format >= version:
471 # The version should contain the specified hash.
474 # Import hashes from the changes
475 rejmsg = parse_checksums(".changes", files, changes, hashname)
479 # We need to calculate the hash because it can't possibly
482 return func(".changes", files, hashname, hashfunc)
484 # We could add the orig which might be in the pool to the files dict to
485 # access the checksums easily.
487 def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
489 ensure_dsc_hashes' task is to ensure that each and every *present* hash
490 in the dsc is correct, i.e. identical to the changes file and if necessary
491 the pool. The latter task is delegated to check_hash.
495 if not dsc.has_key('Checksums-%s' % (hashname,)):
497 # Import hashes from the dsc
498 parse_checksums(".dsc", dsc_files, dsc, hashname)
500 rejmsg.extend(check_hash(".dsc", dsc_files, hashname, hashfunc))
503 ################################################################################
505 def parse_checksums(where, files, manifest, hashname):
507 field = 'checksums-%s' % hashname
508 if not field in manifest:
510 for line in manifest[field].split('\n'):
513 clist = line.strip().split(' ')
515 checksum, size, checkfile = clist
517 rejmsg.append("Cannot parse checksum line [%s]" % (line))
519 if not files.has_key(checkfile):
520 # TODO: check for the file's entry in the original files dict, not
521 # the one modified by (auto)byhand and other weird stuff
522 # rejmsg.append("%s: not present in files but in checksums-%s in %s" %
523 # (file, hashname, where))
525 if not files[checkfile]["size"] == size:
526 rejmsg.append("%s: size differs for files and checksums-%s entry "\
527 "in %s" % (checkfile, hashname, where))
529 files[checkfile][hash_key(hashname)] = checksum
530 for f in files.keys():
531 if not files[f].has_key(hash_key(hashname)):
532 rejmsg.append("%s: no entry in checksums-%s in %s" % (f, hashname, where))
535 ################################################################################
537 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
539 def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
542 # Make sure we have a Files: field to parse...
543 if not changes.has_key(field):
544 raise NoFilesFieldError
546 # Validate .changes Format: field
548 validate_changes_format(parse_format(changes['format']), field)
550 includes_section = (not is_a_dsc) and field == "files"
552 # Parse each entry/line:
553 for i in changes[field].split('\n'):
557 section = priority = ""
560 (md5, size, section, priority, name) = s
562 (md5, size, name) = s
564 raise ParseChangesError(i)
571 (section, component) = extract_component_from_section(section)
573 files[name] = dict(size=size, section=section,
574 priority=priority, component=component)
575 files[name][hashname] = md5
579 ################################################################################
581 # see http://bugs.debian.org/619131
582 def build_package_list(dsc, session = None):
583 if not dsc.has_key("package-list"):
588 for line in dsc["package-list"].split("\n"):
592 fields = line.split()
594 package_type = fields[1]
595 (section, component) = extract_component_from_section(fields[2])
598 # Validate type if we have a session
599 if session and get_override_type(package_type, session) is None:
600 # Maybe just warn and ignore? exit(1) might be a bit hard...
601 utils.fubar("invalid type (%s) in Package-List." % (package_type))
603 if name not in packages or packages[name]["type"] == "dsc":
604 packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[])
608 ################################################################################
610 def send_mail (message, filename=""):
611 """sendmail wrapper, takes _either_ a message string or a file as arguments"""
613 maildir = Cnf.get('Dir::Mail')
615 path = os.path.join(maildir, datetime.datetime.now().isoformat())
616 path = find_next_free(path)
621 # Check whether we're supposed to be sending mail
622 if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]:
625 # If we've been passed a string dump it into a temporary file
627 (fd, filename) = tempfile.mkstemp()
628 os.write (fd, message)
631 if Cnf.has_key("Dinstall::MailWhiteList") and \
632 Cnf["Dinstall::MailWhiteList"] != "":
633 message_in = open_file(filename)
634 message_raw = modemail.message_from_file(message_in)
638 whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
640 for line in whitelist_in:
641 if not re_whitespace_comment.match(line):
642 if re_re_mark.match(line):
643 whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
645 whitelist.append(re.compile(re.escape(line.strip())))
650 fields = ["To", "Bcc", "Cc"]
653 value = message_raw.get(field, None)
656 for item in value.split(","):
657 (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip())
663 if not mail_whitelisted:
664 print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
668 # Doesn't have any mail in whitelist so remove the header
670 del message_raw[field]
672 message_raw.replace_header(field, ', '.join(match))
674 # Change message fields in order if we don't have a To header
675 if not message_raw.has_key("To"):
678 if message_raw.has_key(field):
679 message_raw[fields[-1]] = message_raw[field]
680 del message_raw[field]
683 # Clean up any temporary files
684 # and return, as we removed all recipients.
686 os.unlink (filename);
689 fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0o700);
690 os.write (fd, message_raw.as_string(True));
694 (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
696 raise SendmailFailedError(output)
698 # Clean up any temporary files
702 ################################################################################
704 def poolify (source, component=None):
705 if source[:3] == "lib":
706 return source[:4] + '/' + source + '/'
708 return source[:1] + '/' + source + '/'
710 ################################################################################
712 def move (src, dest, overwrite = 0, perms = 0o664):
713 if os.path.exists(dest) and os.path.isdir(dest):
716 dest_dir = os.path.dirname(dest)
717 if not os.path.exists(dest_dir):
718 umask = os.umask(00000)
719 os.makedirs(dest_dir, 0o2775)
721 #print "Moving %s to %s..." % (src, dest)
722 if os.path.exists(dest) and os.path.isdir(dest):
723 dest += '/' + os.path.basename(src)
724 # Don't overwrite unless forced to
725 if os.path.exists(dest):
727 fubar("Can't move %s to %s - file already exists." % (src, dest))
729 if not os.access(dest, os.W_OK):
730 fubar("Can't move %s to %s - can't write to existing file." % (src, dest))
731 shutil.copy2(src, dest)
732 os.chmod(dest, perms)
735 def copy (src, dest, overwrite = 0, perms = 0o664):
736 if os.path.exists(dest) and os.path.isdir(dest):
739 dest_dir = os.path.dirname(dest)
740 if not os.path.exists(dest_dir):
741 umask = os.umask(00000)
742 os.makedirs(dest_dir, 0o2775)
744 #print "Copying %s to %s..." % (src, dest)
745 if os.path.exists(dest) and os.path.isdir(dest):
746 dest += '/' + os.path.basename(src)
747 # Don't overwrite unless forced to
748 if os.path.exists(dest):
750 raise FileExistsError
752 if not os.access(dest, os.W_OK):
753 raise CantOverwriteError
754 shutil.copy2(src, dest)
755 os.chmod(dest, perms)
757 ################################################################################
760 res = socket.getfqdn()
761 database_hostname = Cnf.get("Config::" + res + "::DatabaseHostname")
762 if database_hostname:
763 return database_hostname
767 def which_conf_file ():
768 if os.getenv('DAK_CONFIG'):
769 return os.getenv('DAK_CONFIG')
771 res = socket.getfqdn()
772 # In case we allow local config files per user, try if one exists
773 if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
774 homedir = os.getenv("HOME")
775 confpath = os.path.join(homedir, "/etc/dak.conf")
776 if os.path.exists(confpath):
777 apt_pkg.ReadConfigFileISC(Cnf,confpath)
779 # We are still in here, so there is no local config file or we do
780 # not allow local files. Do the normal stuff.
781 if Cnf.get("Config::" + res + "::DakConfig"):
782 return Cnf["Config::" + res + "::DakConfig"]
784 return default_config
786 def which_apt_conf_file ():
787 res = socket.getfqdn()
788 # In case we allow local config files per user, try if one exists
789 if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
790 homedir = os.getenv("HOME")
791 confpath = os.path.join(homedir, "/etc/dak.conf")
792 if os.path.exists(confpath):
793 apt_pkg.ReadConfigFileISC(Cnf,default_config)
795 if Cnf.get("Config::" + res + "::AptConfig"):
796 return Cnf["Config::" + res + "::AptConfig"]
798 return default_apt_config
800 def which_alias_file():
801 hostname = socket.getfqdn()
802 aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
803 if os.path.exists(aliasfn):
808 ################################################################################
810 def TemplateSubst(subst_map, filename):
811 """ Perform a substition of template """
812 templatefile = open_file(filename)
813 template = templatefile.read()
814 for k, v in subst_map.iteritems():
815 template = template.replace(k, str(v))
819 ################################################################################
821 def fubar(msg, exit_code=1):
822 sys.stderr.write("E: %s\n" % (msg))
826 sys.stderr.write("W: %s\n" % (msg))
828 ################################################################################
830 # Returns the user name with a laughable attempt at rfc822 conformancy
831 # (read: removing stray periods).
833 return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '')
836 return pwd.getpwuid(os.getuid())[0]
838 ################################################################################
848 return ("%d%s" % (c, t))
850 ################################################################################
852 def cc_fix_changes (changes):
853 o = changes.get("architecture", "")
855 del changes["architecture"]
856 changes["architecture"] = {}
858 changes["architecture"][j] = 1
860 def changes_compare (a, b):
861 """ Sort by source name, source version, 'have source', and then by filename """
863 a_changes = parse_changes(a)
868 b_changes = parse_changes(b)
872 cc_fix_changes (a_changes)
873 cc_fix_changes (b_changes)
875 # Sort by source name
876 a_source = a_changes.get("source")
877 b_source = b_changes.get("source")
878 q = cmp (a_source, b_source)
882 # Sort by source version
883 a_version = a_changes.get("version", "0")
884 b_version = b_changes.get("version", "0")
885 q = apt_pkg.version_compare(a_version, b_version)
889 # Sort by 'have source'
890 a_has_source = a_changes["architecture"].get("source")
891 b_has_source = b_changes["architecture"].get("source")
892 if a_has_source and not b_has_source:
894 elif b_has_source and not a_has_source:
897 # Fall back to sort by filename
900 ################################################################################
902 def find_next_free (dest, too_many=100):
905 while os.path.exists(dest) and extra < too_many:
906 dest = orig_dest + '.' + repr(extra)
908 if extra >= too_many:
909 raise NoFreeFilenameError
912 ################################################################################
914 def result_join (original, sep = '\t'):
916 for i in xrange(len(original)):
917 if original[i] == None:
918 resultlist.append("")
920 resultlist.append(original[i])
921 return sep.join(resultlist)
923 ################################################################################
925 def prefix_multi_line_string(str, prefix, include_blank_lines=0):
927 for line in str.split('\n'):
929 if line or include_blank_lines:
930 out += "%s%s\n" % (prefix, line)
931 # Strip trailing new line
936 ################################################################################
938 def validate_changes_file_arg(filename, require_changes=1):
940 'filename' is either a .changes or .dak file. If 'filename' is a
941 .dak file, it's changed to be the corresponding .changes file. The
942 function then checks if the .changes file a) exists and b) is
943 readable and returns the .changes filename if so. If there's a
944 problem, the next action depends on the option 'require_changes'
947 - If 'require_changes' == -1, errors are ignored and the .changes
948 filename is returned.
949 - If 'require_changes' == 0, a warning is given and 'None' is returned.
950 - If 'require_changes' == 1, a fatal error is raised.
955 orig_filename = filename
956 if filename.endswith(".dak"):
957 filename = filename[:-4]+".changes"
959 if not filename.endswith(".changes"):
960 error = "invalid file type; not a changes file"
962 if not os.access(filename,os.R_OK):
963 if os.path.exists(filename):
964 error = "permission denied"
966 error = "file not found"
969 if require_changes == 1:
970 fubar("%s: %s." % (orig_filename, error))
971 elif require_changes == 0:
972 warn("Skipping %s - %s" % (orig_filename, error))
974 else: # We only care about the .dak file
979 ################################################################################
982 return (arch != "source" and arch != "all")
984 ################################################################################
986 def join_with_commas_and(list):
987 if len(list) == 0: return "nothing"
988 if len(list) == 1: return list[0]
989 return ", ".join(list[:-1]) + " and " + list[-1]
991 ################################################################################
996 (pkg, version, constraint) = atom
998 pp_dep = "%s (%s %s)" % (pkg, constraint, version)
1001 pp_deps.append(pp_dep)
1002 return " |".join(pp_deps)
1004 ################################################################################
1009 ################################################################################
1011 def parse_args(Options):
1012 """ Handle -a, -c and -s arguments; returns them as SQL constraints """
1013 # XXX: This should go away and everything which calls it be converted
1014 # to use SQLA properly. For now, we'll just fix it not to use
1015 # the old Pg interface though
1016 session = DBConn().session()
1018 if Options["Suite"]:
1020 for suitename in split_args(Options["Suite"]):
1021 suite = get_suite(suitename, session=session)
1022 if not suite or suite.suite_id is None:
1023 warn("suite '%s' not recognised." % (suite and suite.suite_name or suitename))
1025 suite_ids_list.append(suite.suite_id)
1027 con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
1029 fubar("No valid suite given.")
1034 if Options["Component"]:
1035 component_ids_list = []
1036 for componentname in split_args(Options["Component"]):
1037 component = get_component(componentname, session=session)
1038 if component is None:
1039 warn("component '%s' not recognised." % (componentname))
1041 component_ids_list.append(component.component_id)
1042 if component_ids_list:
1043 con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
1045 fubar("No valid component given.")
1049 # Process architecture
1050 con_architectures = ""
1052 if Options["Architecture"]:
1054 for archname in split_args(Options["Architecture"]):
1055 if archname == "source":
1058 arch = get_architecture(archname, session=session)
1060 warn("architecture '%s' not recognised." % (archname))
1062 arch_ids_list.append(arch.arch_id)
1064 con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
1066 if not check_source:
1067 fubar("No valid architecture given.")
1071 return (con_suites, con_architectures, con_components, check_source)
1073 ################################################################################
1075 def arch_compare_sw (a, b):
1077 Function for use in sorting lists of architectures.
1079 Sorts normally except that 'source' dominates all others.
1082 if a == "source" and b == "source":
1091 ################################################################################
1093 def split_args (s, dwim=1):
1095 Split command line arguments which can be separated by either commas
1096 or whitespace. If dwim is set, it will complain about string ending
1097 in comma since this usually means someone did 'dak ls -a i386, m68k
1098 foo' or something and the inevitable confusion resulting from 'm68k'
1099 being treated as an argument is undesirable.
1102 if s.find(",") == -1:
1105 if s[-1:] == "," and dwim:
1106 fubar("split_args: found trailing comma, spurious space maybe?")
1109 ################################################################################
1111 def gpgv_get_status_output(cmd, status_read, status_write):
1113 Our very own version of commands.getouputstatus(), hacked to support
1117 cmd = ['/bin/sh', '-c', cmd]
1118 p2cread, p2cwrite = os.pipe()
1119 c2pread, c2pwrite = os.pipe()
1120 errout, errin = os.pipe()
1130 for i in range(3, 256):
1131 if i != status_write:
1137 os.execvp(cmd[0], cmd)
1143 os.dup2(c2pread, c2pwrite)
1144 os.dup2(errout, errin)
1146 output = status = ""
1148 i, o, e = select.select([c2pwrite, errin, status_read], [], [])
1151 r = os.read(fd, 8196)
1153 more_data.append(fd)
1154 if fd == c2pwrite or fd == errin:
1156 elif fd == status_read:
1159 fubar("Unexpected file descriptor [%s] returned from select\n" % (fd))
1161 pid, exit_status = os.waitpid(pid, 0)
1163 os.close(status_write)
1164 os.close(status_read)
1174 return output, status, exit_status
1176 ################################################################################
1178 def process_gpgv_output(status):
1179 # Process the status-fd output
1182 for line in status.split('\n'):
1186 split = line.split()
1188 internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
1190 (gnupg, keyword) = split[:2]
1191 if gnupg != "[GNUPG:]":
1192 internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
1195 if keywords.has_key(keyword) and keyword not in [ "NODATA", "SIGEXPIRED", "KEYEXPIRED" ]:
1196 internal_error += "found duplicate status token ('%s').\n" % (keyword)
1199 keywords[keyword] = args
1201 return (keywords, internal_error)
1203 ################################################################################
1205 def retrieve_key (filename, keyserver=None, keyring=None):
1207 Retrieve the key that signed 'filename' from 'keyserver' and
1208 add it to 'keyring'. Returns nothing on success, or an error message
1212 # Defaults for keyserver and keyring
1214 keyserver = Cnf["Dinstall::KeyServer"]
1216 keyring = get_primary_keyring_path()
1218 # Ensure the filename contains no shell meta-characters or other badness
1219 if not re_taint_free.match(filename):
1220 return "%s: tainted filename" % (filename)
1222 # Invoke gpgv on the file
1223 status_read, status_write = os.pipe()
1224 cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
1225 (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
1227 # Process the status-fd output
1228 (keywords, internal_error) = process_gpgv_output(status)
1230 return internal_error
1232 if not keywords.has_key("NO_PUBKEY"):
1233 return "didn't find expected NO_PUBKEY in gpgv status-fd output"
1235 fingerprint = keywords["NO_PUBKEY"][0]
1236 # XXX - gpg sucks. You can't use --secret-keyring=/dev/null as
1237 # it'll try to create a lockfile in /dev. A better solution might
1238 # be a tempfile or something.
1239 cmd = "gpg --no-default-keyring --secret-keyring=%s --no-options" \
1240 % (Cnf["Dinstall::SigningKeyring"])
1241 cmd += " --keyring %s --keyserver %s --recv-key %s" \
1242 % (keyring, keyserver, fingerprint)
1243 (result, output) = commands.getstatusoutput(cmd)
1245 return "'%s' failed with exit code %s" % (cmd, result)
1249 ################################################################################
1251 def gpg_keyring_args(keyrings=None):
1253 keyrings = get_active_keyring_paths()
1255 return " ".join(["--keyring %s" % x for x in keyrings])
1257 ################################################################################
1259 def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None, session=None):
1261 Check the signature of a file and return the fingerprint if the
1262 signature is valid or 'None' if it's not. The first argument is the
1263 filename whose signature should be checked. The second argument is a
1264 reject function and is called when an error is found. The reject()
1265 function must allow for two arguments: the first is the error message,
1266 the second is an optional prefix string. It's possible for reject()
1267 to be called more than once during an invocation of check_signature().
1268 The third argument is optional and is the name of the files the
1269 detached signature applies to. The fourth argument is optional and is
1270 a *list* of keyrings to use. 'autofetch' can either be None, True or
1271 False. If None, the default behaviour specified in the config will be
1277 # Ensure the filename contains no shell meta-characters or other badness
1278 if not re_taint_free.match(sig_filename):
1279 rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
1280 return (None, rejects)
1282 if data_filename and not re_taint_free.match(data_filename):
1283 rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
1284 return (None, rejects)
1287 keyrings = [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).all() ]
1289 # Autofetch the signing key if that's enabled
1290 if autofetch == None:
1291 autofetch = Cnf.get("Dinstall::KeyAutoFetch")
1293 error_msg = retrieve_key(sig_filename)
1295 rejects.append(error_msg)
1296 return (None, rejects)
1298 # Build the command line
1299 status_read, status_write = os.pipe()
1300 cmd = "gpgv --status-fd %s %s %s %s" % (
1301 status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
1303 # Invoke gpgv on the file
1304 (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
1306 # Process the status-fd output
1307 (keywords, internal_error) = process_gpgv_output(status)
1309 # If we failed to parse the status-fd output, let's just whine and bail now
1311 rejects.append("internal error while performing signature check on %s." % (sig_filename))
1312 rejects.append(internal_error, "")
1313 rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
1314 return (None, rejects)
1316 # Now check for obviously bad things in the processed output
1317 if keywords.has_key("KEYREVOKED"):
1318 rejects.append("The key used to sign %s has been revoked." % (sig_filename))
1319 if keywords.has_key("BADSIG"):
1320 rejects.append("bad signature on %s." % (sig_filename))
1321 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
1322 rejects.append("failed to check signature on %s." % (sig_filename))
1323 if keywords.has_key("NO_PUBKEY"):
1324 args = keywords["NO_PUBKEY"]
1327 rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
1328 if keywords.has_key("BADARMOR"):
1329 rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))
1330 if keywords.has_key("NODATA"):
1331 rejects.append("no signature found in %s." % (sig_filename))
1332 if keywords.has_key("EXPKEYSIG"):
1333 args = keywords["EXPKEYSIG"]
1336 rejects.append("Signature made by expired key 0x%s" % (key))
1337 if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
1338 args = keywords["KEYEXPIRED"]
1342 if timestamp.count("T") == 0:
1344 expiredate = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
1346 expiredate = "unknown (%s)" % (timestamp)
1348 expiredate = timestamp
1349 rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))
1351 if len(rejects) > 0:
1352 return (None, rejects)
1354 # Next check gpgv exited with a zero return code
1356 rejects.append("gpgv failed while checking %s." % (sig_filename))
1358 rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "))
1360 rejects.append(prefix_multi_line_string(output, " [GPG output:] "))
1361 return (None, rejects)
1363 # Sanity check the good stuff we expect
1364 if not keywords.has_key("VALIDSIG"):
1365 rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
1367 args = keywords["VALIDSIG"]
1369 rejects.append("internal error while checking signature on %s." % (sig_filename))
1371 fingerprint = args[0]
1372 if not keywords.has_key("GOODSIG"):
1373 rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
1374 if not keywords.has_key("SIG_ID"):
1375 rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
1377 # Finally ensure there's not something we don't recognise
1378 known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
1379 SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
1380 NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="")
1382 for keyword in keywords.keys():
1383 if not known_keywords.has_key(keyword):
1384 rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
1386 if len(rejects) > 0:
1387 return (None, rejects)
1389 return (fingerprint, [])
1391 ################################################################################
1393 def gpg_get_key_addresses(fingerprint):
1394 """retreive email addresses from gpg key uids for a given fingerprint"""
1395 addresses = key_uid_email_cache.get(fingerprint)
1396 if addresses != None:
1399 cmd = "gpg --no-default-keyring %s --fingerprint %s" \
1400 % (gpg_keyring_args(), fingerprint)
1401 (result, output) = commands.getstatusoutput(cmd)
1403 for l in output.split('\n'):
1404 m = re_gpg_uid.match(l)
1406 addresses.append(m.group(1))
1407 key_uid_email_cache[fingerprint] = addresses
1410 ################################################################################
1412 def clean_symlink (src, dest, root):
1414 Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
1417 src = src.replace(root, '', 1)
1418 dest = dest.replace(root, '', 1)
1419 dest = os.path.dirname(dest)
1420 new_src = '../' * len(dest.split('/'))
1421 return new_src + src
1423 ################################################################################
1425 def temp_filename(directory=None, prefix="dak", suffix=""):
1427 Return a secure and unique filename by pre-creating it.
1428 If 'directory' is non-null, it will be the directory the file is pre-created in.
1429 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1430 If 'suffix' is non-null, the filename will end with it.
1432 Returns a pair (fd, name).
1435 return tempfile.mkstemp(suffix, prefix, directory)
1437 ################################################################################
1439 def temp_dirname(parent=None, prefix="dak", suffix=""):
1441 Return a secure and unique directory by pre-creating it.
1442 If 'parent' is non-null, it will be the directory the directory is pre-created in.
1443 If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
1444 If 'suffix' is non-null, the filename will end with it.
1446 Returns a pathname to the new directory
1449 return tempfile.mkdtemp(suffix, prefix, parent)
1451 ################################################################################
1453 def is_email_alias(email):
1454 """ checks if the user part of the email is listed in the alias file """
1456 if alias_cache == None:
1457 aliasfn = which_alias_file()
1460 for l in open(aliasfn):
1461 alias_cache.add(l.split(':')[0])
1462 uid = email.split('@')[0]
1463 return uid in alias_cache
1465 ################################################################################
1467 def get_changes_files(from_dir):
1469 Takes a directory and lists all .changes files in it (as well as chdir'ing
1470 to the directory; this is due to broken behaviour on the part of p-u/p-a
1471 when you're not in the right place)
1473 Returns a list of filenames
1476 # Much of the rest of p-u/p-a depends on being in the right place
1478 changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')]
1479 except OSError as e:
1480 fubar("Failed to read list from directory %s (%s)" % (from_dir, e))
1482 return changes_files
1484 ################################################################################
1488 Cnf = apt_pkg.Configuration()
1489 if not os.getenv("DAK_TEST"):
1490 apt_pkg.read_config_file_isc(Cnf,default_config)
1492 if which_conf_file() != default_config:
1493 apt_pkg.read_config_file_isc(Cnf,which_conf_file())
1495 ################################################################################
1497 def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"):
1499 Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm
1500 Well, actually it parsed a local copy, but let's document the source
1503 returns a dict associating source package name with a list of open wnpp
1504 bugs (Yes, there might be more than one)
1510 lines = f.readlines()
1511 except IOError as e:
1512 print "Warning: Couldn't open %s; don't know about WNPP bugs, so won't close any." % file
1517 splited_line = line.split(": ", 1)
1518 if len(splited_line) > 1:
1519 wnpp[splited_line[0]] = splited_line[1].split("|")
1521 for source in wnpp.keys():
1523 for wnpp_bug in wnpp[source]:
1524 bug_no = re.search("(\d)+", wnpp_bug).group()
1530 ################################################################################
1532 def get_packages_from_ftp(root, suite, component, architecture):
1534 Returns an object containing apt_pkg-parseable data collected by
1535 aggregating Packages.gz files gathered for each architecture.
1538 @param root: path to ftp archive root directory
1541 @param suite: suite to extract files from
1543 @type component: string
1544 @param component: component to extract files from
1546 @type architecture: string
1547 @param architecture: architecture to extract files from
1550 @return: apt_pkg class containing package data
1552 filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture)
1553 (fd, temp_file) = temp_filename()
1554 (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_file))
1556 fubar("Gunzip invocation failed!\n%s\n" % (output), result)
1557 filename = "%s/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % (root, suite, component, architecture)
1558 if os.path.exists(filename):
1559 (result, output) = commands.getstatusoutput("gunzip -c %s >> %s" % (filename, temp_file))
1561 fubar("Gunzip invocation failed!\n%s\n" % (output), result)
1562 packages = open_file(temp_file)
1563 Packages = apt_pkg.ParseTagFile(packages)
1564 os.unlink(temp_file)
1567 ################################################################################
1569 def deb_extract_control(fh):
1570 """extract DEBIAN/control from a binary package"""
1571 return apt_inst.DebFile(fh).control.extractdata("control")
1573 ################################################################################
1575 def mail_addresses_for_upload(maintainer, changed_by, fingerprint):
1576 """mail addresses to contact for an upload
1578 @type maintainer: str
1579 @param maintainer: Maintainer field of the .changes file
1581 @type changed_by: str
1582 @param changed_by: Changed-By field of the .changes file
1584 @type fingerprint: str
1585 @param fingerprint: fingerprint of the key used to sign the upload
1588 @return: list of RFC 2047-encoded mail addresses to contact regarding
1591 addresses = [maintainer]
1592 if changed_by != maintainer:
1593 addresses.append(changed_by)
1595 fpr_addresses = gpg_get_key_addresses(fingerprint)
1596 if len(fpr_addresses) > 0 and fix_maintainer(changed_by)[3] not in fpr_addresses and fix_maintainer(maintainer)[3] not in fpr_addresses:
1597 addresses.append(fpr_addresses[0])
1599 encoded_addresses = [ fix_maintainer(e)[1] for e in addresses ]
1600 return encoded_addresses
1602 ################################################################################
1604 def call_editor(text="", suffix=".txt"):
1605 """run editor and return the result as a string
1608 @param text: initial text
1611 @param suffix: extension for temporary file
1614 @return: string with the edited text
1616 editor = os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi'))
1617 tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
1621 subprocess.check_call([editor, tmp.name])
1622 return open(tmp.name, 'r').read()
1626 ################################################################################
1628 def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
1629 dbsuite = get_suite(suite, session)
1634 all_arches = set(arches)
1636 all_arches = set([x.arch_string for x in get_suite_architectures(suite)])
1637 all_arches -= set(["source", "all"])
1638 metakey_d = get_or_set_metadatakey("Depends", session)
1639 metakey_p = get_or_set_metadatakey("Provides", session)
1641 'suite_id': dbsuite.suite_id,
1642 'metakey_d_id': metakey_d.key_id,
1643 'metakey_p_id': metakey_p.key_id,
1645 for architecture in all_arches | set(['all']):
1648 virtual_packages = {}
1649 params['arch_id'] = get_architecture(architecture, session).arch_id
1652 SELECT b.id, b.package, s.source, c.name as component,
1653 (SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
1654 (SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
1656 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
1657 JOIN source s ON b.source = s.id
1658 JOIN files f ON b.file = f.id
1659 JOIN location l ON f.location = l.id
1660 JOIN component c ON l.component = c.id
1661 WHERE b.architecture = :arch_id'''
1662 query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
1663 from_statement(statement).params(params)
1664 for binary_id, package, source, component, depends, provides in query:
1665 sources[package] = source
1666 p2c[package] = component
1667 if depends is not None:
1668 deps[package] = depends
1669 # Maintain a counter for each virtual package. If a
1670 # Provides: exists, set the counter to 0 and count all
1671 # provides by a package not in the list for removal.
1672 # If the counter stays 0 at the end, we know that only
1673 # the to-be-removed packages provided this virtual
1675 if provides is not None:
1676 for virtual_pkg in provides.split(","):
1677 virtual_pkg = virtual_pkg.strip()
1678 if virtual_pkg == package: continue
1679 if not virtual_packages.has_key(virtual_pkg):
1680 virtual_packages[virtual_pkg] = 0
1681 if package not in removals:
1682 virtual_packages[virtual_pkg] += 1
1684 # If a virtual package is only provided by the to-be-removed
1685 # packages, treat the virtual package as to-be-removed too.
1686 for virtual_pkg in virtual_packages.keys():
1687 if virtual_packages[virtual_pkg] == 0:
1688 removals.append(virtual_pkg)
1690 # Check binary dependencies (Depends)
1691 for package in deps.keys():
1692 if package in removals: continue
1695 parsed_dep += apt_pkg.ParseDepends(deps[package])
1696 except ValueError as e:
1697 print "Error for package %s: %s" % (package, e)
1698 for dep in parsed_dep:
1699 # Check for partial breakage. If a package has a ORed
1700 # dependency, there is only a dependency problem if all
1701 # packages in the ORed depends will be removed.
1703 for dep_package, _, _ in dep:
1704 if dep_package in removals:
1706 if unsat == len(dep):
1707 component = p2c[package]
1708 source = sources[package]
1709 if component != "main":
1710 source = "%s/%s" % (source, component)
1711 all_broken.setdefault(source, {}).setdefault(package, set()).add(architecture)
1716 print " - broken Depends:"
1718 print "# Broken Depends:"
1719 for source, bindict in sorted(all_broken.items()):
1721 for binary, arches in sorted(bindict.items()):
1722 if arches == all_arches or 'all' in arches:
1723 lines.append(binary)
1725 lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
1727 print ' %s: %s' % (source, lines[0])
1729 print '%s: %s' % (source, lines[0])
1730 for line in lines[1:]:
1732 print ' ' + ' ' * (len(source) + 2) + line
1734 print ' ' * (len(source) + 2) + line
1738 # Check source dependencies (Build-Depends and Build-Depends-Indep)
1740 metakey_bd = get_or_set_metadatakey("Build-Depends", session)
1741 metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
1743 'suite_id': dbsuite.suite_id,
1744 'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
1747 SELECT s.id, s.source, string_agg(sm.value, ', ') as build_dep
1749 JOIN source_metadata sm ON s.id = sm.src_id
1751 (SELECT source FROM src_associations
1752 WHERE suite = :suite_id)
1753 AND sm.key_id in :metakey_ids
1754 GROUP BY s.id, s.source'''
1755 query = session.query('id', 'source', 'build_dep').from_statement(statement). \
1757 for source_id, source, build_dep in query:
1758 if source in removals: continue
1760 if build_dep is not None:
1761 # Remove [arch] information since we want to see breakage on all arches
1762 build_dep = re_build_dep_arch.sub("", build_dep)
1764 parsed_dep += apt_pkg.ParseDepends(build_dep)
1765 except ValueError as e:
1766 print "Error for source %s: %s" % (source, e)
1767 for dep in parsed_dep:
1769 for dep_package, _, _ in dep:
1770 if dep_package in removals:
1772 if unsat == len(dep):
1773 component = DBSource.get(source_id, session).get_component_name()
1774 if component != "main":
1775 source = "%s/%s" % (source, component)
1776 all_broken.setdefault(source, set()).add(pp_deps(dep))
1781 print " - broken Build-Depends:"
1783 print "# Broken Build-Depends:"
1784 for source, bdeps in sorted(all_broken.items()):
1785 bdeps = sorted(bdeps)
1787 print ' %s: %s' % (source, bdeps[0])
1789 print '%s: %s' % (source, bdeps[0])
1790 for bdep in bdeps[1:]:
1792 print ' ' + ' ' * (len(source) + 2) + bdep
1794 print ' ' * (len(source) + 2) + bdep