X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Futils.py;h=cfbf8aa00199f22a3f333e1410c7847d18c145d8;hb=1a7c6855a41abed329800622d8990eed5c009cae;hp=ea2f8b7fda427e136bf037d5f22d73468bae4814;hpb=5e1a34528993b82c0c57f93b16e09cc686af9c82;p=dak.git diff --git a/daklib/utils.py b/daklib/utils.py old mode 100755 new mode 100644 index ea2f8b7f..cfbf8aa0 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import codecs import commands import email.Header import os @@ -37,18 +36,19 @@ import stat import apt_pkg import time import re -import string import email as modemail +import subprocess from dbconn import DBConn, get_architecture, get_component, get_suite from dak_exceptions import * from textutils import fix_maintainer from regexes import re_html_escaping, html_escaping, re_single_line_field, \ - re_multi_line_field, re_srchasver, re_verwithext, \ - re_parse_maintainer, re_taint_free, re_gpg_uid, \ - re_re_mark, re_whitespace_comment, re_issource + re_multi_line_field, re_srchasver, re_taint_free, \ + re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource -from srcformats import srcformats +from formats import parse_format, validate_changes_format +from srcformats import get_format_from_string +from collections import defaultdict ################################################################################ @@ -62,6 +62,25 @@ key_uid_email_cache = {} #: Cache for email addresses from gpg key uids known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)), ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc +# Monkeypatch commands.getstatusoutput as it may not return the correct exit +# code in lenny's Python. This also affects commands.getoutput and +# commands.getstatus. +def dak_getstatusoutput(cmd): + pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + output = "".join(pipe.stdout.readlines()) + + if output[-1:] == '\n': + output = output[:-1] + + ret = pipe.wait() + if ret is None: + ret = 0 + + return ret, output +commands.getstatusoutput = dak_getstatusoutput + ################################################################################ def html_escape(s): @@ -96,7 +115,12 @@ def open_file(filename, mode='r'): def our_raw_input(prompt=""): if prompt: - sys.stdout.write(prompt) + while 1: + try: + sys.stdout.write(prompt) + break + except IOError: + pass sys.stdout.flush() try: ret = raw_input() @@ -287,13 +311,13 @@ def check_hash(where, files, hashname, hashfunc): try: try: file_handle = open_file(f) - + # Check for the hash entry, to not trigger a KeyError. if not files[f].has_key(hash_key(hashname)): rejmsg.append("%s: misses %s checksum in %s" % (f, hashname, where)) continue - + # Actually check the hash for correctness. if hashfunc(file_handle) != files[f][hash_key(hashname)]: rejmsg.append("%s: %s check failed in %s" % (f, hashname, @@ -354,65 +378,61 @@ def check_dsc_files(dsc_filename, dsc=None, dsc_files=None): rejmsg = [] # Parse the file if needed - if dsc == None: + if dsc is None: dsc = parse_changes(dsc_filename, signing_rules=1); - if dsc_files == None: + + if dsc_files is None: dsc_files = build_file_list(dsc, is_a_dsc=1) # Ensure .dsc lists proper set of source files according to the format # announced - has_native_tar = 0 - has_native_tar_gz = 0 - has_orig_tar = 0 - has_orig_tar_gz = 0 - has_more_orig_tar = 0 - has_debian_tar = 0 - has_debian_diff = 0 + has = defaultdict(lambda: 0) + + ftype_lookup = ( + (r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')), + (r'diff.gz', ('debian_diff',)), + (r'tar.gz', ('native_tar_gz', 'native_tar')), + (r'debian\.tar\.(gz|bz2)', ('debian_tar',)), + (r'orig\.tar\.(gz|bz2)', ('orig_tar',)), + (r'tar\.(gz|bz2)', ('native_tar',)), + (r'orig-.+\.tar\.(gz|bz2)', ('more_orig_tar',)), + ) + for f in dsc_files.keys(): m = re_issource.match(f) if not m: rejmsg.append("%s: %s in Files field not recognised as source." % (dsc_filename, f)) continue - ftype = m.group(3) - if ftype == "orig.tar.gz": - has_orig_tar_gz += 1 - has_orig_tar += 1 - elif ftype == "diff.gz": - has_debian_diff += 1 - elif ftype == "tar.gz": - has_native_tar_gz += 1 - has_native_tar += 1 - elif re.match(r"debian\.tar\.(gz|bz2|lzma)", ftype): - has_debian_tar += 1 - elif re.match(r"orig\.tar\.(gz|bz2|lzma)", ftype): - has_orig_tar += 1 - elif re.match(r"tar\.(gz|bz2|lzma)", ftype): - has_native_tar += 1 - elif re.match(r"orig-.+\.tar\.(gz|bz2|lzma)", ftype): - has_more_orig_tar += 1 - else: + + # Populate 'has' dictionary by resolving keys in lookup table + matched = False + for regex, keys in ftype_lookup: + if re.match(regex, m.group(3)): + matched = True + for key in keys: + has[key] += 1 + break + + # File does not match anything in lookup table; reject + if not matched: reject("%s: unexpected source file '%s'" % (dsc_filename, f)) - if has_orig_tar > 1: - rejmsg.append("%s: lists multiple .orig tarballs." % (dsc_filename)) - if has_native_tar > 1: - rejmsg.append("%s: lists multiple native tarballs." % (dsc_filename)) - if has_debian_tar > 1 or has_debian_diff > 1: - rejmsg.append("%s: lists multiple debian diff/tarballs." % (dsc_filename)) - - for format in srcformats: - if format.re_format.match(dsc['format']): - rejmsg.extend(format.reject_msgs( - dsc_filename, - has_native_tar, - has_native_tar_gz, - has_debian_tar, - has_debian_diff, - has_orig_tar, - has_orig_tar_gz, - has_more_orig_tar - )) - break + + # Check for multiple files + for file_type in ('orig_tar', 'native_tar', 'debian_tar', 'debian_diff'): + if has[file_type] > 1: + rejmsg.append("%s: lists multiple %s" % (dsc_filename, file_type)) + + # Source format specific tests + try: + format = get_format_from_string(dsc['format']) + rejmsg.extend([ + '%s: %s' % (dsc_filename, x) for x in format.reject_msgs(has) + ]) + + except UnknownFormatError: + # Not an error here for now + pass return rejmsg @@ -514,30 +534,9 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): if not changes.has_key(field): raise NoFilesFieldError - # Make sure we recognise the format of the Files: field - format = re_verwithext.search(changes.get("format", "0.0")) - if not format: - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) - - format = format.groups() - if format[1] == None: - format = int(float(format[0])), 0, format[2] - else: - format = int(format[0]), int(format[1]), format[2] - if format[2] == None: - format = format[:2] - - if is_a_dsc: - # format = (0,0) are missing format headers of which we still - # have some in the archive. - if format != (1,0) and format != (0,0) and \ - format != (3,0,"quilt") and format != (3,0,"native"): - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) - else: - if (format < (1,5) or format > (1,8)): - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) - if field != "files" and format < (1,8): - raise UnknownFormatError, "%s" % (changes.get("format","0.0")) + # Validate .changes Format: field + if not is_a_dsc: + validate_changes_format(parse_format(changes['format']), field) includes_section = (not is_a_dsc) and field == "files" @@ -562,7 +561,7 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): (section, component) = extract_component_from_section(section) - files[name] = Dict(size=size, section=section, + files[name] = dict(size=size, section=section, priority=priority, component=component) files[name][hashname] = md5 @@ -620,7 +619,7 @@ def send_mail (message, filename=""): if len(match) == 0: del message_raw[field] else: - message_raw.replace_header(field, string.join(match, ", ")) + message_raw.replace_header(field, ', '.join(match)) # Change message fields in order if we don't have a To header if not message_raw.has_key("To"): @@ -718,6 +717,9 @@ def where_am_i (): return res[0] def which_conf_file (): + if os.getenv('DAK_CONFIG'): + return os.getenv('DAK_CONFIG') + res = socket.gethostbyaddr(socket.gethostname()) # In case we allow local config files per user, try if one exists if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): @@ -730,8 +732,8 @@ def which_conf_file (): # not allow local files. Do the normal stuff. if Cnf.get("Config::" + res[0] + "::DakConfig"): return Cnf["Config::" + res[0] + "::DakConfig"] - else: - return default_config + + return default_config def which_apt_conf_file (): res = socket.gethostbyaddr(socket.gethostname()) @@ -757,12 +759,12 @@ def which_alias_file(): ################################################################################ -def TemplateSubst(map, filename): +def TemplateSubst(subst_map, filename): """ Perform a substition of template """ templatefile = open_file(filename) template = templatefile.read() - for x in map.keys(): - template = template.replace(x, str(map[x])) + for k, v in subst_map.iteritems(): + template = template.replace(k, str(v)) templatefile.close() return template @@ -1095,10 +1097,6 @@ def split_args (s, dwim=1): ################################################################################ -def Dict(**dict): return dict - -######################################## - def gpgv_get_status_output(cmd, status_read, status_write): """ Our very own version of commands.getouputstatus(), hacked to support @@ -1366,9 +1364,9 @@ def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=No rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)) # Finally ensure there's not something we don't recognise - known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", + known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", - NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="") + NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="") for keyword in keywords.keys(): if not known_keywords.has_key(keyword): @@ -1488,7 +1486,7 @@ def is_email_alias(email): ################################################################################ -def get_changes_files(dir): +def get_changes_files(from_dir): """ Takes a directory and lists all .changes files in it (as well as chdir'ing to the directory; this is due to broken behaviour on the part of p-u/p-a @@ -1498,10 +1496,10 @@ def get_changes_files(dir): """ try: # Much of the rest of p-u/p-a depends on being in the right place - os.chdir(dir) - changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')] + os.chdir(from_dir) + changes_files = [x for x in os.listdir(from_dir) if x.endswith('.changes')] except OSError, e: - fubar("Failed to read list from directory %s (%s)" % (dir, e)) + fubar("Failed to read list from directory %s (%s)" % (from_dir, e)) return changes_files @@ -1510,9 +1508,8 @@ def get_changes_files(dir): apt_pkg.init() Cnf = apt_pkg.newConfiguration() -apt_pkg.ReadConfigFileISC(Cnf,default_config) +if not os.getenv("DAK_TEST"): + apt_pkg.ReadConfigFileISC(Cnf,default_config) if which_conf_file() != default_config: apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) - -###############################################################################