X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Futils.py;h=bdb573af2210330f78f8279ef429cb4e77095093;hb=b67d98ef5542c0258021b03a015c775a6db66d56;hp=27c3af38e422e1c5658462e5a0ccc4197cff749b;hpb=e0ef48cd237d4af31499adbb16606ce3d8b7ee4f;p=dak.git diff --git a/daklib/utils.py b/daklib/utils.py index 27c3af38..bdb573af 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -35,16 +35,18 @@ import tempfile import traceback import stat import apt_pkg -import database import time -import tarfile import re import string import email as modemail + +from dbconn import DBConn, get_architecture, get_component, get_suite from dak_exceptions import * +from textutils import fix_maintainer from regexes import re_html_escaping, html_escaping, re_single_line_field, \ re_multi_line_field, re_srchasver, re_verwithext, \ - re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark + re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark, \ + re_whitespace_comment ################################################################################ @@ -261,6 +263,7 @@ def create_hash(where, files, hashname, hashfunc): file_handle = open_file(f) except CantOpenError: rejmsg.append("Could not open file %s for checksumming" % (f)) + continue files[f][hash_key(hashname)] = hashfunc(file_handle) @@ -511,92 +514,6 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): ################################################################################ -def force_to_utf8(s): - """ - Forces a string to UTF-8. If the string isn't already UTF-8, - it's assumed to be ISO-8859-1. - """ - try: - unicode(s, 'utf-8') - return s - except UnicodeError: - latin1_s = unicode(s,'iso8859-1') - return latin1_s.encode('utf-8') - -def rfc2047_encode(s): - """ - Encodes a (header) string per RFC2047 if necessary. If the - string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1. - """ - try: - codecs.lookup('ascii')[1](s) - return s - except UnicodeError: - pass - try: - codecs.lookup('utf-8')[1](s) - h = email.Header.Header(s, 'utf-8', 998) - return str(h) - except UnicodeError: - h = email.Header.Header(s, 'iso-8859-1', 998) - return str(h) - -################################################################################ - -# 'The standard sucks, but my tool is supposed to interoperate -# with it. I know - I'll fix the suckage and make things -# incompatible!' - -def fix_maintainer (maintainer): - """ - Parses a Maintainer or Changed-By field and returns: - 1. an RFC822 compatible version, - 2. an RFC2047 compatible version, - 3. the name - 4. the email - - The name is forced to UTF-8 for both 1. and 3.. If the name field - contains '.' or ',' (as allowed by Debian policy), 1. and 2. are - switched to 'email (name)' format. - - """ - maintainer = maintainer.strip() - if not maintainer: - return ('', '', '', '') - - if maintainer.find("<") == -1: - email = maintainer - name = "" - elif (maintainer[0] == "<" and maintainer[-1:] == ">"): - email = maintainer[1:-1] - name = "" - else: - m = re_parse_maintainer.match(maintainer) - if not m: - raise ParseMaintError, "Doesn't parse as a valid Maintainer field." - name = m.group(1) - email = m.group(2) - - # Get an RFC2047 compliant version of the name - rfc2047_name = rfc2047_encode(name) - - # Force the name to be UTF-8 - name = force_to_utf8(name) - - if name.find(',') != -1 or name.find('.') != -1: - rfc822_maint = "%s (%s)" % (email, name) - rfc2047_maint = "%s (%s)" % (email, rfc2047_name) - else: - rfc822_maint = "%s <%s>" % (name, email) - rfc2047_maint = "%s <%s>" % (rfc2047_name, email) - - if email.find("@") == -1 and email.find("buildd_") != 0: - raise ParseMaintError, "No @ found in email address part." - - return (rfc822_maint, rfc2047_maint, name, email) - -################################################################################ - def send_mail (message, filename=""): """sendmail wrapper, takes _either_ a message string or a file as arguments""" @@ -616,10 +533,11 @@ def send_mail (message, filename=""): whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"]) try: for line in whitelist_in: - if re_re_mark.match(line): - whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1))) - else: - whitelist.append(re.compile(re.escape(line.strip()))) + if not re_whitespace_comment.match(line): + if re_re_mark.match(line): + whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1))) + else: + whitelist.append(re.compile(re.escape(line.strip()))) finally: whitelist_in.close() @@ -745,6 +663,15 @@ def where_am_i (): def which_conf_file (): res = socket.gethostbyaddr(socket.gethostname()) + # In case we allow local config files per user, try if one exists + if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): + homedir = os.getenv("HOME") + confpath = os.path.join(homedir, "/etc/dak.conf") + if os.path.exists(confpath): + apt_pkg.ReadConfigFileISC(Cnf,default_config) + + # We are still in here, so there is no local config file or we do + # not allow local files. Do the normal stuff. if Cnf.get("Config::" + res[0] + "::DakConfig"): return Cnf["Config::" + res[0] + "::DakConfig"] else: @@ -752,6 +679,13 @@ def which_conf_file (): def which_apt_conf_file (): res = socket.gethostbyaddr(socket.gethostname()) + # In case we allow local config files per user, try if one exists + if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"): + homedir = os.getenv("HOME") + confpath = os.path.join(homedir, "/etc/dak.conf") + if os.path.exists(confpath): + apt_pkg.ReadConfigFileISC(Cnf,default_config) + if Cnf.get("Config::" + res[0] + "::AptConfig"): return Cnf["Config::" + res[0] + "::AptConfig"] else: @@ -977,15 +911,19 @@ def get_conf(): def parse_args(Options): """ Handle -a, -c and -s arguments; returns them as SQL constraints """ + # XXX: This should go away and everything which calls it be converted + # to use SQLA properly. For now, we'll just fix it not to use + # the old Pg interface though + session = DBConn().session() # Process suite if Options["Suite"]: suite_ids_list = [] - for suite in split_args(Options["Suite"]): - suite_id = database.get_suite_id(suite) - if suite_id == -1: - warn("suite '%s' not recognised." % (suite)) + for suitename in split_args(Options["Suite"]): + suite = get_suite(suitename, session=session) + if suite_id is None: + warn("suite '%s' not recognised." % (suitename)) else: - suite_ids_list.append(suite_id) + suite_ids_list.append(suite.suite_id) if suite_ids_list: con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ]) else: @@ -996,12 +934,12 @@ def parse_args(Options): # Process component if Options["Component"]: component_ids_list = [] - for component in split_args(Options["Component"]): - component_id = database.get_component_id(component) - if component_id == -1: - warn("component '%s' not recognised." % (component)) + for componentname in split_args(Options["Component"]): + component = get_component(componentname, session=session) + if component is None: + warn("component '%s' not recognised." % (componentname)) else: - component_ids_list.append(component_id) + component_ids_list.append(component.component_id) if component_ids_list: con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ]) else: @@ -1011,18 +949,18 @@ def parse_args(Options): # Process architecture con_architectures = "" + check_source = 0 if Options["Architecture"]: arch_ids_list = [] - check_source = 0 - for architecture in split_args(Options["Architecture"]): - if architecture == "source": + for archname in split_args(Options["Architecture"]): + if archname == "source": check_source = 1 else: - architecture_id = database.get_architecture_id(architecture) - if architecture_id == -1: - warn("architecture '%s' not recognised." % (architecture)) + arch = get_architecture(archname, session=session) + if arch is None: + warn("architecture '%s' not recognised." % (archname)) else: - arch_ids_list.append(architecture_id) + arch_ids_list.append(arch.arch_id) if arch_ids_list: con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ]) else: @@ -1485,6 +1423,20 @@ def temp_filename(directory=None, prefix="dak", suffix=""): ################################################################################ +def temp_dirname(parent=None, prefix="dak", suffix=""): + """ + Return a secure and unique directory by pre-creating it. + If 'parent' is non-null, it will be the directory the directory is pre-created in. + If 'prefix' is non-null, the filename will be prefixed with it, default is dak. + If 'suffix' is non-null, the filename will end with it. + + Returns a pathname to the new directory + """ + + return tempfile.mkdtemp(suffix, prefix, parent) + +################################################################################ + def is_email_alias(email): """ checks if the user part of the email is listed in the alias file """ global alias_cache @@ -1526,54 +1478,4 @@ apt_pkg.ReadConfigFileISC(Cnf,default_config) if which_conf_file() != default_config: apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) -################################################################################ - -def generate_contents_information(filename): - """ - Generate a list of flies contained in a .deb - - @ptype filename: string - @param filename: the path to a .deb - - @rtype: list - @return: a list of files in the data.tar.* portion of the .deb - """ - cmd = "ar t %s" % (filename) - (result, output) = commands.getstatusoutput(cmd) - if result != 0: - reject("%s: 'ar t' invocation failed." % (filename)) - reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") - - # Ugh ... this is ugly ... Code ripped from process_unchecked.py - chunks = output.split('\n') - - contents = [] - try: - cmd = "ar x %s %s" % (filename, chunks[2]) - (result, output) = commands.getstatusoutput(cmd) - if result != 0: - reject("%s: '%s' invocation failed." % (filename, cmd)) - reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") - - # Got deb tarballs, now lets go through and determine what bits - # and pieces the deb had ... - if chunks[2] == "data.tar.gz": - data = tarfile.open("data.tar.gz", "r:gz") - elif chunks[2] == "data.tar.bz2": - data = tarfile.open("data.tar.bz2", "r:bz2") - else: - os.remove(chunks[2]) - reject("couldn't find data.tar.*") - - for tarinfo in data: - if not tarinfo.isdir(): - contents.append(tarinfo.name[2:]) - - finally: - if os.path.exists( chunks[2] ): - shutil.rmtree( chunks[2] ) - os.remove( chunks[2] ) - - return contents - ###############################################################################