Use more https://

[dak.git] / daklib / utils.py

diff --git a/daklib/utils.py b/daklib/utils.py
old mode 100755 (executable)
new mode 100644 (file)
index abe8f52..a30107b
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -23,6 +23,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 import commands
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 import commands

+import codecs

 import datetime
 import email.Header
 import os
 import datetime
 import email.Header
 import os


@@ -42,8 +43,10 @@ import re
 import email as modemail
 import subprocess
 import ldap
 import email as modemail
 import subprocess
 import ldap

+import errno

 
 import daklib.config as config
 
 import daklib.config as config

+import daklib.daksubprocess

 from dbconn import DBConn, get_architecture, get_component, get_suite, \
                    get_override_type, Keyring, session_wrapper, \
                    get_active_keyring_paths, get_primary_keyring_path, \
 from dbconn import DBConn, get_architecture, get_component, get_suite, \
                    get_override_type, Keyring, session_wrapper, \
                    get_active_keyring_paths, get_primary_keyring_path, \


@@ -55,8 +58,8 @@ from gpg import SignedFile
 from textutils import fix_maintainer
 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
                     re_multi_line_field, re_srchasver, re_taint_free, \
 from textutils import fix_maintainer
 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
                     re_multi_line_field, re_srchasver, re_taint_free, \

-                    re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \
-                    re_is_orig_source, re_build_dep_arch
+                    re_re_mark, re_whitespace_comment, re_issource, \
+                    re_is_orig_source, re_build_dep_arch, re_parse_maintainer

 
 from formats import parse_format, validate_changes_format
 from srcformats import get_format_from_string
 
 from formats import parse_format, validate_changes_format
 from srcformats import get_format_from_string


@@ -65,7 +68,6 @@ from collections import defaultdict
 ################################################################################
 
 default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties
 ################################################################################
 
 default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties

-default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used

 
 alias_cache = None        #: Cache for email alias checks
 key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids
 
 alias_cache = None        #: Cache for email alias checks
 key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids


@@ -78,7 +80,7 @@ known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
 # code in lenny's Python. This also affects commands.getoutput and
 # commands.getstatus.
 def dak_getstatusoutput(cmd):
 # code in lenny's Python. This also affects commands.getoutput and
 # commands.getstatus.
 def dak_getstatusoutput(cmd):

-    pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
+    pipe = daklib.daksubprocess.Popen(cmd, shell=True, universal_newlines=True,

         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     output = pipe.stdout.read()
         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     output = pipe.stdout.read()


@@ -153,11 +155,7 @@ def extract_component_from_section(section, session=None):
 
     # Expand default component
     if component == "":
 
     # Expand default component
     if component == "":

-        comp = get_component(section, session)
-        if comp is None:
-            component = "main"
-        else:
-            component = comp.component_name
+        component = "main"

 
     return (section, component)
 
 
     return (section, component)
 


@@ -258,9 +256,8 @@ def parse_changes(filename, signing_rules=0, dsc_file=0, keyrings=None):
         "-----BEGIN PGP SIGNATURE-----".
     """
 
         "-----BEGIN PGP SIGNATURE-----".
     """
 

-    changes_in = open_file(filename)
-    content = changes_in.read()
-    changes_in.close()
+    with open_file(filename) as changes_in:
+        content = changes_in.read()

     try:
         unicode(content, 'utf-8')
     except UnicodeError:
     try:
         unicode(content, 'utf-8')
     except UnicodeError:


@@ -321,11 +318,8 @@ def check_hash(where, files, hashname, hashfunc):
 
     rejmsg = []
     for f in files.keys():
 
     rejmsg = []
     for f in files.keys():

-        file_handle = None

         try:
         try:

-            try:
-                file_handle = open_file(f)
-
+            with open_file(f) as file_handle:

                 # Check for the hash entry, to not trigger a KeyError.
                 if not files[f].has_key(hash_key(hashname)):
                     rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,
                 # Check for the hash entry, to not trigger a KeyError.
                 if not files[f].has_key(hash_key(hashname)):
                     rejmsg.append("%s: misses %s checksum in %s" % (f, hashname,


@@ -336,13 +330,10 @@ def check_hash(where, files, hashname, hashfunc):
                 if hashfunc(file_handle) != files[f][hash_key(hashname)]:
                     rejmsg.append("%s: %s check failed in %s" % (f, hashname,
                         where))
                 if hashfunc(file_handle) != files[f][hash_key(hashname)]:
                     rejmsg.append("%s: %s check failed in %s" % (f, hashname,
                         where))

-            except CantOpenError:
-                # TODO: This happens when the file is in the pool.
-                # warn("Cannot open file %s" % f)
-                continue
-        finally:
-            if file_handle:
-                file_handle.close()
+        except CantOpenError:
+            # TODO: This happens when the file is in the pool.
+            # warn("Cannot open file %s" % f)
+            continue

     return rejmsg
 
 ################################################################################
     return rejmsg
 
 ################################################################################


@@ -358,7 +349,7 @@ def check_size(where, files):
         try:
             entry = os.stat(f)
         except OSError as exc:
         try:
             entry = os.stat(f)
         except OSError as exc:

-            if exc.errno == 2:
+            if exc.errno == errno.ENOENT:

                 # TODO: This happens when the file is in the pool.
                 continue
             raise
                 # TODO: This happens when the file is in the pool.
                 continue
             raise


@@ -575,7 +566,7 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
 
 ################################################################################
 
 
 ################################################################################
 

-# see http://bugs.debian.org/619131
+# see https://bugs.debian.org/619131

 def build_package_list(dsc, session = None):
     if not dsc.has_key("package-list"):
         return {}
 def build_package_list(dsc, session = None):
     if not dsc.has_key("package-list"):
         return {}


@@ -619,9 +610,8 @@ def send_mail (message, filename="", whitelists=None):
     if maildir:
         path = os.path.join(maildir, datetime.datetime.now().isoformat())
         path = find_next_free(path)
     if maildir:
         path = os.path.join(maildir, datetime.datetime.now().isoformat())
         path = find_next_free(path)

-        fh = open(path, 'w')
-        print >>fh, message,
-        fh.close()
+        with open(path, 'w') as fh:
+            print >>fh, message,

 
     # Check whether we're supposed to be sending mail
     if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]:
 
     # Check whether we're supposed to be sending mail
     if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]:


@@ -638,9 +628,8 @@ def send_mail (message, filename="", whitelists=None):
     if Cnf.get('Dinstall::MailWhiteList', ''):
         whitelists.append(Cnf['Dinstall::MailWhiteList'])
     if len(whitelists) != 0:
     if Cnf.get('Dinstall::MailWhiteList', ''):
         whitelists.append(Cnf['Dinstall::MailWhiteList'])
     if len(whitelists) != 0:

-        message_in = open_file(filename)
-        message_raw = modemail.message_from_file(message_in)
-        message_in.close();
+        with open_file(filename) as message_in:
+            message_raw = modemail.message_from_file(message_in)

 
         whitelist = [];
         for path in whitelists:
 
         whitelist = [];
         for path in whitelists:


@@ -720,7 +709,7 @@ def move (src, dest, overwrite = 0, perms = 0o664):
         dest_dir = dest
     else:
         dest_dir = os.path.dirname(dest)
         dest_dir = dest
     else:
         dest_dir = os.path.dirname(dest)

-    if not os.path.exists(dest_dir):
+    if not os.path.lexists(dest_dir):

         umask = os.umask(00000)
         os.makedirs(dest_dir, 0o2775)
         os.umask(umask)
         umask = os.umask(00000)
         os.makedirs(dest_dir, 0o2775)
         os.umask(umask)


@@ -728,7 +717,7 @@ def move (src, dest, overwrite = 0, perms = 0o664):
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to

-    if os.path.exists(dest):
+    if os.path.lexists(dest):

         if not overwrite:
             fubar("Can't move %s to %s - file already exists." % (src, dest))
         else:
         if not overwrite:
             fubar("Can't move %s to %s - file already exists." % (src, dest))
         else:


@@ -751,7 +740,7 @@ def copy (src, dest, overwrite = 0, perms = 0o664):
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to

-    if os.path.exists(dest):
+    if os.path.lexists(dest):

         if not overwrite:
             raise FileExistsError
         else:
         if not overwrite:
             raise FileExistsError
         else:


@@ -762,14 +751,6 @@ def copy (src, dest, overwrite = 0, perms = 0o664):
 
 ################################################################################
 
 
 ################################################################################
 

-def where_am_i ():
-    res = socket.getfqdn()
-    database_hostname = Cnf.get("Config::" + res + "::DatabaseHostname")
-    if database_hostname:
-        return database_hostname
-    else:
-        return res
-

 def which_conf_file ():
     if os.getenv('DAK_CONFIG'):
         return os.getenv('DAK_CONFIG')
 def which_conf_file ():
     if os.getenv('DAK_CONFIG'):
         return os.getenv('DAK_CONFIG')


@@ -780,7 +761,7 @@ def which_conf_file ():
         homedir = os.getenv("HOME")
         confpath = os.path.join(homedir, "/etc/dak.conf")
         if os.path.exists(confpath):
         homedir = os.getenv("HOME")
         confpath = os.path.join(homedir, "/etc/dak.conf")
         if os.path.exists(confpath):

-            apt_pkg.ReadConfigFileISC(Cnf,confpath)
+            apt_pkg.read_config_file_isc(Cnf,confpath)

 
     # We are still in here, so there is no local config file or we do
     # not allow local files. Do the normal stuff.
 
     # We are still in here, so there is no local config file or we do
     # not allow local files. Do the normal stuff.


@@ -789,37 +770,14 @@ def which_conf_file ():
 
     return default_config
 
 
     return default_config
 

-def which_apt_conf_file ():
-    res = socket.getfqdn()
-    # In case we allow local config files per user, try if one exists
-    if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
-        homedir = os.getenv("HOME")
-        confpath = os.path.join(homedir, "/etc/dak.conf")
-        if os.path.exists(confpath):
-            apt_pkg.ReadConfigFileISC(Cnf,default_config)
-
-    if Cnf.get("Config::" + res + "::AptConfig"):
-        return Cnf["Config::" + res + "::AptConfig"]
-    else:
-        return default_apt_config
-
-def which_alias_file():
-    hostname = socket.getfqdn()
-    aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
-    if os.path.exists(aliasfn):
-        return aliasfn
-    else:
-        return None
-

 ################################################################################
 
 def TemplateSubst(subst_map, filename):
     """ Perform a substition of template """
 ################################################################################
 
 def TemplateSubst(subst_map, filename):
     """ Perform a substition of template """

-    templatefile = open_file(filename)
-    template = templatefile.read()
+    with open_file(filename) as templatefile:
+        template = templatefile.read()

     for k, v in subst_map.iteritems():
         template = template.replace(k, str(v))
     for k, v in subst_map.iteritems():
         template = template.replace(k, str(v))

-    templatefile.close()

     return template
 
 ################################################################################
     return template
 
 ################################################################################


@@ -908,7 +866,7 @@ def changes_compare (a, b):
 def find_next_free (dest, too_many=100):
     extra = 0
     orig_dest = dest
 def find_next_free (dest, too_many=100):
     extra = 0
     orig_dest = dest

-    while os.path.exists(dest) and extra < too_many:
+    while os.path.lexists(dest) and extra < too_many:

         dest = orig_dest + '.' + repr(extra)
         extra += 1
     if extra >= too_many:
         dest = orig_dest + '.' + repr(extra)
         extra += 1
     if extra >= too_many:


@@ -1096,7 +1054,7 @@ def arch_compare_sw (a, b):
 
 ################################################################################
 
 
 ################################################################################
 

-def split_args (s, dwim=1):
+def split_args (s, dwim=True):

     """
     Split command line arguments which can be separated by either commas
     or whitespace.  If dwim is set, it will complain about string ending
     """
     Split command line arguments which can be separated by either commas
     or whitespace.  If dwim is set, it will complain about string ending


@@ -1402,21 +1360,38 @@ def gpg_get_key_addresses(fingerprint):
     if addresses != None:
         return addresses
     addresses = list()
     if addresses != None:
         return addresses
     addresses = list()

-    cmd = "gpg --no-default-keyring %s --fingerprint %s" \
-                % (gpg_keyring_args(), fingerprint)
-    (result, output) = commands.getstatusoutput(cmd)
-    if result == 0:
+    try:
+        with open(os.devnull, "wb") as devnull:
+            output = daklib.daksubprocess.check_output(
+                ["gpg", "--no-default-keyring"] + gpg_keyring_args().split() +
+                ["--with-colons", "--list-keys", fingerprint], stderr=devnull)
+    except subprocess.CalledProcessError:
+        pass
+    else:

         for l in output.split('\n'):
         for l in output.split('\n'):

-            m = re_gpg_uid.match(l)
+            parts = l.split(':')
+            if parts[0] not in ("uid", "pub"):
+                continue
+            try:
+                uid = parts[9]
+            except IndexError:
+                continue
+            try:
+                # Do not use unicode_escape, because it is locale-specific
+                uid = codecs.decode(uid, "string_escape").decode("utf-8")
+            except UnicodeDecodeError:
+                uid = uid.decode("latin1") # does not fail
+            m = re_parse_maintainer.match(uid)

             if not m:
                 continue
             if not m:
                 continue

-            address = m.group(1)
+            address = m.group(2)
+            address = address.encode("utf8") # dak still uses bytes

             if address.endswith('@debian.org'):
                 # prefer @debian.org addresses
                 # TODO: maybe not hardcode the domain
                 addresses.insert(0, address)
             else:
             if address.endswith('@debian.org'):
                 # prefer @debian.org addresses
                 # TODO: maybe not hardcode the domain
                 addresses.insert(0, address)
             else:

-                addresses.append(m.group(1))
+                addresses.append(address)

     key_uid_email_cache[fingerprint] = addresses
     return addresses
 
     key_uid_email_cache[fingerprint] = addresses
     return addresses
 


@@ -1439,6 +1414,30 @@ def get_logins_from_ldap(fingerprint='*'):
 
 ################################################################################
 
 
 ################################################################################
 

+def get_users_from_ldap():
+    """retrieve login and user names from LDAP"""
+
+    LDAPDn = Cnf['Import-LDAP-Fingerprints::LDAPDn']
+    LDAPServer = Cnf['Import-LDAP-Fingerprints::LDAPServer']
+    l = ldap.open(LDAPServer)
+    l.simple_bind_s('','')
+    Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
+                       '(uid=*)', ['uid', 'cn', 'mn', 'sn'])
+    users = {}
+    for elem in Attrs:
+        elem = elem[1]
+        name = []
+        for k in ('cn', 'mn', 'sn'):
+            try:
+                if elem[k][0] != '-':
+                    name.append(elem[k][0])
+            except KeyError:
+                pass
+        users[' '.join(name)] = elem['uid'][0]
+    return users
+
+################################################################################
+

 def clean_symlink (src, dest, root):
     """
     Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
 def clean_symlink (src, dest, root):
     """
     Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.


@@ -1558,7 +1557,7 @@ Cnf = config.Config().Cnf
 
 def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"):
     """
 
 def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"):
     """

-    Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm
+    Parses the wnpp bug list available at https://qa.debian.org/data/bts/wnpp_rm

     Well, actually it parsed a local copy, but let's document the source
     somewhere ;)
 
     Well, actually it parsed a local copy, but let's document the source
     somewhere ;)
 


@@ -1572,7 +1571,7 @@ def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/w
         lines = f.readlines()
     except IOError as e:
         print "Warning:  Couldn't open %s; don't know about WNPP bugs, so won't close any." % file
         lines = f.readlines()
     except IOError as e:
         print "Warning:  Couldn't open %s; don't know about WNPP bugs, so won't close any." % file

-       lines = []
+        lines = []

     wnpp = {}
 
     for line in lines:
     wnpp = {}
 
     for line in lines:


@@ -1622,7 +1621,7 @@ def get_packages_from_ftp(root, suite, component, architecture):
         if (result != 0):
             fubar("Gunzip invocation failed!\n%s\n" % (output), result)
     packages = open_file(temp_file)
         if (result != 0):
             fubar("Gunzip invocation failed!\n%s\n" % (output), result)
     packages = open_file(temp_file)

-    Packages = apt_pkg.ParseTagFile(packages)
+    Packages = apt_pkg.TagFile(packages)

     os.unlink(temp_file)
     return Packages
 
     os.unlink(temp_file)
     return Packages
 


@@ -1680,7 +1679,7 @@ def call_editor(text="", suffix=".txt"):
     try:
         print >>tmp, text,
         tmp.close()
     try:
         print >>tmp, text,
         tmp.close()

-        subprocess.check_call([editor, tmp.name])
+        daklib.daksubprocess.check_call([editor, tmp.name])

         return open(tmp.name, 'r').read()
     finally:
         os.unlink(tmp.name)
         return open(tmp.name, 'r').read()
     finally:
         os.unlink(tmp.name)


@@ -1756,7 +1755,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
             if package in removals: continue
             parsed_dep = []
             try:
             if package in removals: continue
             parsed_dep = []
             try:

-                parsed_dep += apt_pkg.ParseDepends(deps[package])
+                parsed_dep += apt_pkg.parse_depends(deps[package])

             except ValueError as e:
                 print "Error for package %s: %s" % (package, e)
             for dep in parsed_dep:
             except ValueError as e:
                 print "Error for package %s: %s" % (package, e)
             for dep in parsed_dep:


@@ -1825,7 +1824,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
             # Remove [arch] information since we want to see breakage on all arches
             build_dep = re_build_dep_arch.sub("", build_dep)
             try:
             # Remove [arch] information since we want to see breakage on all arches
             build_dep = re_build_dep_arch.sub("", build_dep)
             try:

-                parsed_dep += apt_pkg.ParseDepends(build_dep)
+                parsed_dep += apt_pkg.parse_depends(build_dep)

             except ValueError as e:
                 print "Error for source %s: %s" % (source, e)
         for dep in parsed_dep:
             except ValueError as e:
                 print "Error for source %s: %s" % (source, e)
         for dep in parsed_dep:


@@ -1840,9 +1839,10 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
                     .filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
                     .join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
                     .first()
                     .filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
                     .join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
                     .first()

+                key = source

                 if component != "main":
                 if component != "main":

-                    source = "%s/%s" % (source, component)
-                all_broken.setdefault(source, set()).add(pp_deps(dep))
+                    key = "%s/%s" % (source, component)
+                all_broken.setdefault(key, set()).add(pp_deps(dep))

                 dep_problem = 1
 
     if all_broken:
                 dep_problem = 1
 
     if all_broken: