Fix permissions for files in daklib/

[dak.git] / daklib / utils.py

diff --git a/daklib/utils.py b/daklib/utils.py
index 4ee00a1c263dd1bd8c484f1e0baae5a057aa3376..655e3817f0aa2fba2bc8cb370cdcdcd1e8df745d 100644 (file)
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -27,6 +27,7 @@ import datetime
 import email.Header
 import os
 import pwd
 import email.Header
 import os
 import pwd

+import grp

 import select
 import socket
 import shutil
 import select
 import socket
 import shutil


@@ -40,11 +41,15 @@ import time
 import re
 import email as modemail
 import subprocess
 import re
 import email as modemail
 import subprocess

+import ldap

 
 

+import daklib.config as config
+import daklib.daksubprocess

 from dbconn import DBConn, get_architecture, get_component, get_suite, \
                    get_override_type, Keyring, session_wrapper, \
                    get_active_keyring_paths, get_primary_keyring_path, \
 from dbconn import DBConn, get_architecture, get_component, get_suite, \
                    get_override_type, Keyring, session_wrapper, \
                    get_active_keyring_paths, get_primary_keyring_path, \

-                   get_suite_architectures, get_or_set_metadatakey, DBSource
+                   get_suite_architectures, get_or_set_metadatakey, DBSource, \
+                   Component, Override, OverrideType

 from sqlalchemy import desc
 from dak_exceptions import *
 from gpg import SignedFile
 from sqlalchemy import desc
 from dak_exceptions import *
 from gpg import SignedFile


@@ -61,7 +66,6 @@ from collections import defaultdict
 ################################################################################
 
 default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties
 ################################################################################
 
 default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties

-default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used

 
 alias_cache = None        #: Cache for email alias checks
 key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids
 
 alias_cache = None        #: Cache for email alias checks
 key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids


@@ -74,7 +78,7 @@ known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
 # code in lenny's Python. This also affects commands.getoutput and
 # commands.getstatus.
 def dak_getstatusoutput(cmd):
 # code in lenny's Python. This also affects commands.getoutput and
 # commands.getstatus.
 def dak_getstatusoutput(cmd):

-    pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
+    pipe = daklib.daksubprocess.Popen(cmd, shell=True, universal_newlines=True,

         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     output = pipe.stdout.read()
         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     output = pipe.stdout.read()


@@ -368,7 +372,7 @@ def check_size(where, files):
 
 ################################################################################
 
 
 ################################################################################
 

-def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
+def check_dsc_files(dsc_filename, dsc, dsc_files):

     """
     Verify that the files listed in the Files field of the .dsc are
     those expected given the announced Format.
     """
     Verify that the files listed in the Files field of the .dsc are
     those expected given the announced Format.


@@ -387,13 +391,6 @@ def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
     """
     rejmsg = []
 
     """
     rejmsg = []
 

-    # Parse the file if needed
-    if dsc is None:
-        dsc = parse_changes(dsc_filename, signing_rules=1, dsc_file=1);
-
-    if dsc_files is None:
-        dsc_files = build_file_list(dsc, is_a_dsc=1)
-

     # Ensure .dsc lists proper set of source files according to the format
     # announced
     has = defaultdict(lambda: 0)
     # Ensure .dsc lists proper set of source files according to the format
     # announced
     has = defaultdict(lambda: 0)


@@ -408,7 +405,7 @@ def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
         (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
     )
 
         (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
     )
 

-    for f in dsc_files.keys():
+    for f in dsc_files:

         m = re_issource.match(f)
         if not m:
             rejmsg.append("%s: %s in Files field not recognised as source."
         m = re_issource.match(f)
         if not m:
             rejmsg.append("%s: %s in Files field not recognised as source."


@@ -607,8 +604,16 @@ def build_package_list(dsc, session = None):
 
 ################################################################################
 
 
 ################################################################################
 

-def send_mail (message, filename=""):
-    """sendmail wrapper, takes _either_ a message string or a file as arguments"""
+def send_mail (message, filename="", whitelists=None):
+    """sendmail wrapper, takes _either_ a message string or a file as arguments
+
+    @type  whitelists: list of (str or None)
+    @param whitelists: path to whitelists. C{None} or an empty list whitelists
+                       everything, otherwise an address is whitelisted if it is
+                       included in any of the lists.
+                       In addition a global whitelist can be specified in
+                       Dinstall::MailWhiteList.
+    """

 
     maildir = Cnf.get('Dir::Mail')
     if maildir:
 
     maildir = Cnf.get('Dir::Mail')
     if maildir:


@@ -628,23 +633,24 @@ def send_mail (message, filename=""):
         os.write (fd, message)
         os.close (fd)
 
         os.write (fd, message)
         os.close (fd)
 

-    if Cnf.has_key("Dinstall::MailWhiteList") and \
-           Cnf["Dinstall::MailWhiteList"] != "":
+    if whitelists is None or None in whitelists:
+        whitelists = []
+    if Cnf.get('Dinstall::MailWhiteList', ''):
+        whitelists.append(Cnf['Dinstall::MailWhiteList'])
+    if len(whitelists) != 0:

         message_in = open_file(filename)
         message_raw = modemail.message_from_file(message_in)
         message_in.close();
 
         whitelist = [];
         message_in = open_file(filename)
         message_raw = modemail.message_from_file(message_in)
         message_in.close();
 
         whitelist = [];

-        whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
-        try:
+        for path in whitelists:
+          with open_file(path, 'r') as whitelist_in:

             for line in whitelist_in:
                 if not re_whitespace_comment.match(line):
                     if re_re_mark.match(line):
                         whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
                     else:
                         whitelist.append(re.compile(re.escape(line.strip())))
             for line in whitelist_in:
                 if not re_whitespace_comment.match(line):
                     if re_re_mark.match(line):
                         whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
                     else:
                         whitelist.append(re.compile(re.escape(line.strip())))

-        finally:
-            whitelist_in.close()

 
         # Fields to check.
         fields = ["To", "Bcc", "Cc"]
 
         # Fields to check.
         fields = ["To", "Bcc", "Cc"]


@@ -661,7 +667,7 @@ def send_mail (message, filename=""):
                             mail_whitelisted = 1
                             break
                     if not mail_whitelisted:
                             mail_whitelisted = 1
                             break
                     if not mail_whitelisted:

-                        print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
+                        print "Skipping {0} since it's not whitelisted".format(item)

                         continue
                     match.append(item)
 
                         continue
                     match.append(item)
 


@@ -701,13 +707,11 @@ def send_mail (message, filename=""):
 
 ################################################################################
 
 
 ################################################################################
 

-def poolify (source, component):
-    if component:
-        component += '/'
+def poolify (source, component=None):

     if source[:3] == "lib":
     if source[:3] == "lib":

-        return component + source[:4] + '/' + source + '/'
+        return source[:4] + '/' + source + '/'

     else:
     else:

-        return component + source[:1] + '/' + source + '/'
+        return source[:1] + '/' + source + '/'

 
 ################################################################################
 
 
 ################################################################################
 


@@ -716,7 +720,7 @@ def move (src, dest, overwrite = 0, perms = 0o664):
         dest_dir = dest
     else:
         dest_dir = os.path.dirname(dest)
         dest_dir = dest
     else:
         dest_dir = os.path.dirname(dest)

-    if not os.path.exists(dest_dir):
+    if not os.path.lexists(dest_dir):

         umask = os.umask(00000)
         os.makedirs(dest_dir, 0o2775)
         os.umask(umask)
         umask = os.umask(00000)
         os.makedirs(dest_dir, 0o2775)
         os.umask(umask)


@@ -724,7 +728,7 @@ def move (src, dest, overwrite = 0, perms = 0o664):
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to

-    if os.path.exists(dest):
+    if os.path.lexists(dest):

         if not overwrite:
             fubar("Can't move %s to %s - file already exists." % (src, dest))
         else:
         if not overwrite:
             fubar("Can't move %s to %s - file already exists." % (src, dest))
         else:


@@ -747,7 +751,7 @@ def copy (src, dest, overwrite = 0, perms = 0o664):
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to
     if os.path.exists(dest) and os.path.isdir(dest):
         dest += '/' + os.path.basename(src)
     # Don't overwrite unless forced to

-    if os.path.exists(dest):
+    if os.path.lexists(dest):

         if not overwrite:
             raise FileExistsError
         else:
         if not overwrite:
             raise FileExistsError
         else:


@@ -776,7 +780,7 @@ def which_conf_file ():
         homedir = os.getenv("HOME")
         confpath = os.path.join(homedir, "/etc/dak.conf")
         if os.path.exists(confpath):
         homedir = os.getenv("HOME")
         confpath = os.path.join(homedir, "/etc/dak.conf")
         if os.path.exists(confpath):

-            apt_pkg.ReadConfigFileISC(Cnf,confpath)
+            apt_pkg.read_config_file_isc(Cnf,confpath)

 
     # We are still in here, so there is no local config file or we do
     # not allow local files. Do the normal stuff.
 
     # We are still in here, so there is no local config file or we do
     # not allow local files. Do the normal stuff.


@@ -785,20 +789,6 @@ def which_conf_file ():
 
     return default_config
 
 
     return default_config
 

-def which_apt_conf_file ():
-    res = socket.getfqdn()
-    # In case we allow local config files per user, try if one exists
-    if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
-        homedir = os.getenv("HOME")
-        confpath = os.path.join(homedir, "/etc/dak.conf")
-        if os.path.exists(confpath):
-            apt_pkg.ReadConfigFileISC(Cnf,default_config)
-
-    if Cnf.get("Config::" + res + "::AptConfig"):
-        return Cnf["Config::" + res + "::AptConfig"]
-    else:
-        return default_apt_config
-

 def which_alias_file():
     hostname = socket.getfqdn()
     aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
 def which_alias_file():
     hostname = socket.getfqdn()
     aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'


@@ -904,7 +894,7 @@ def changes_compare (a, b):
 def find_next_free (dest, too_many=100):
     extra = 0
     orig_dest = dest
 def find_next_free (dest, too_many=100):
     extra = 0
     orig_dest = dest

-    while os.path.exists(dest) and extra < too_many:
+    while os.path.lexists(dest) and extra < too_many:

         dest = orig_dest + '.' + repr(extra)
         extra += 1
     if extra >= too_many:
         dest = orig_dest + '.' + repr(extra)
         extra += 1
     if extra >= too_many:


@@ -1404,13 +1394,61 @@ def gpg_get_key_addresses(fingerprint):
     if result == 0:
         for l in output.split('\n'):
             m = re_gpg_uid.match(l)
     if result == 0:
         for l in output.split('\n'):
             m = re_gpg_uid.match(l)

-            if m:
+            if not m:
+                continue
+            address = m.group(1)
+            if address.endswith('@debian.org'):
+                # prefer @debian.org addresses
+                # TODO: maybe not hardcode the domain
+                addresses.insert(0, address)
+            else:

                 addresses.append(m.group(1))
     key_uid_email_cache[fingerprint] = addresses
     return addresses
 
 ################################################################################
 
                 addresses.append(m.group(1))
     key_uid_email_cache[fingerprint] = addresses
     return addresses
 
 ################################################################################
 

+def get_logins_from_ldap(fingerprint='*'):
+    """retrieve login from LDAP linked to a given fingerprint"""
+
+    LDAPDn = Cnf['Import-LDAP-Fingerprints::LDAPDn']
+    LDAPServer = Cnf['Import-LDAP-Fingerprints::LDAPServer']
+    l = ldap.open(LDAPServer)
+    l.simple_bind_s('','')
+    Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
+                       '(keyfingerprint=%s)' % fingerprint,
+                       ['uid', 'keyfingerprint'])
+    login = {}
+    for elem in Attrs:
+        login[elem[1]['keyFingerPrint'][0]] = elem[1]['uid'][0]
+    return login
+
+################################################################################
+
+def get_users_from_ldap():
+    """retrieve login and user names from LDAP"""
+
+    LDAPDn = Cnf['Import-LDAP-Fingerprints::LDAPDn']
+    LDAPServer = Cnf['Import-LDAP-Fingerprints::LDAPServer']
+    l = ldap.open(LDAPServer)
+    l.simple_bind_s('','')
+    Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
+                       '(uid=*)', ['uid', 'cn', 'mn', 'sn'])
+    users = {}
+    for elem in Attrs:
+        elem = elem[1]
+        name = []
+        for k in ('cn', 'mn', 'sn'):
+            try:
+                if elem[k][0] != '-':
+                    name.append(elem[k][0])
+            except KeyError:
+                pass
+        users[' '.join(name)] = elem['uid'][0]
+    return users
+
+################################################################################
+

 def clean_symlink (src, dest, root):
     """
     Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
 def clean_symlink (src, dest, root):
     """
     Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.


@@ -1424,31 +1462,70 @@ def clean_symlink (src, dest, root):
 
 ################################################################################
 
 
 ################################################################################
 

-def temp_filename(directory=None, prefix="dak", suffix=""):
+def temp_filename(directory=None, prefix="dak", suffix="", mode=None, group=None):

     """
     Return a secure and unique filename by pre-creating it.
     """
     Return a secure and unique filename by pre-creating it.

-    If 'directory' is non-null, it will be the directory the file is pre-created in.
-    If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
-    If 'suffix' is non-null, the filename will end with it.

 
 

-    Returns a pair (fd, name).
+    @type directory: str
+    @param directory: If non-null it will be the directory the file is pre-created in.
+
+    @type prefix: str
+    @param prefix: The filename will be prefixed with this string
+
+    @type suffix: str
+    @param suffix: The filename will end with this string
+
+    @type mode: str
+    @param mode: If set the file will get chmodded to those permissions
+
+    @type group: str
+    @param group: If set the file will get chgrped to the specified group.
+
+    @rtype: list
+    @return: Returns a pair (fd, name)

     """
 
     """
 

-    return tempfile.mkstemp(suffix, prefix, directory)
+    (tfd, tfname) = tempfile.mkstemp(suffix, prefix, directory)
+    if mode:
+        os.chmod(tfname, mode)
+    if group:
+        gid = grp.getgrnam(group).gr_gid
+        os.chown(tfname, -1, gid)
+    return (tfd, tfname)

 
 ################################################################################
 
 
 ################################################################################
 

-def temp_dirname(parent=None, prefix="dak", suffix=""):
+def temp_dirname(parent=None, prefix="dak", suffix="", mode=None, group=None):

     """
     Return a secure and unique directory by pre-creating it.
     """
     Return a secure and unique directory by pre-creating it.

-    If 'parent' is non-null, it will be the directory the directory is pre-created in.
-    If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
-    If 'suffix' is non-null, the filename will end with it.

 
 

-    Returns a pathname to the new directory
+    @type parent: str
+    @param parent: If non-null it will be the directory the directory is pre-created in.
+
+    @type prefix: str
+    @param prefix: The filename will be prefixed with this string
+
+    @type suffix: str
+    @param suffix: The filename will end with this string
+
+    @type mode: str
+    @param mode: If set the file will get chmodded to those permissions
+
+    @type group: str
+    @param group: If set the file will get chgrped to the specified group.
+
+    @rtype: list
+    @return: Returns a pair (fd, name)
+

     """
 
     """
 

-    return tempfile.mkdtemp(suffix, prefix, parent)
+    tfname = tempfile.mkdtemp(suffix, prefix, parent)
+    if mode:
+        os.chmod(tfname, mode)
+    if group:
+        gid = grp.getgrnam(group).gr_gid
+        os.chown(tfname, -1, gid)
+    return tfname

 
 ################################################################################
 
 
 ################################################################################
 


@@ -1485,14 +1562,7 @@ def get_changes_files(from_dir):
 
 ################################################################################
 
 
 ################################################################################
 

-apt_pkg.init()
-
-Cnf = apt_pkg.Configuration()
-if not os.getenv("DAK_TEST"):
-    apt_pkg.read_config_file_isc(Cnf,default_config)
-
-if which_conf_file() != default_config:
-    apt_pkg.read_config_file_isc(Cnf,which_conf_file())
+Cnf = config.Config().Cnf

 
 ################################################################################
 
 
 ################################################################################
 


@@ -1562,7 +1632,7 @@ def get_packages_from_ftp(root, suite, component, architecture):
         if (result != 0):
             fubar("Gunzip invocation failed!\n%s\n" % (output), result)
     packages = open_file(temp_file)
         if (result != 0):
             fubar("Gunzip invocation failed!\n%s\n" % (output), result)
     packages = open_file(temp_file)

-    Packages = apt_pkg.ParseTagFile(packages)
+    Packages = apt_pkg.TagFile(packages)

     os.unlink(temp_file)
     return Packages
 
     os.unlink(temp_file)
     return Packages
 


@@ -1620,7 +1690,7 @@ def call_editor(text="", suffix=".txt"):
     try:
         print >>tmp, text,
         tmp.close()
     try:
         print >>tmp, text,
         tmp.close()

-        subprocess.check_call([editor, tmp.name])
+        daklib.daksubprocess.check_call([editor, tmp.name])

         return open(tmp.name, 'r').read()
     finally:
         os.unlink(tmp.name)
         return open(tmp.name, 'r').read()
     finally:
         os.unlink(tmp.name)


@@ -1629,6 +1699,9 @@ def call_editor(text="", suffix=".txt"):
 
 def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
     dbsuite = get_suite(suite, session)
 
 def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
     dbsuite = get_suite(suite, session)

+    overridesuite = dbsuite
+    if dbsuite.overridesuite is not None:
+        overridesuite = get_suite(dbsuite.overridesuite, session)

     dep_problem = 0
     p2c = {}
     all_broken = {}
     dep_problem = 0
     p2c = {}
     all_broken = {}


@@ -1657,9 +1730,8 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
                 FROM binaries b
                 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
                 JOIN source s ON b.source = s.id
                 FROM binaries b
                 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
                 JOIN source s ON b.source = s.id

-                JOIN files f ON b.file = f.id
-                JOIN location l ON f.location = l.id
-                JOIN component c ON l.component = c.id
+                JOIN files_archive_map af ON b.file = af.file_id
+                JOIN component c ON af.component_id = c.id

                 WHERE b.architecture = :arch_id'''
         query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
             from_statement(statement).params(params)
                 WHERE b.architecture = :arch_id'''
         query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
             from_statement(statement).params(params)


@@ -1694,7 +1766,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
             if package in removals: continue
             parsed_dep = []
             try:
             if package in removals: continue
             parsed_dep = []
             try:

-                parsed_dep += apt_pkg.ParseDepends(deps[package])
+                parsed_dep += apt_pkg.parse_depends(deps[package])

             except ValueError as e:
                 print "Error for package %s: %s" % (package, e)
             for dep in parsed_dep:
             except ValueError as e:
                 print "Error for package %s: %s" % (package, e)
             for dep in parsed_dep:


@@ -1763,7 +1835,7 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
             # Remove [arch] information since we want to see breakage on all arches
             build_dep = re_build_dep_arch.sub("", build_dep)
             try:
             # Remove [arch] information since we want to see breakage on all arches
             build_dep = re_build_dep_arch.sub("", build_dep)
             try:

-                parsed_dep += apt_pkg.ParseDepends(build_dep)
+                parsed_dep += apt_pkg.parse_depends(build_dep)

             except ValueError as e:
                 print "Error for source %s: %s" % (source, e)
         for dep in parsed_dep:
             except ValueError as e:
                 print "Error for source %s: %s" % (source, e)
         for dep in parsed_dep:


@@ -1772,7 +1844,12 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
                 if dep_package in removals:
                     unsat += 1
             if unsat == len(dep):
                 if dep_package in removals:
                     unsat += 1
             if unsat == len(dep):

-                component = DBSource.get(source_id, session).get_component_name()
+                component, = session.query(Component.component_name) \
+                    .join(Component.overrides) \
+                    .filter(Override.suite == overridesuite) \
+                    .filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
+                    .join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
+                    .first()

                 if component != "main":
                     source = "%s/%s" % (source, component)
                 all_broken.setdefault(source, set()).add(pp_deps(dep))
                 if component != "main":
                     source = "%s/%s" % (source, component)
                 all_broken.setdefault(source, set()).add(pp_deps(dep))