Eliminate all references to apt.conf

[dak.git] / daklib / utils.py

diff --git a/daklib/utils.py b/daklib/utils.py
old mode 100644 (file)
new mode 100755 (executable)
index cd5eb79..8a8c69b
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -27,6 +27,7 @@ import datetime
 import email.Header
 import os
 import pwd
 import email.Header
 import os
 import pwd

+import grp

 import select
 import socket
 import shutil
 import select
 import socket
 import shutil


@@ -40,11 +41,14 @@ import time
 import re
 import email as modemail
 import subprocess
 import re
 import email as modemail
 import subprocess

+import ldap

 
 

+import daklib.config as config

 from dbconn import DBConn, get_architecture, get_component, get_suite, \
                    get_override_type, Keyring, session_wrapper, \
                    get_active_keyring_paths, get_primary_keyring_path, \
 from dbconn import DBConn, get_architecture, get_component, get_suite, \
                    get_override_type, Keyring, session_wrapper, \
                    get_active_keyring_paths, get_primary_keyring_path, \

-                   get_suite_architectures, get_or_set_metadatakey, DBSource
+                   get_suite_architectures, get_or_set_metadatakey, DBSource, \
+                   Component, Override, OverrideType

 from sqlalchemy import desc
 from dak_exceptions import *
 from gpg import SignedFile
 from sqlalchemy import desc
 from dak_exceptions import *
 from gpg import SignedFile


@@ -61,7 +65,6 @@ from collections import defaultdict
 ################################################################################
 
 default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties
 ################################################################################
 
 default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties

-default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used

 
 alias_cache = None        #: Cache for email alias checks
 key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids
 
 alias_cache = None        #: Cache for email alias checks
 key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids


@@ -368,7 +371,7 @@ def check_size(where, files):
 
 ################################################################################
 
 
 ################################################################################
 

-def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
+def check_dsc_files(dsc_filename, dsc, dsc_files):

     """
     Verify that the files listed in the Files field of the .dsc are
     those expected given the announced Format.
     """
     Verify that the files listed in the Files field of the .dsc are
     those expected given the announced Format.


@@ -387,13 +390,6 @@ def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
     """
     rejmsg = []
 
     """
     rejmsg = []
 

-    # Parse the file if needed
-    if dsc is None:
-        dsc = parse_changes(dsc_filename, signing_rules=1, dsc_file=1);
-
-    if dsc_files is None:
-        dsc_files = build_file_list(dsc, is_a_dsc=1)
-

     # Ensure .dsc lists proper set of source files according to the format
     # announced
     has = defaultdict(lambda: 0)
     # Ensure .dsc lists proper set of source files according to the format
     # announced
     has = defaultdict(lambda: 0)


@@ -408,7 +404,7 @@ def check_dsc_files(dsc_filename, dsc=None, dsc_files=None):
         (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
     )
 
         (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
     )
 

-    for f in dsc_files.keys():
+    for f in dsc_files:

         m = re_issource.match(f)
         if not m:
             rejmsg.append("%s: %s in Files field not recognised as source."
         m = re_issource.match(f)
         if not m:
             rejmsg.append("%s: %s in Files field not recognised as source."


@@ -607,8 +603,16 @@ def build_package_list(dsc, session = None):
 
 ################################################################################
 
 
 ################################################################################
 

-def send_mail (message, filename=""):
-    """sendmail wrapper, takes _either_ a message string or a file as arguments"""
+def send_mail (message, filename="", whitelists=None):
+    """sendmail wrapper, takes _either_ a message string or a file as arguments
+
+    @type  whitelists: list of (str or None)
+    @param whitelists: path to whitelists. C{None} or an empty list whitelists
+                       everything, otherwise an address is whitelisted if it is
+                       included in any of the lists.
+                       In addition a global whitelist can be specified in
+                       Dinstall::MailWhiteList.
+    """

 
     maildir = Cnf.get('Dir::Mail')
     if maildir:
 
     maildir = Cnf.get('Dir::Mail')
     if maildir:


@@ -628,23 +632,24 @@ def send_mail (message, filename=""):
         os.write (fd, message)
         os.close (fd)
 
         os.write (fd, message)
         os.close (fd)
 

-    if Cnf.has_key("Dinstall::MailWhiteList") and \
-           Cnf["Dinstall::MailWhiteList"] != "":
+    if whitelists is None or None in whitelists:
+        whitelists = []
+    if Cnf.get('Dinstall::MailWhiteList', ''):
+        whitelists.append(Cnf['Dinstall::MailWhiteList'])
+    if len(whitelists) != 0:

         message_in = open_file(filename)
         message_raw = modemail.message_from_file(message_in)
         message_in.close();
 
         whitelist = [];
         message_in = open_file(filename)
         message_raw = modemail.message_from_file(message_in)
         message_in.close();
 
         whitelist = [];

-        whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
-        try:
+        for path in whitelists:
+          with open_file(path, 'r') as whitelist_in:

             for line in whitelist_in:
                 if not re_whitespace_comment.match(line):
                     if re_re_mark.match(line):
                         whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
                     else:
                         whitelist.append(re.compile(re.escape(line.strip())))
             for line in whitelist_in:
                 if not re_whitespace_comment.match(line):
                     if re_re_mark.match(line):
                         whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
                     else:
                         whitelist.append(re.compile(re.escape(line.strip())))

-        finally:
-            whitelist_in.close()

 
         # Fields to check.
         fields = ["To", "Bcc", "Cc"]
 
         # Fields to check.
         fields = ["To", "Bcc", "Cc"]


@@ -661,7 +666,7 @@ def send_mail (message, filename=""):
                             mail_whitelisted = 1
                             break
                     if not mail_whitelisted:
                             mail_whitelisted = 1
                             break
                     if not mail_whitelisted:

-                        print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
+                        print "Skipping {0} since it's not whitelisted".format(item)

                         continue
                     match.append(item)
 
                         continue
                     match.append(item)
 


@@ -701,13 +706,11 @@ def send_mail (message, filename=""):
 
 ################################################################################
 
 
 ################################################################################
 

-def poolify (source, component):
-    if component:
-        component += '/'
+def poolify (source, component=None):

     if source[:3] == "lib":
     if source[:3] == "lib":

-        return component + source[:4] + '/' + source + '/'
+        return source[:4] + '/' + source + '/'

     else:
     else:

-        return component + source[:1] + '/' + source + '/'
+        return source[:1] + '/' + source + '/'

 
 ################################################################################
 
 
 ################################################################################
 


@@ -785,20 +788,6 @@ def which_conf_file ():
 
     return default_config
 
 
     return default_config
 

-def which_apt_conf_file ():
-    res = socket.getfqdn()
-    # In case we allow local config files per user, try if one exists
-    if Cnf.find_b("Config::" + res + "::AllowLocalConfig"):
-        homedir = os.getenv("HOME")
-        confpath = os.path.join(homedir, "/etc/dak.conf")
-        if os.path.exists(confpath):
-            apt_pkg.ReadConfigFileISC(Cnf,default_config)
-
-    if Cnf.get("Config::" + res + "::AptConfig"):
-        return Cnf["Config::" + res + "::AptConfig"]
-    else:
-        return default_apt_config
-

 def which_alias_file():
     hostname = socket.getfqdn()
     aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
 def which_alias_file():
     hostname = socket.getfqdn()
     aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'


@@ -1404,13 +1393,37 @@ def gpg_get_key_addresses(fingerprint):
     if result == 0:
         for l in output.split('\n'):
             m = re_gpg_uid.match(l)
     if result == 0:
         for l in output.split('\n'):
             m = re_gpg_uid.match(l)

-            if m:
+            if not m:
+                continue
+            address = m.group(1)
+            if address.endswith('@debian.org'):
+                # prefer @debian.org addresses
+                # TODO: maybe not hardcode the domain
+                addresses.insert(0, address)
+            else:

                 addresses.append(m.group(1))
     key_uid_email_cache[fingerprint] = addresses
     return addresses
 
 ################################################################################
 
                 addresses.append(m.group(1))
     key_uid_email_cache[fingerprint] = addresses
     return addresses
 
 ################################################################################
 

+def get_logins_from_ldap(fingerprint='*'):
+    """retrieve login from LDAP linked to a given fingerprint"""
+
+    LDAPDn = Cnf['Import-LDAP-Fingerprints::LDAPDn']
+    LDAPServer = Cnf['Import-LDAP-Fingerprints::LDAPServer']
+    l = ldap.open(LDAPServer)
+    l.simple_bind_s('','')
+    Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
+                       '(keyfingerprint=%s)' % fingerprint,
+                       ['uid', 'keyfingerprint'])
+    login = {}
+    for elem in Attrs:
+        login[elem[1]['keyFingerPrint'][0]] = elem[1]['uid'][0]
+    return login
+
+################################################################################
+

 def clean_symlink (src, dest, root):
     """
     Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
 def clean_symlink (src, dest, root):
     """
     Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.


@@ -1424,31 +1437,70 @@ def clean_symlink (src, dest, root):
 
 ################################################################################
 
 
 ################################################################################
 

-def temp_filename(directory=None, prefix="dak", suffix=""):
+def temp_filename(directory=None, prefix="dak", suffix="", mode=None, group=None):

     """
     Return a secure and unique filename by pre-creating it.
     """
     Return a secure and unique filename by pre-creating it.

-    If 'directory' is non-null, it will be the directory the file is pre-created in.
-    If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
-    If 'suffix' is non-null, the filename will end with it.

 
 

-    Returns a pair (fd, name).
+    @type directory: str
+    @param directory: If non-null it will be the directory the file is pre-created in.
+
+    @type prefix: str
+    @param prefix: The filename will be prefixed with this string
+
+    @type suffix: str
+    @param suffix: The filename will end with this string
+
+    @type mode: str
+    @param mode: If set the file will get chmodded to those permissions
+
+    @type group: str
+    @param group: If set the file will get chgrped to the specified group.
+
+    @rtype: list
+    @return: Returns a pair (fd, name)

     """
 
     """
 

-    return tempfile.mkstemp(suffix, prefix, directory)
+    (tfd, tfname) = tempfile.mkstemp(suffix, prefix, directory)
+    if mode:
+        os.chmod(tfname, mode)
+    if group:
+        gid = grp.getgrnam(group).gr_gid
+        os.chown(tfname, -1, gid)
+    return (tfd, tfname)

 
 ################################################################################
 
 
 ################################################################################
 

-def temp_dirname(parent=None, prefix="dak", suffix=""):
+def temp_dirname(parent=None, prefix="dak", suffix="", mode=None, group=None):

     """
     Return a secure and unique directory by pre-creating it.
     """
     Return a secure and unique directory by pre-creating it.

-    If 'parent' is non-null, it will be the directory the directory is pre-created in.
-    If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
-    If 'suffix' is non-null, the filename will end with it.

 
 

-    Returns a pathname to the new directory
+    @type parent: str
+    @param parent: If non-null it will be the directory the directory is pre-created in.
+
+    @type prefix: str
+    @param prefix: The filename will be prefixed with this string
+
+    @type suffix: str
+    @param suffix: The filename will end with this string
+
+    @type mode: str
+    @param mode: If set the file will get chmodded to those permissions
+
+    @type group: str
+    @param group: If set the file will get chgrped to the specified group.
+
+    @rtype: list
+    @return: Returns a pair (fd, name)
+

     """
 
     """
 

-    return tempfile.mkdtemp(suffix, prefix, parent)
+    tfname = tempfile.mkdtemp(suffix, prefix, parent)
+    if mode:
+        os.chmod(tfname, mode)
+    if group:
+        gid = grp.getgrnam(group).gr_gid
+        os.chown(tfname, -1, gid)
+    return tfname

 
 ################################################################################
 
 
 ################################################################################
 


@@ -1485,14 +1537,7 @@ def get_changes_files(from_dir):
 
 ################################################################################
 
 
 ################################################################################
 

-apt_pkg.init()
-
-Cnf = apt_pkg.Configuration()
-if not os.getenv("DAK_TEST"):
-    apt_pkg.read_config_file_isc(Cnf,default_config)
-
-if which_conf_file() != default_config:
-    apt_pkg.read_config_file_isc(Cnf,which_conf_file())
+Cnf = config.Config().Cnf

 
 ################################################################################
 
 
 ################################################################################
 


@@ -1550,7 +1595,6 @@ def get_packages_from_ftp(root, suite, component, architecture):
 
     @rtype: TagFile
     @return: apt_pkg class containing package data
 
     @rtype: TagFile
     @return: apt_pkg class containing package data

-

     """
     filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture)
     (fd, temp_file) = temp_filename()
     """
     filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture)
     (fd, temp_file) = temp_filename()


@@ -1576,15 +1620,20 @@ def deb_extract_control(fh):
 ################################################################################
 
 def mail_addresses_for_upload(maintainer, changed_by, fingerprint):
 ################################################################################
 
 def mail_addresses_for_upload(maintainer, changed_by, fingerprint):

-    """Mail addresses to contact for an upload
+    """mail addresses to contact for an upload
+
+    @type  maintainer: str
+    @param maintainer: Maintainer field of the .changes file

 
 

-    Args:
-       maintainer (str): Maintainer field of the changes file
-       changed_by (str): Changed-By field of the changes file
-       fingerprint (str): Fingerprint of the PGP key used to sign the upload
+    @type  changed_by: str
+    @param changed_by: Changed-By field of the .changes file

 
 

-    Returns:
-       List of RFC 2047-encoded mail addresses to contact regarding this upload
+    @type  fingerprint: str
+    @param fingerprint: fingerprint of the key used to sign the upload
+
+    @rtype:  list of str
+    @return: list of RFC 2047-encoded mail addresses to contact regarding
+             this upload

     """
     addresses = [maintainer]
     if changed_by != maintainer:
     """
     addresses = [maintainer]
     if changed_by != maintainer:


@@ -1600,14 +1649,16 @@ def mail_addresses_for_upload(maintainer, changed_by, fingerprint):
 ################################################################################
 
 def call_editor(text="", suffix=".txt"):
 ################################################################################
 
 def call_editor(text="", suffix=".txt"):

-    """Run editor and return the result as a string
+    """run editor and return the result as a string
+
+    @type  text: str
+    @param text: initial text

 
 

-    Kwargs:
-       text (str): initial text
-       suffix (str): extension for temporary file
+    @type  suffix: str
+    @param suffix: extension for temporary file

 
 

-    Returns:
-       string with the edited text
+    @rtype:  str
+    @return: string with the edited text

     """
     editor = os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi'))
     tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
     """
     editor = os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi'))
     tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)


@@ -1623,6 +1674,9 @@ def call_editor(text="", suffix=".txt"):
 
 def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
     dbsuite = get_suite(suite, session)
 
 def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False):
     dbsuite = get_suite(suite, session)

+    overridesuite = dbsuite
+    if dbsuite.overridesuite is not None:
+        overridesuite = get_suite(dbsuite.overridesuite, session)

     dep_problem = 0
     p2c = {}
     all_broken = {}
     dep_problem = 0
     p2c = {}
     all_broken = {}


@@ -1651,9 +1705,8 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
                 FROM binaries b
                 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
                 JOIN source s ON b.source = s.id
                 FROM binaries b
                 JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
                 JOIN source s ON b.source = s.id

-                JOIN files f ON b.file = f.id
-                JOIN location l ON f.location = l.id
-                JOIN component c ON l.component = c.id
+                JOIN files_archive_map af ON b.file = af.file_id
+                JOIN component c ON af.component_id = c.id

                 WHERE b.architecture = :arch_id'''
         query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
             from_statement(statement).params(params)
                 WHERE b.architecture = :arch_id'''
         query = session.query('id', 'package', 'source', 'component', 'depends', 'provides'). \
             from_statement(statement).params(params)


@@ -1766,7 +1819,12 @@ def check_reverse_depends(removals, suite, arches=None, session=None, cruft=Fals
                 if dep_package in removals:
                     unsat += 1
             if unsat == len(dep):
                 if dep_package in removals:
                     unsat += 1
             if unsat == len(dep):

-                component = DBSource.get(source_id, session).get_component_name()
+                component, = session.query(Component.component_name) \
+                    .join(Component.overrides) \
+                    .filter(Override.suite == overridesuite) \
+                    .filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
+                    .join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
+                    .first()

                 if component != "main":
                     source = "%s/%s" % (source, component)
                 all_broken.setdefault(source, set()).add(pp_deps(dep))
                 if component != "main":
                     source = "%s/%s" % (source, component)
                 all_broken.setdefault(source, set()).add(pp_deps(dep))