import email as modemail
import subprocess
-from dbconn import DBConn, get_architecture, get_component, get_suite
+from dbconn import DBConn, get_architecture, get_component, get_suite, get_override_type, Keyring, session_wrapper
from dak_exceptions import *
+from gpg import SignedFile
from textutils import fix_maintainer
from regexes import re_html_escaping, html_escaping, re_single_line_field, \
re_multi_line_field, re_srchasver, re_taint_free, \
- re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource
+ re_gpg_uid, re_re_mark, re_whitespace_comment, re_issource, \
+ re_is_orig_source
from formats import parse_format, validate_changes_format
from srcformats import get_format_from_string
known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc
-# Monkeypatch commands.getstatusoutput as it returns a "0" exit code in
-# all situations under lenny's Python.
-import commands
+# Monkeypatch commands.getstatusoutput as it may not return the correct exit
+# code in lenny's Python. This also affects commands.getoutput and
+# commands.getstatus.
def dak_getstatusoutput(cmd):
pipe = subprocess.Popen(cmd, shell=True, universal_newlines=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = "".join(pipe.stdout.readlines())
+ output = pipe.stdout.read()
+
+ pipe.wait()
if output[-1:] == '\n':
output = output[:-1]
def our_raw_input(prompt=""):
if prompt:
- sys.stdout.write(prompt)
+ while 1:
+ try:
+ sys.stdout.write(prompt)
+ break
+ except IOError:
+ pass
sys.stdout.flush()
try:
ret = raw_input()
################################################################################
-def parse_deb822(contents, signing_rules=0):
+def parse_deb822(armored_contents, signing_rules=0, keyrings=None, session=None):
+ require_signature = True
+ if keyrings == None:
+ keyrings = []
+ require_signature = False
+
+ signed_file = SignedFile(armored_contents, keyrings=keyrings, require_signature=require_signature)
+ contents = signed_file.contents
+
error = ""
changes = {}
index += 1
indexed_lines[index] = line[:-1]
- inside_signature = 0
-
num_of_lines = len(indexed_lines.keys())
index = 0
first = -1
while index < num_of_lines:
index += 1
line = indexed_lines[index]
- if line == "":
- if signing_rules == 1:
- index += 1
- if index > num_of_lines:
- raise InvalidDscError, index
- line = indexed_lines[index]
- if not line.startswith("-----BEGIN PGP SIGNATURE"):
- raise InvalidDscError, index
- inside_signature = 0
- break
- else:
- continue
- if line.startswith("-----BEGIN PGP SIGNATURE"):
+ if line == "" and signing_rules == 1:
+ if index != num_of_lines:
+ raise InvalidDscError, index
break
- if line.startswith("-----BEGIN PGP SIGNED MESSAGE"):
- inside_signature = 1
- if signing_rules == 1:
- while index < num_of_lines and line != "":
- index += 1
- line = indexed_lines[index]
- continue
- # If we're not inside the signed data, don't process anything
- if signing_rules >= 0 and not inside_signature:
- continue
slf = re_single_line_field.match(line)
if slf:
field = slf.groups()[0].lower()
continue
error += line
- if signing_rules == 1 and inside_signature:
- raise InvalidDscError, index
-
- changes["filecontents"] = "".join(lines)
+ changes["filecontents"] = armored_contents
if changes.has_key("source"):
# Strip the source version in brackets from the source field,
################################################################################
-def parse_changes(filename, signing_rules=0):
+def parse_changes(filename, signing_rules=0, dsc_file=0, keyrings=None):
"""
Parses a changes file and returns a dictionary where each field is a
key. The mandatory first argument is the filename of the .changes
unicode(content, 'utf-8')
except UnicodeError:
raise ChangesUnicodeError, "Changes file not proper utf-8"
- return parse_deb822(content, signing_rules)
+ changes = parse_deb822(content, signing_rules, keyrings=keyrings)
+
+
+ if not dsc_file:
+ # Finally ensure that everything needed for .changes is there
+ must_keywords = ('Format', 'Date', 'Source', 'Binary', 'Architecture', 'Version',
+ 'Distribution', 'Maintainer', 'Description', 'Changes', 'Files')
+
+ missingfields=[]
+ for keyword in must_keywords:
+ if not changes.has_key(keyword.lower()):
+ missingfields.append(keyword)
+
+ if len(missingfields):
+ raise ParseChangesError, "Missing mandantory field(s) in changes file (policy 5.5): %s" % (missingfields)
+
+ return changes
################################################################################
# Parse the file if needed
if dsc is None:
- dsc = parse_changes(dsc_filename, signing_rules=1);
+ dsc = parse_changes(dsc_filename, signing_rules=1, dsc_file=1);
if dsc_files is None:
dsc_files = build_file_list(dsc, is_a_dsc=1)
(r'orig.tar.gz', ('orig_tar_gz', 'orig_tar')),
(r'diff.gz', ('debian_diff',)),
(r'tar.gz', ('native_tar_gz', 'native_tar')),
- (r'debian\.tar\.(gz|bz2)', ('debian_tar',)),
- (r'orig\.tar\.(gz|bz2)', ('orig_tar',)),
- (r'tar\.(gz|bz2)', ('native_tar',)),
- (r'orig-.+\.tar\.(gz|bz2)', ('more_orig_tar',)),
+ (r'debian\.tar\.(gz|bz2|xz)', ('debian_tar',)),
+ (r'orig\.tar\.(gz|bz2|xz)', ('orig_tar',)),
+ (r'tar\.(gz|bz2|xz)', ('native_tar',)),
+ (r'orig-.+\.tar\.(gz|bz2|xz)', ('more_orig_tar',)),
)
for f in dsc_files.keys():
(section, component) = extract_component_from_section(section)
- files[name] = Dict(size=size, section=section,
+ files[name] = dict(size=size, section=section,
priority=priority, component=component)
files[name][hashname] = md5
################################################################################
+# see http://bugs.debian.org/619131
+def build_package_set(dsc, session = None):
+ if not dsc.has_key("package-set"):
+ return {}
+
+ packages = {}
+
+ for line in dsc["package-set"].split("\n"):
+ if not line:
+ break
+
+ (name, section, priority) = line.split()
+ (section, component) = extract_component_from_section(section)
+
+ package_type = "deb"
+ if name.find(":") != -1:
+ (package_type, name) = name.split(":", 1)
+ if package_type == "src":
+ package_type = "dsc"
+
+ # Validate type if we have a session
+ if session and get_override_type(package_type, session) is None:
+ # Maybe just warn and ignore? exit(1) might be a bit hard...
+ utils.fubar("invalid type (%s) in Package-Set." % (package_type))
+
+ if section == "":
+ section = "-"
+ if priority == "":
+ priority = "-"
+
+ if package_type == "dsc":
+ priority = "source"
+
+ if not packages.has_key(name) or packages[name]["type"] == "dsc":
+ packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[])
+
+ return packages
+
+################################################################################
+
def send_mail (message, filename=""):
"""sendmail wrapper, takes _either_ a message string or a file as arguments"""
+ # Check whether we're supposed to be sending mail
+ if Cnf.has_key("Dinstall::Options::No-Mail") and Cnf["Dinstall::Options::No-Mail"]:
+ return
+
# If we've been passed a string dump it into a temporary file
if message:
(fd, filename) = tempfile.mkstemp()
################################################################################
def where_am_i ():
- res = socket.gethostbyaddr(socket.gethostname())
- database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname")
+ res = socket.getfqdn()
+ database_hostname = Cnf.get("Config::" + res + "::DatabaseHostname")
if database_hostname:
return database_hostname
else:
- return res[0]
+ return res
def which_conf_file ():
- if os.getenv("DAK_CONFIG"):
- print(os.getenv("DAK_CONFIG"))
- return os.getenv("DAK_CONFIG")
- else:
- res = socket.gethostbyaddr(socket.gethostname())
- # In case we allow local config files per user, try if one exists
- if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
- homedir = os.getenv("HOME")
- confpath = os.path.join(homedir, "/etc/dak.conf")
- if os.path.exists(confpath):
- apt_pkg.ReadConfigFileISC(Cnf,default_config)
-
- # We are still in here, so there is no local config file or we do
- # not allow local files. Do the normal stuff.
- if Cnf.get("Config::" + res[0] + "::DakConfig"):
- return Cnf["Config::" + res[0] + "::DakConfig"]
- else:
- return default_config
+ if os.getenv('DAK_CONFIG'):
+ return os.getenv('DAK_CONFIG')
+
+ res = socket.getfqdn()
+ # In case we allow local config files per user, try if one exists
+ if Cnf.FindB("Config::" + res + "::AllowLocalConfig"):
+ homedir = os.getenv("HOME")
+ confpath = os.path.join(homedir, "/etc/dak.conf")
+ if os.path.exists(confpath):
+ apt_pkg.ReadConfigFileISC(Cnf,default_config)
+
+ # We are still in here, so there is no local config file or we do
+ # not allow local files. Do the normal stuff.
+ if Cnf.get("Config::" + res + "::DakConfig"):
+ return Cnf["Config::" + res + "::DakConfig"]
+
+ return default_config
def which_apt_conf_file ():
- res = socket.gethostbyaddr(socket.gethostname())
+ res = socket.getfqdn()
# In case we allow local config files per user, try if one exists
- if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
+ if Cnf.FindB("Config::" + res + "::AllowLocalConfig"):
homedir = os.getenv("HOME")
confpath = os.path.join(homedir, "/etc/dak.conf")
if os.path.exists(confpath):
apt_pkg.ReadConfigFileISC(Cnf,default_config)
- if Cnf.get("Config::" + res[0] + "::AptConfig"):
- return Cnf["Config::" + res[0] + "::AptConfig"]
+ if Cnf.get("Config::" + res + "::AptConfig"):
+ return Cnf["Config::" + res + "::AptConfig"]
else:
return default_apt_config
def which_alias_file():
- hostname = socket.gethostbyaddr(socket.gethostname())[0]
+ hostname = socket.getfqdn()
aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
if os.path.exists(aliasfn):
return aliasfn
################################################################################
-def TemplateSubst(map, filename):
+def TemplateSubst(subst_map, filename):
""" Perform a substition of template """
templatefile = open_file(filename)
template = templatefile.read()
- for x in map.keys():
- template = template.replace(x, str(map[x]))
+ for k, v in subst_map.iteritems():
+ template = template.replace(k, str(v))
templatefile.close()
return template
################################################################################
-def Dict(**dict): return dict
-
-########################################
-
def gpgv_get_status_output(cmd, status_read, status_write):
"""
Our very own version of commands.getouputstatus(), hacked to support
return " ".join(["--keyring %s" % x for x in keyrings])
################################################################################
-
-def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None):
+@session_wrapper
+def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None, session=None):
"""
Check the signature of a file and return the fingerprint if the
signature is valid or 'None' if it's not. The first argument is the
return (None, rejects)
if not keyrings:
- keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
+ keyrings = [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).all() ]
# Autofetch the signing key if that's enabled
if autofetch == None:
if exit_status:
rejects.append("gpgv failed while checking %s." % (sig_filename))
if status.strip():
- rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
+ rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "))
else:
- rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "")
+ rejects.append(prefix_multi_line_string(output, " [GPG output:] "))
return (None, rejects)
# Sanity check the good stuff we expect
rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
# Finally ensure there's not something we don't recognise
- known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
+ known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
- NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="")
+ NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="")
for keyword in keywords.keys():
if not known_keywords.has_key(keyword):
apt_pkg.init()
Cnf = apt_pkg.newConfiguration()
-apt_pkg.ReadConfigFileISC(Cnf,default_config)
+if not os.getenv("DAK_TEST"):
+ apt_pkg.ReadConfigFileISC(Cnf,default_config)
if which_conf_file() != default_config:
apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
+
+################################################################################
+
+def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm"):
+ """
+ Parses the wnpp bug list available at http://qa.debian.org/data/bts/wnpp_rm
+ Well, actually it parsed a local copy, but let's document the source
+ somewhere ;)
+
+ returns a dict associating source package name with a list of open wnpp
+ bugs (Yes, there might be more than one)
+ """
+
+ line = []
+ try:
+ f = open(file)
+ lines = f.readlines()
+ except IOError, e:
+ print "Warning: Couldn't open %s; don't know about WNPP bugs, so won't close any." % file
+ lines = []
+ wnpp = {}
+
+ for line in lines:
+ splited_line = line.split(": ", 1)
+ if len(splited_line) > 1:
+ wnpp[splited_line[0]] = splited_line[1].split("|")
+
+ for source in wnpp.keys():
+ bugs = []
+ for wnpp_bug in wnpp[source]:
+ bug_no = re.search("(\d)+", wnpp_bug).group()
+ if bug_no:
+ bugs.append(bug_no)
+ wnpp[source] = bugs
+ return wnpp
+
+################################################################################
+
+def get_packages_from_ftp(root, suite, component, architecture):
+ """
+ Returns an object containing apt_pkg-parseable data collected by
+ aggregating Packages.gz files gathered for each architecture.
+
+ @type root: string
+ @param root: path to ftp archive root directory
+
+ @type suite: string
+ @param suite: suite to extract files from
+
+ @type component: string
+ @param component: component to extract files from
+
+ @type architecture: string
+ @param architecture: architecture to extract files from
+
+ @rtype: TagFile
+ @return: apt_pkg class containing package data
+
+ """
+ filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture)
+ (fd, temp_file) = temp_filename()
+ (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_file))
+ if (result != 0):
+ fubar("Gunzip invocation failed!\n%s\n" % (output), result)
+ filename = "%s/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % (root, suite, component, architecture)
+ if os.path.exists(filename):
+ (result, output) = commands.getstatusoutput("gunzip -c %s >> %s" % (filename, temp_file))
+ if (result != 0):
+ fubar("Gunzip invocation failed!\n%s\n" % (output), result)
+ packages = open_file(temp_file)
+ Packages = apt_pkg.ParseTagFile(packages)
+ os.unlink(temp_file)
+ return Packages
projects / dak.git / blobdiff