daklib/binary.py

   1 #!/usr/bin/python
   2
   3 """
   4 Functions related debian binary packages
   5
   6 @contact: Debian FTPMaster <ftpmaster@debian.org>
   7 @copyright: 2009  Mike O'Connor <stew@debian.org>
   8 @license: GNU General Public License version 2 or later
   9 """
  10
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # <Ganneff> are we going the xorg way?
  28 # <Ganneff> a dak without a dak.conf?
  29 # <stew> automatically detect the wrong settings at runtime?
  30 # <Ganneff> yes!
  31 # <mhy> well, we'll probably always need dak.conf (how do you get the database setting
  32 # <mhy> but removing most of the config into the database seems sane
  33 # <Ganneff> mhy: dont spoil the fun
  34 # <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
  35 # <Ganneff> maybe one answers to sql
  36 # <stew> we will discover projectb via avahi
  37 # <mhy> you're both sick
  38 # <mhy> really fucking sick
  39
  40 ################################################################################
  41
  42 import os
  43 import sys
  44 import shutil
  45 import tarfile
  46 import commands
  47 import traceback
  48 import atexit
  49
  50 from debian_bundle import deb822
  51
  52 from dbconn import *
  53 from config import Config
  54 import utils
  55
  56 ################################################################################
  57
  58 __all__ = []
  59
  60 ################################################################################
  61
  62 class Binary(object):
  63     def __init__(self, filename, reject=None):
  64         """
  65         @type filename: string
  66         @param filename: path of a .deb
  67
  68         @type reject: function
  69         @param reject: a function to log reject messages to
  70         """
  71         self.filename = filename
  72         self.tmpdir = None
  73         self.chunks = None
  74         self.wrapped_reject = reject
  75
  76     def reject(self, message):
  77         """
  78         if we were given a reject function, send the reject message,
  79         otherwise send it to stderr.
  80         """
  81         print >> sys.stderr, message
  82         if self.wrapped_reject:
  83             self.wrapped_reject(message)
  84
  85     def __del__(self):
  86         """
  87         make sure we cleanup when we are garbage collected.
  88         """
  89         self._cleanup()
  90
  91     def _cleanup(self):
  92         """
  93         we need to remove the temporary directory, if we created one
  94         """
  95         if self.tmpdir and os.path.exists(self.tmpdir):
  96             shutil.rmtree(self.tmpdir)
  97             self.tmpdir = None
  98
  99     def __scan_ar(self):
 100         # get a list of the ar contents
 101         if not self.chunks:
 102
 103             cmd = "ar t %s" % (self.filename)
 104             (result, output) = commands.getstatusoutput(cmd)
 105             if result != 0:
 106                 rejected = True
 107                 print("%s: 'ar t' invocation failed." % (self.filename))
 108                 self.reject("%s: 'ar t' invocation failed." % (self.filename))
 109                 self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
 110             self.chunks = output.split('\n')
 111
 112
 113
 114     def __unpack(self):
 115         # Internal function which extracts the contents of the .ar to
 116         # a temporary directory
 117
 118         if not self.tmpdir:
 119             tmpdir = utils.temp_dirname()
 120             cwd = os.getcwd()
 121             try:
 122                 os.chdir( tmpdir )
 123                 cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
 124                 (result, output) = commands.getstatusoutput(cmd)
 125                 if result != 0:
 126                     print("%s: '%s' invocation failed." % (self.filename, cmd))
 127                     self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
 128                     self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
 129                 else:
 130                     self.tmpdir = tmpdir
 131                     atexit.register( self._cleanup )
 132
 133             finally:
 134                 os.chdir( cwd )
 135
 136     def valid_deb(self, relaxed=False):
 137         """
 138         Check deb contents making sure the .deb contains:
 139           1. debian-binary
 140           2. control.tar.gz
 141           3. data.tar.gz or data.tar.bz2
 142         in that order, and nothing else.
 143         """
 144         self.__scan_ar()
 145         rejected = not self.chunks
 146         if relaxed:
 147             if len(self.chunks) < 3:
 148                 rejected = True
 149                 self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
 150         else:
 151             if len(self.chunks) != 3:
 152                 rejected = True
 153                 self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
 154         if self.chunks[0] != "debian-binary":
 155             rejected = True
 156             self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
 157         if not rejected and self.chunks[1] != "control.tar.gz":
 158             rejected = True
 159             self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
 160         if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
 161             rejected = True
 162             self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
 163
 164         return not rejected
 165
 166     def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
 167         """
 168         Unpack the .deb, do sanity checking, and gather info from it.
 169
 170         Currently information gathering consists of getting the contents list. In
 171         the hopefully near future, it should also include gathering info from the
 172         control file.
 173
 174         @type bootstrap_id: int
 175         @param bootstrap_id: the id of the binary these packages
 176           should be associated or zero meaning we are not bootstrapping
 177           so insert into a temporary table
 178
 179         @return: True if the deb is valid and contents were imported
 180         """
 181         result = False
 182         rejected = not self.valid_deb(relaxed)
 183         if not rejected:
 184             self.__unpack()
 185
 186
 187             cwd = os.getcwd()
 188             if not rejected and self.tmpdir:
 189                 try:
 190                     os.chdir(self.tmpdir)
 191                     if self.chunks[1] == "control.tar.gz":
 192                         control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
 193                         control.extract('./control', self.tmpdir )
 194                     if self.chunks[2] == "data.tar.gz":
 195                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
 196                     elif self.chunks[2] == "data.tar.bz2":
 197                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 198
 199                     if bootstrap_id:
 200                         result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
 201                     else:
 202                         pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
 203                         pkg = pkgs.next()
 204                         result = insert_pending_content_paths(pkg, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
 205
 206                 except:
 207                     traceback.print_exc()
 208
 209             os.chdir(cwd)
 210         self._cleanup()
 211         return result
 212
 213     def check_utf8_package(self, package):
 214         """
 215         Unpack the .deb, do sanity checking, and gather info from it.
 216
 217         Currently information gathering consists of getting the contents list. In
 218         the hopefully near future, it should also include gathering info from the
 219         control file.
 220
 221         @type package: string
 222         @param package: the name of the package to be checked
 223
 224         @rtype: boolean
 225         @return: True if the deb is valid and contents were imported
 226         """
 227         rejected = not self.valid_deb(True)
 228         self.__unpack()
 229
 230         if not rejected and self.tmpdir:
 231             cwd = os.getcwd()
 232             try:
 233                 os.chdir(self.tmpdir)
 234                 if self.chunks[1] == "control.tar.gz":
 235                     control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
 236                     control.extract('control', self.tmpdir )
 237                 if self.chunks[2] == "data.tar.gz":
 238                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
 239                 elif self.chunks[2] == "data.tar.bz2":
 240                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 241
 242                 for tarinfo in data:
 243                     try:
 244                         unicode( tarinfo.name )
 245                     except:
 246                         print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
 247
 248             except:
 249                 traceback.print_exc()
 250                 result = False
 251
 252             os.chdir(cwd)
 253
 254 __all__.append('Binary')
 255
 256 def copy_temporary_contents(package, version, archname, deb, reject, session=None):
 257     """
 258     copy the previously stored contents from the temp table to the permanant one
 259
 260     during process-unchecked, the deb should have been scanned and the
 261     contents stored in pending_content_associations
 262     """
 263
 264     # first see if contents exist:
 265     cnf = Config()
 266
 267     if session is None:
 268         session = DBConn().session()
 269
 270     arch = get_architecture(archname, session=session)
 271
 272     in_pcaq = """SELECT 1 FROM pending_content_associations
 273                                WHERE package=:package
 274                                AND version=:version
 275                                AND architecture=:archid LIMIT 1"""
 276
 277     vals = {'package': package,
 278             'version': version,
 279             'archid': arch.arch_id}
 280
 281     exists = True
 282     check = session.execute(in_pcaq, vals)
 283
 284     if check.rowcount > 0:
 285         # This should NOT happen.  We should have added contents
 286         # during process-unchecked.  if it did, log an error, and send
 287         # an email.
 288         subst = {
 289             "__PACKAGE__": package,
 290             "__VERSION__": version,
 291             "__ARCH__": arch,
 292             "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
 293             "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
 294
 295         message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
 296         utils.send_mail(message)
 297
 298         exists = Binary(deb, reject).scan_package()
 299
 300     if exists:
 301         sql = """INSERT INTO content_associations(binary_pkg,filepath,filename)
 302                  SELECT currval('binaries_id_seq'), filepath, filename FROM pending_content_associations
 303                  WHERE package=:package AND version=:version AND architecture=:archid"""
 304         session.execute(sql, vals)
 305
 306         sql = """DELETE from pending_content_associations
 307                  WHERE package=:package AND version=:version AND architecture=:archid"""
 308         session.execute(sql, vals)
 309
 310     return exists
 311
 312 __all__.append('copy_temporary_contents')