daklib/binary.py

   1 #!/usr/bin/python
   2
   3 """
   4 Functions related debian binary packages
   5
   6 @contact: Debian FTPMaster <ftpmaster@debian.org>
   7 @copyright: 2009  Mike O'Connor <stew@debian.org>
   8 @license: GNU General Public License version 2 or later
   9 """
  10
  11 # This program is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version 2 of the License, or
  14 # (at your option) any later version.
  15
  16 # This program is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20
  21 # You should have received a copy of the GNU General Public License
  22 # along with this program; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 ################################################################################
  26
  27 # <Ganneff> are we going the xorg way?
  28 # <Ganneff> a dak without a dak.conf?
  29 # <stew> automatically detect the wrong settings at runtime?
  30 # <Ganneff> yes!
  31 # <mhy> well, we'll probably always need dak.conf (how do you get the database setting
  32 # <mhy> but removing most of the config into the database seems sane
  33 # <Ganneff> mhy: dont spoil the fun
  34 # <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
  35 # <Ganneff> maybe one answers to sql
  36 # <stew> we will discover projectb via avahi
  37 # <mhy> you're both sick
  38 # <mhy> really fucking sick
  39
  40 ################################################################################
  41
  42 import os
  43 import sys
  44 import shutil
  45 import tarfile
  46 import commands
  47 import traceback
  48 import atexit
  49
  50 from debian_bundle import deb822
  51
  52 from dbconn import *
  53 from config import Config
  54 import utils
  55
  56 ################################################################################
  57
  58 __all__ = []
  59
  60 ################################################################################
  61
  62 class Binary(object):
  63     def __init__(self, filename, reject=None):
  64         """
  65         @type filename: string
  66         @param filename: path of a .deb
  67
  68         @type reject: function
  69         @param reject: a function to log reject messages to
  70         """
  71         self.filename = filename
  72         self.tmpdir = None
  73         self.chunks = None
  74         self.wrapped_reject = reject
  75         # Store rejects for later use
  76         self.rejects = []
  77
  78     def reject(self, message):
  79         """
  80         if we were given a reject function, send the reject message,
  81         otherwise send it to stderr.
  82         """
  83         print >> sys.stderr, message
  84         self.rejects.append(message)
  85         if self.wrapped_reject:
  86             self.wrapped_reject(message)
  87
  88     def __del__(self):
  89         """
  90         make sure we cleanup when we are garbage collected.
  91         """
  92         self._cleanup()
  93
  94     def _cleanup(self):
  95         """
  96         we need to remove the temporary directory, if we created one
  97         """
  98         if self.tmpdir and os.path.exists(self.tmpdir):
  99             shutil.rmtree(self.tmpdir)
 100             self.tmpdir = None
 101
 102     def __scan_ar(self):
 103         # get a list of the ar contents
 104         if not self.chunks:
 105
 106             cmd = "ar t %s" % (self.filename)
 107             (result, output) = commands.getstatusoutput(cmd)
 108             if result != 0:
 109                 rejected = True
 110                 print("%s: 'ar t' invocation failed." % (self.filename))
 111                 self.reject("%s: 'ar t' invocation failed." % (self.filename))
 112                 self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
 113             self.chunks = output.split('\n')
 114
 115
 116
 117     def __unpack(self):
 118         # Internal function which extracts the contents of the .ar to
 119         # a temporary directory
 120
 121         if not self.tmpdir:
 122             tmpdir = utils.temp_dirname()
 123             cwd = os.getcwd()
 124             try:
 125                 os.chdir( tmpdir )
 126                 cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
 127                 (result, output) = commands.getstatusoutput(cmd)
 128                 if result != 0:
 129                     print("%s: '%s' invocation failed." % (self.filename, cmd))
 130                     self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
 131                     self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
 132                 else:
 133                     self.tmpdir = tmpdir
 134                     atexit.register( self._cleanup )
 135
 136             finally:
 137                 os.chdir( cwd )
 138
 139     def valid_deb(self, relaxed=False):
 140         """
 141         Check deb contents making sure the .deb contains:
 142           1. debian-binary
 143           2. control.tar.gz
 144           3. data.tar.gz or data.tar.bz2
 145         in that order, and nothing else.
 146         """
 147         self.__scan_ar()
 148         rejected = not self.chunks
 149         if relaxed:
 150             if len(self.chunks) < 3:
 151                 rejected = True
 152                 self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
 153         else:
 154             if len(self.chunks) != 3:
 155                 rejected = True
 156                 self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
 157         if self.chunks[0] != "debian-binary":
 158             rejected = True
 159             self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
 160         if not rejected and self.chunks[1] != "control.tar.gz":
 161             rejected = True
 162             self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
 163         if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
 164             rejected = True
 165             self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
 166
 167         return not rejected
 168
 169     def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
 170         """
 171         Unpack the .deb, do sanity checking, and gather info from it.
 172
 173         Currently information gathering consists of getting the contents list. In
 174         the hopefully near future, it should also include gathering info from the
 175         control file.
 176
 177         @type bootstrap_id: int
 178         @param bootstrap_id: the id of the binary these packages
 179           should be associated or zero meaning we are not bootstrapping
 180           so insert into a temporary table
 181
 182         @return: True if the deb is valid and contents were imported
 183         """
 184         result = False
 185         rejected = not self.valid_deb(relaxed)
 186         if not rejected:
 187             self.__unpack()
 188
 189
 190             cwd = os.getcwd()
 191             if not rejected and self.tmpdir:
 192                 try:
 193                     os.chdir(self.tmpdir)
 194                     if self.chunks[1] == "control.tar.gz":
 195                         control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
 196                         control.extract('./control', self.tmpdir )
 197                     if self.chunks[2] == "data.tar.gz":
 198                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
 199                     elif self.chunks[2] == "data.tar.bz2":
 200                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 201
 202                     if bootstrap_id:
 203                         result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
 204                     else:
 205                         pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
 206                         pkg = pkgs.next()
 207                         result = insert_pending_content_paths(pkg,
 208                                                               self.filename.endswith('.udeb'),
 209                                                               [tarinfo.name for tarinfo in data if not tarinfo.isdir()],
 210                                                               session)
 211
 212                 except:
 213                     traceback.print_exc()
 214
 215             os.chdir(cwd)
 216         self._cleanup()
 217         return result
 218
 219     def check_utf8_package(self, package):
 220         """
 221         Unpack the .deb, do sanity checking, and gather info from it.
 222
 223         Currently information gathering consists of getting the contents list. In
 224         the hopefully near future, it should also include gathering info from the
 225         control file.
 226
 227         @type package: string
 228         @param package: the name of the package to be checked
 229
 230         @rtype: boolean
 231         @return: True if the deb is valid and contents were imported
 232         """
 233         rejected = not self.valid_deb(True)
 234         self.__unpack()
 235
 236         if not rejected and self.tmpdir:
 237             cwd = os.getcwd()
 238             try:
 239                 os.chdir(self.tmpdir)
 240                 if self.chunks[1] == "control.tar.gz":
 241                     control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
 242                     control.extract('control', self.tmpdir )
 243                 if self.chunks[2] == "data.tar.gz":
 244                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
 245                 elif self.chunks[2] == "data.tar.bz2":
 246                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
 247
 248                 for tarinfo in data:
 249                     try:
 250                         unicode( tarinfo.name )
 251                     except:
 252                         print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
 253
 254                 result = True
 255
 256             except:
 257                 traceback.print_exc()
 258                 result = False
 259
 260             os.chdir(cwd)
 261
 262         return result
 263
 264 __all__.append('Binary')
 265
 266
 267 def copy_temporary_contents(binary, bin_association, reject, session=None):
 268     """
 269     copy the previously stored contents from the temp table to the permanant one
 270
 271     during process-unchecked, the deb should have been scanned and the
 272     contents stored in pending_content_associations
 273     """
 274
 275     cnf = Config()
 276
 277     privatetrans = False
 278     if session is None:
 279         session = DBConn().session()
 280         privatetrans = True
 281
 282     arch = get_architecture(archname, session=session)
 283
 284     pending = session.query(PendingBinContents).filter_by(package=binary.package,
 285                                                           version=binary.version,
 286                                                           arch=binary.arch).first()
 287
 288     if pending:
 289         # This should NOT happen.  We should have added contents
 290         # during process-unchecked.  if it did, log an error, and send
 291         # an email.
 292         subst = {
 293             "__PACKAGE__": package,
 294             "__VERSION__": version,
 295             "__ARCH__": arch,
 296             "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
 297             "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
 298
 299         message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
 300         utils.send_mail(message)
 301
 302         # rescan it now
 303         exists = Binary(deb, reject).scan_package()
 304
 305         if not exists:
 306             # LOG?
 307             return False
 308
 309     component = binary.poolfile.location.component
 310     override = session.query(Override).filter_by(package=binary.package,
 311                                                  suite=bin_association.suite,
 312                                                  component=component.id).first()
 313     if not override:
 314         # LOG?
 315         return False
 316
 317
 318     if not override.overridetype.type.endswith('deb'):
 319         return True
 320
 321     if override.overridetype.type == "udeb":
 322         table = "udeb_contents"
 323     elif override.overridetype.type == "deb":
 324         table = "deb_contents"
 325     else:
 326         return False
 327
 328
 329     if component.name == "main":
 330         component_str = ""
 331     else:
 332         component_str = component.name + "/"
 333
 334     vals = { 'package':binary.package,
 335              'version':binary.version,
 336              'arch':binary.architecture,
 337              'binary_id': binary.id,
 338              'component':component_str,
 339              'section':override.section.section
 340              }
 341
 342     session.execute( """INSERT INTO %s
 343     (binary_id,package,version.component,arch,section,filename)
 344     SELECT :binary_id, :package, :version, :component, :arch, :section
 345     FROM pending_bin_contents pbc
 346     WHERE pbc.package=:package
 347     AND pbc.version=:version
 348     AND pbc.arch=:arch""" % table, vals )
 349
 350     session.execute( """DELETE from pending_bin_contents package=:package
 351     AND version=:version
 352     AND arch=:arch""", vals )
 353
 354     if privatetrans:
 355         session.commit()
 356         session.close()
 357
 358     return exists
 359
 360 __all__.append('copy_temporary_contents')
 361
 362