]> git.decadent.org.uk Git - dak.git/blob - daklib/binary.py
silent map first
[dak.git] / daklib / binary.py
1 #!/usr/bin/python
2
3 """
4 Functions related debian binary packages
5
6 @contact: Debian FTPMaster <ftpmaster@debian.org>
7 @copyright: 2009  Mike O'Connor <stew@debian.org>
8 @license: GNU General Public License version 2 or later
9 """
10
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
15
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 # GNU General Public License for more details.
20
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24
25 ################################################################################
26
27 # <Ganneff> are we going the xorg way?
28 # <Ganneff> a dak without a dak.conf?
29 # <stew> automatically detect the wrong settings at runtime?
30 # <Ganneff> yes!
31 # <mhy> well, we'll probably always need dak.conf (how do you get the database setting
32 # <mhy> but removing most of the config into the database seems sane
33 # <Ganneff> mhy: dont spoil the fun
34 # <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
35 # <Ganneff> maybe one answers to sql
36 # <stew> we will discover projectb via avahi
37 # <mhy> you're both sick
38 # <mhy> really fucking sick
39
40 ################################################################################
41
42 import os
43 import sys
44 import shutil
45 import tarfile
46 import commands
47 import traceback
48 import atexit
49
50 try:
51     # starting with squeeze
52     from debian import deb822
53 except:
54     # up to lenny
55     from debian_bundle import deb822
56
57 from dbconn import *
58 from config import Config
59 import utils
60
61 ################################################################################
62
63 __all__ = []
64
65 ################################################################################
66
67 class Binary(object):
68     def __init__(self, filename, reject=None):
69         """
70         @type filename: string
71         @param filename: path of a .deb
72
73         @type reject: function
74         @param reject: a function to log reject messages to
75         """
76         self.filename = filename
77         self.tmpdir = None
78         self.chunks = None
79         self.wrapped_reject = reject
80         # Store rejects for later use
81         self.rejects = []
82
83     def reject(self, message):
84         """
85         if we were given a reject function, send the reject message,
86         otherwise send it to stderr.
87         """
88         print >> sys.stderr, message
89         self.rejects.append(message)
90         if self.wrapped_reject:
91             self.wrapped_reject(message)
92
93     def __del__(self):
94         """
95         make sure we cleanup when we are garbage collected.
96         """
97         self._cleanup()
98
99     def _cleanup(self):
100         """
101         we need to remove the temporary directory, if we created one
102         """
103         if self.tmpdir and os.path.exists(self.tmpdir):
104             shutil.rmtree(self.tmpdir)
105             self.tmpdir = None
106
107     def __scan_ar(self):
108         # get a list of the ar contents
109         if not self.chunks:
110
111             cmd = "ar t %s" % (self.filename)
112             (result, output) = commands.getstatusoutput(cmd)
113             if result != 0:
114                 rejected = True
115                 print("%s: 'ar t' invocation failed." % (self.filename))
116                 self.reject("%s: 'ar t' invocation failed." % (self.filename))
117                 self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
118             self.chunks = output.split('\n')
119
120
121
122     def __unpack(self):
123         # Internal function which extracts the contents of the .ar to
124         # a temporary directory
125
126         if not self.tmpdir:
127             tmpdir = utils.temp_dirname()
128             cwd = os.getcwd()
129             try:
130                 os.chdir( tmpdir )
131                 cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
132                 (result, output) = commands.getstatusoutput(cmd)
133                 if result != 0:
134                     print("%s: '%s' invocation failed." % (self.filename, cmd))
135                     self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
136                     self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
137                 else:
138                     self.tmpdir = tmpdir
139                     atexit.register( self._cleanup )
140
141             finally:
142                 os.chdir( cwd )
143
144     def valid_deb(self, relaxed=False):
145         """
146         Check deb contents making sure the .deb contains:
147           1. debian-binary
148           2. control.tar.gz
149           3. data.tar.gz or data.tar.bz2
150         in that order, and nothing else.
151         """
152         self.__scan_ar()
153         rejected = not self.chunks
154         if relaxed:
155             if len(self.chunks) < 3:
156                 rejected = True
157                 self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
158         else:
159             if len(self.chunks) != 3:
160                 rejected = True
161                 self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
162         if self.chunks[0] != "debian-binary":
163             rejected = True
164             self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
165         if not rejected and self.chunks[1] != "control.tar.gz":
166             rejected = True
167             self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
168         if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
169             rejected = True
170             self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
171
172         return not rejected
173
174     def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
175         """
176         Unpack the .deb, do sanity checking, and gather info from it.
177
178         Currently information gathering consists of getting the contents list. In
179         the hopefully near future, it should also include gathering info from the
180         control file.
181
182         @type bootstrap_id: int
183         @param bootstrap_id: the id of the binary these packages
184           should be associated or zero meaning we are not bootstrapping
185           so insert into a temporary table
186
187         @return: True if the deb is valid and contents were imported
188         """
189         result = False
190         rejected = not self.valid_deb(relaxed)
191         if not rejected:
192             self.__unpack()
193
194
195             cwd = os.getcwd()
196             if not rejected and self.tmpdir:
197                 try:
198                     os.chdir(self.tmpdir)
199                     if self.chunks[1] == "control.tar.gz":
200                         control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
201                         control.extract('./control', self.tmpdir )
202                     if self.chunks[2] == "data.tar.gz":
203                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
204                     elif self.chunks[2] == "data.tar.bz2":
205                         data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
206
207                     if bootstrap_id:
208                         result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
209                     else:
210                         pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
211                         pkg = pkgs.next()
212                         result = insert_pending_content_paths(pkg,
213                                                               self.filename.endswith('.udeb'),
214                                                               [tarinfo.name for tarinfo in data if not tarinfo.isdir()],
215                                                               session)
216
217                 except:
218                     traceback.print_exc()
219
220             os.chdir(cwd)
221         self._cleanup()
222         return result
223
224     def check_utf8_package(self, package):
225         """
226         Unpack the .deb, do sanity checking, and gather info from it.
227
228         Currently information gathering consists of getting the contents list. In
229         the hopefully near future, it should also include gathering info from the
230         control file.
231
232         @type package: string
233         @param package: the name of the package to be checked
234
235         @rtype: boolean
236         @return: True if the deb is valid and contents were imported
237         """
238         rejected = not self.valid_deb(True)
239         self.__unpack()
240
241         if not rejected and self.tmpdir:
242             cwd = os.getcwd()
243             try:
244                 os.chdir(self.tmpdir)
245                 if self.chunks[1] == "control.tar.gz":
246                     control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
247                     control.extract('control', self.tmpdir )
248                 if self.chunks[2] == "data.tar.gz":
249                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
250                 elif self.chunks[2] == "data.tar.bz2":
251                     data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
252
253                 for tarinfo in data:
254                     try:
255                         unicode( tarinfo.name )
256                     except:
257                         print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
258
259                 result = True
260
261             except:
262                 traceback.print_exc()
263                 result = False
264
265             os.chdir(cwd)
266
267         return result
268
269 __all__.append('Binary')
270
271
272 def copy_temporary_contents(binary, bin_association, reject, session=None):
273     """
274     copy the previously stored contents from the temp table to the permanant one
275
276     during process-unchecked, the deb should have been scanned and the
277     contents stored in pending_content_associations
278     """
279
280     cnf = Config()
281
282     privatetrans = False
283     if session is None:
284         session = DBConn().session()
285         privatetrans = True
286
287     arch = get_architecture(archname, session=session)
288
289     pending = session.query(PendingBinContents).filter_by(package=binary.package,
290                                                           version=binary.version,
291                                                           arch=binary.arch).first()
292
293     if pending:
294         # This should NOT happen.  We should have added contents
295         # during process-unchecked.  if it did, log an error, and send
296         # an email.
297         subst = {
298             "__PACKAGE__": package,
299             "__VERSION__": version,
300             "__ARCH__": arch,
301             "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
302             "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
303
304         message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
305         utils.send_mail(message)
306
307         # rescan it now
308         exists = Binary(deb, reject).scan_package()
309
310         if not exists:
311             # LOG?
312             return False
313
314     component = binary.poolfile.location.component
315     override = session.query(Override).filter_by(package=binary.package,
316                                                  suite=bin_association.suite,
317                                                  component=component.id).first()
318     if not override:
319         # LOG?
320         return False
321
322
323     if not override.overridetype.type.endswith('deb'):
324         return True
325
326     if override.overridetype.type == "udeb":
327         table = "udeb_contents"
328     elif override.overridetype.type == "deb":
329         table = "deb_contents"
330     else:
331         return False
332
333
334     if component.name == "main":
335         component_str = ""
336     else:
337         component_str = component.name + "/"
338
339     vals = { 'package':binary.package,
340              'version':binary.version,
341              'arch':binary.architecture,
342              'binary_id': binary.id,
343              'component':component_str,
344              'section':override.section.section
345              }
346
347     session.execute( """INSERT INTO %s
348     (binary_id,package,version.component,arch,section,filename)
349     SELECT :binary_id, :package, :version, :component, :arch, :section
350     FROM pending_bin_contents pbc
351     WHERE pbc.package=:package
352     AND pbc.version=:version
353     AND pbc.arch=:arch""" % table, vals )
354
355     session.execute( """DELETE from pending_bin_contents package=:package
356     AND version=:version
357     AND arch=:arch""", vals )
358
359     if privatetrans:
360         session.commit()
361         session.close()
362
363     return exists
364
365 __all__.append('copy_temporary_contents')
366
367