]> git.decadent.org.uk Git - dak.git/blob - daklib/changes.py
daklib/archive.py, daklib/checks.py: implement upload blocks
[dak.git] / daklib / changes.py
1 #!/usr/bin/env python
2 # vim:set et sw=4:
3
4 """
5 Changes class for dak
6
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
10 @copyright: 2009  Mark Hymers <mhy@debian.org>
11 @license: GNU General Public License version 2 or later
12 """
13
14 # This program is free software; you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation; either version 2 of the License, or
17 # (at your option) any later version.
18
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22 # GNU General Public License for more details.
23
24 # You should have received a copy of the GNU General Public License
25 # along with this program; if not, write to the Free Software
26 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
27
28 ###############################################################################
29
30 import os
31 import stat
32
33 import datetime
34 from cPickle import Unpickler, Pickler
35 from errno import EPERM
36
37 from apt_pkg import TagSection
38
39 from utils import open_file, fubar, poolify, deb_extract_control
40 from config import *
41 from dbconn import *
42
43 ###############################################################################
44
45 __all__ = []
46
47 ###############################################################################
48
49 CHANGESFIELDS_MANDATORY = [ "distribution", "source", "architecture",
50         "version", "maintainer", "urgency", "fingerprint", "changedby822",
51         "changedby2047", "changedbyname", "maintainer822", "maintainer2047",
52         "maintainername", "maintaineremail", "closes", "changes" ]
53
54 __all__.append('CHANGESFIELDS_MANDATORY')
55
56 CHANGESFIELDS_OPTIONAL = [ "changed-by", "filecontents", "format",
57         "process-new note", "adv id", "distribution-version", "sponsoremail" ]
58
59 __all__.append('CHANGESFIELDS_OPTIONAL')
60
61 CHANGESFIELDS_FILES = [ "package", "version", "architecture", "type", "size",
62         "md5sum", "sha1sum", "sha256sum", "component", "location id",
63         "source package", "source version", "maintainer", "dbtype", "files id",
64         "new", "section", "priority", "othercomponents", "pool name",
65         "original component" ]
66
67 __all__.append('CHANGESFIELDS_FILES')
68
69 CHANGESFIELDS_DSC = [ "source", "version", "maintainer", "fingerprint",
70         "uploaders", "bts changelog", "dm-upload-allowed" ]
71
72 __all__.append('CHANGESFIELDS_DSC')
73
74 CHANGESFIELDS_DSCFILES_MANDATORY = [ "size", "md5sum" ]
75
76 __all__.append('CHANGESFIELDS_DSCFILES_MANDATORY')
77
78 CHANGESFIELDS_DSCFILES_OPTIONAL = [ "files id" ]
79
80 __all__.append('CHANGESFIELDS_DSCFILES_OPTIONAL')
81
82 CHANGESFIELDS_ORIGFILES = [ "id", "location" ]
83
84 __all__.append('CHANGESFIELDS_ORIGFILES')
85
86 ###############################################################################
87
88 class Changes(object):
89     """ Convenience wrapper to carry around all the package information """
90
91     def __init__(self, **kwds):
92         self.reset()
93
94     def reset(self):
95         self.changes_file = ""
96
97         self.changes = {}
98         self.dsc = {}
99         self.files = {}
100         self.dsc_files = {}
101         self.orig_files = {}
102
103     def file_summary(self):
104         # changes["distribution"] may not exist in corner cases
105         # (e.g. unreadable changes files)
106         if not self.changes.has_key("distribution") or not \
107                isinstance(self.changes["distribution"], dict):
108             self.changes["distribution"] = {}
109
110         byhand = False
111         new = False
112         summary = ""
113         override_summary = ""
114
115         for name, entry in sorted(self.files.items()):
116             if entry.has_key("byhand"):
117                 byhand = True
118                 summary += name + " byhand\n"
119
120             elif entry.has_key("new"):
121                 new = True
122                 summary += "(new) %s %s %s\n" % (name, entry["priority"], entry["section"])
123
124                 if entry.has_key("othercomponents"):
125                     summary += "WARNING: Already present in %s distribution.\n" % (entry["othercomponents"])
126
127                 if entry["type"] == "deb":
128                     deb_fh = open_file(name)
129                     summary += TagSection(deb_extract_control(deb_fh))["Description"] + '\n'
130                     deb_fh.close()
131
132             else:
133                 entry["pool name"] = poolify(self.changes.get("source", ""), entry["component"])
134                 destination = entry["pool name"] + name
135                 summary += name + "\n  to " + destination + "\n"
136
137                 if not entry.has_key("type"):
138                     entry["type"] = "unknown"
139
140                 if entry["type"] in ["deb", "udeb", "dsc"]:
141                     # (queue/unchecked), there we have override entries already, use them
142                     # (process-new), there we dont have override entries, use the newly generated ones.
143                     override_prio = entry.get("override priority", entry["priority"])
144                     override_sect = entry.get("override section", entry["section"])
145                     override_summary += "%s - %s %s\n" % (name, override_prio, override_sect)
146
147         return (byhand, new, summary, override_summary)
148
149     def check_override(self):
150         """
151         Checks override entries for validity.
152
153         Returns an empty string if there are no problems
154         or the text of a warning if there are
155         """
156
157         summary = ""
158
159         # Abandon the check if it's a non-sourceful upload
160         if not self.changes["architecture"].has_key("source"):
161             return summary
162
163         for name, entry in sorted(self.files.items()):
164             if not entry.has_key("new") and entry["type"] == "deb":
165                 if entry["section"] != "-":
166                     if entry["section"].lower() != entry["override section"].lower():
167                         summary += "%s: package says section is %s, override says %s.\n" % (name,
168                                                                                             entry["section"],
169                                                                                             entry["override section"])
170
171                 if entry["priority"] != "-":
172                     if entry["priority"] != entry["override priority"]:
173                         summary += "%s: package says priority is %s, override says %s.\n" % (name,
174                                                                                              entry["priority"],
175                                                                                              entry["override priority"])
176
177         return summary
178
179     @session_wrapper
180     def remove_known_changes(self, session=None):
181         session.delete(get_dbchange(self.changes_file, session))
182
183     def mark_missing_fields(self):
184         """add "missing" in fields which we will require for the known_changes table"""
185         for key in ['urgency', 'maintainer', 'fingerprint', 'changed-by' ]:
186             if (not self.changes.has_key(key)) or (not self.changes[key]):
187                 self.changes[key]='missing'
188
189     def __get_file_from_pool(self, filename, entry, session, logger):
190         cnf = Config()
191
192         if cnf.has_key("Dinstall::SuiteSuffix"):
193             component = cnf["Dinstall::SuiteSuffix"] + entry["component"]
194         else:
195             component = entry["component"]
196
197         poolname = poolify(entry["source"], component)
198         l = get_location(cnf["Dir::Pool"], component, session=session)
199
200         found, poolfile = check_poolfile(os.path.join(poolname, filename),
201                                          entry['size'],
202                                          entry["md5sum"],
203                                          l.location_id,
204                                          session=session)
205
206         if found is None:
207             if logger is not None:
208                 logger.log(["E: Found multiple files for pool (%s) for %s" % (filename, component)])
209             return None
210         elif found is False and poolfile is not None:
211             if logger is not None:
212                 logger.log(["E: md5sum/size mismatch for %s in pool" % (filename)])
213             return None
214         else:
215             if poolfile is None:
216                 if logger is not None:
217                     logger.log(["E: Could not find %s in pool" % (filename)])
218                 return None
219             else:
220                 return poolfile
221
222     @session_wrapper
223     def add_known_changes(self, dirpath, in_queue=None, session=None, logger=None):
224         """add "missing" in fields which we will require for the known_changes table"""
225         cnf = Config()
226
227         changesfile = os.path.join(dirpath, self.changes_file)
228         filetime = datetime.datetime.fromtimestamp(os.path.getctime(changesfile))
229
230         self.mark_missing_fields()
231
232         multivalues = {}
233         for key in ("distribution", "architecture", "binary"):
234             if isinstance(self.changes[key], dict):
235                 multivalues[key] = " ".join(self.changes[key].keys())
236             else:
237                 multivalues[key] = self.changes[key]
238
239         chg = DBChange()
240         chg.changesname = self.changes_file
241         chg.seen = filetime
242         chg.in_queue_id = in_queue
243         chg.source = self.changes["source"]
244         chg.binaries = multivalues["binary"]
245         chg.architecture = multivalues["architecture"]
246         chg.version = self.changes["version"]
247         chg.distribution = multivalues["distribution"]
248         chg.urgency = self.changes["urgency"]
249         chg.maintainer = self.changes["maintainer"]
250         chg.fingerprint = self.changes["fingerprint"]
251         chg.changedby = self.changes["changed-by"]
252         chg.date = self.changes["date"]
253
254         session.add(chg)
255
256         files = []
257         for chg_fn, entry in self.files.items():
258             try:
259                 f = open(os.path.join(dirpath, chg_fn))
260                 cpf = ChangePendingFile()
261                 cpf.filename = chg_fn
262                 cpf.size = entry['size']
263                 cpf.md5sum = entry['md5sum']
264
265                 if entry.has_key('sha1sum'):
266                     cpf.sha1sum = entry['sha1sum']
267                 else:
268                     f.seek(0)
269                     cpf.sha1sum = apt_pkg.sha1sum(f)
270
271                 if entry.has_key('sha256sum'):
272                     cpf.sha256sum = entry['sha256sum']
273                 else:
274                     f.seek(0)
275                     cpf.sha256sum = apt_pkg.sha256sum(f)
276
277                 session.add(cpf)
278                 files.append(cpf)
279                 f.close()
280
281             except IOError:
282                 # Can't find the file, try to look it up in the pool
283                 poolfile = self.__get_file_from_pool(chg_fn, entry, session)
284                 if poolfile:
285                     chg.poolfiles.append(poolfile)
286
287         chg.files = files
288
289         # Add files referenced in .dsc, but not included in .changes
290         for name, entry in self.dsc_files.items():
291             if self.files.has_key(name):
292                 continue
293
294             entry['source'] = self.changes['source']
295             poolfile = self.__get_file_from_pool(name, entry, session, logger)
296             if poolfile:
297                 chg.poolfiles.append(poolfile)
298
299         session.commit()
300         chg = session.query(DBChange).filter_by(changesname = self.changes_file).one();
301
302         return chg
303
304     def unknown_files_fields(self, name):
305         return sorted(list( set(self.files[name].keys()) -
306                             set(CHANGESFIELDS_FILES)))
307
308     def unknown_changes_fields(self):
309         return sorted(list( set(self.changes.keys()) -
310                             set(CHANGESFIELDS_MANDATORY + CHANGESFIELDS_OPTIONAL)))
311
312     def unknown_dsc_fields(self):
313         return sorted(list( set(self.dsc.keys()) -
314                             set(CHANGESFIELDS_DSC)))
315
316     def unknown_dsc_files_fields(self, name):
317         return sorted(list( set(self.dsc_files[name].keys()) -
318                             set(CHANGESFIELDS_DSCFILES_MANDATORY + CHANGESFIELDS_DSCFILES_OPTIONAL)))
319
320     def str_files(self):
321         r = []
322         for name, entry in self.files.items():
323             r.append("  %s:" % (name))
324             for i in CHANGESFIELDS_FILES:
325                 if entry.has_key(i):
326                     r.append("   %s: %s" % (i.capitalize(), entry[i]))
327             xfields = self.unknown_files_fields(name)
328             if len(xfields) > 0:
329                 r.append("files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
330
331         return r
332
333     def str_changes(self):
334         r = []
335         for i in CHANGESFIELDS_MANDATORY:
336             val = self.changes[i]
337             if isinstance(val, list):
338                 val = " ".join(val)
339             elif isinstance(val, dict):
340                 val = " ".join(val.keys())
341             r.append('  %s: %s' % (i.capitalize(), val))
342
343         for i in CHANGESFIELDS_OPTIONAL:
344             if self.changes.has_key(i):
345                 r.append('  %s: %s' % (i.capitalize(), self.changes[i]))
346
347         xfields = self.unknown_changes_fields()
348         if len(xfields) > 0:
349             r.append("Warning: changes still has the following unrecognised fields: %s" % ", ".join(xfields))
350
351         return r
352
353     def str_dsc(self):
354         r = []
355         for i in CHANGESFIELDS_DSC:
356             if self.dsc.has_key(i):
357                 r.append('  %s: %s' % (i.capitalize(), self.dsc[i]))
358
359         xfields = self.unknown_dsc_fields()
360         if len(xfields) > 0:
361             r.append("Warning: dsc still has the following unrecognised fields: %s" % ", ".join(xfields))
362
363         return r
364
365     def str_dsc_files(self):
366         r = []
367         for name, entry in self.dsc_files.items():
368             r.append("  %s:" % (name))
369             for i in CHANGESFIELDS_DSCFILES_MANDATORY:
370                 r.append("   %s: %s" % (i.capitalize(), entry[i]))
371             for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
372                 if entry.has_key(i):
373                     r.append("   %s: %s" % (i.capitalize(), entry[i]))
374             xfields = self.unknown_dsc_files_fields(name)
375             if len(xfields) > 0:
376                 r.append("dsc_files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
377
378         return r
379
380     def __str__(self):
381         r = []
382
383         r.append(" Changes:")
384         r += self.str_changes()
385
386         r.append("")
387
388         r.append(" Dsc:")
389         r += self.str_dsc()
390
391         r.append("")
392
393         r.append(" Files:")
394         r += self.str_files()
395
396         r.append("")
397
398         r.append(" Dsc Files:")
399         r += self.str_dsc_files()
400
401         return "\n".join(r)
402
403 __all__.append('Changes')