]> git.decadent.org.uk Git - dak.git/blob - daklib/changes.py
Merge remote branch 'drkranz/master' into merge
[dak.git] / daklib / changes.py
1 #!/usr/bin/env python
2 # vim:set et sw=4:
3
4 """
5 Changes class for dak
6
7 @contact: Debian FTP Master <ftpmaster@debian.org>
8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
10 @copyright: 2009  Mark Hymers <mhy@debian.org>
11 @license: GNU General Public License version 2 or later
12 """
13
14 # This program is free software; you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation; either version 2 of the License, or
17 # (at your option) any later version.
18
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22 # GNU General Public License for more details.
23
24 # You should have received a copy of the GNU General Public License
25 # along with this program; if not, write to the Free Software
26 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
27
28 ###############################################################################
29
30 import os
31 import stat
32
33 import datetime
34 from cPickle import Unpickler, Pickler
35 from errno import EPERM
36
37 from apt_inst import debExtractControl
38 from apt_pkg import ParseSection
39
40 from utils import open_file, fubar, poolify
41 from config import *
42 from dbconn import *
43
44 ###############################################################################
45
46 __all__ = []
47
48 ###############################################################################
49
50 CHANGESFIELDS_MANDATORY = [ "distribution", "source", "architecture",
51         "version", "maintainer", "urgency", "fingerprint", "changedby822",
52         "changedby2047", "changedbyname", "maintainer822", "maintainer2047",
53         "maintainername", "maintaineremail", "closes", "changes" ]
54
55 __all__.append('CHANGESFIELDS_MANDATORY')
56
57 CHANGESFIELDS_OPTIONAL = [ "changed-by", "filecontents", "format",
58         "process-new note", "adv id", "distribution-version", "sponsoremail" ]
59
60 __all__.append('CHANGESFIELDS_OPTIONAL')
61
62 CHANGESFIELDS_FILES = [ "package", "version", "architecture", "type", "size",
63         "md5sum", "sha1sum", "sha256sum", "component", "location id",
64         "source package", "source version", "maintainer", "dbtype", "files id",
65         "new", "section", "priority", "othercomponents", "pool name",
66         "original component" ]
67
68 __all__.append('CHANGESFIELDS_FILES')
69
70 CHANGESFIELDS_DSC = [ "source", "version", "maintainer", "fingerprint",
71         "uploaders", "bts changelog", "dm-upload-allowed" ]
72
73 __all__.append('CHANGESFIELDS_DSC')
74
75 CHANGESFIELDS_DSCFILES_MANDATORY = [ "size", "md5sum" ]
76
77 __all__.append('CHANGESFIELDS_DSCFILES_MANDATORY')
78
79 CHANGESFIELDS_DSCFILES_OPTIONAL = [ "files id" ]
80
81 __all__.append('CHANGESFIELDS_DSCFILES_OPTIONAL')
82
83 CHANGESFIELDS_ORIGFILES = [ "id", "location" ]
84
85 __all__.append('CHANGESFIELDS_ORIGFILES')
86
87 ###############################################################################
88
89 class Changes(object):
90     """ Convenience wrapper to carry around all the package information """
91
92     def __init__(self, **kwds):
93         self.reset()
94
95     def reset(self):
96         self.changes_file = ""
97
98         self.changes = {}
99         self.dsc = {}
100         self.files = {}
101         self.dsc_files = {}
102         self.orig_files = {}
103
104     def file_summary(self):
105         # changes["distribution"] may not exist in corner cases
106         # (e.g. unreadable changes files)
107         if not self.changes.has_key("distribution") or not \
108                isinstance(self.changes["distribution"], dict):
109             self.changes["distribution"] = {}
110
111         byhand = False
112         new = False
113         summary = ""
114         override_summary = ""
115
116         for name, entry in sorted(self.files.items()):
117             if entry.has_key("byhand"):
118                 byhand = True
119                 summary += name + " byhand\n"
120
121             elif entry.has_key("new"):
122                 new = True
123                 summary += "(new) %s %s %s\n" % (name, entry["priority"], entry["section"])
124
125                 if entry.has_key("othercomponents"):
126                     summary += "WARNING: Already present in %s distribution.\n" % (entry["othercomponents"])
127
128                 if entry["type"] == "deb":
129                     deb_fh = open_file(name)
130                     summary += ParseSection(debExtractControl(deb_fh))["Description"] + '\n'
131                     deb_fh.close()
132
133             else:
134                 entry["pool name"] = poolify(self.changes.get("source", ""), entry["component"])
135                 destination = entry["pool name"] + name
136                 summary += name + "\n  to " + destination + "\n"
137
138                 if not entry.has_key("type"):
139                     entry["type"] = "unknown"
140
141                 if entry["type"] in ["deb", "udeb", "dsc"]:
142                     # (queue/unchecked), there we have override entries already, use them
143                     # (process-new), there we dont have override entries, use the newly generated ones.
144                     override_prio = entry.get("override priority", entry["priority"])
145                     override_sect = entry.get("override section", entry["section"])
146                     override_summary += "%s - %s %s\n" % (name, override_prio, override_sect)
147
148         return (byhand, new, summary, override_summary)
149
150     def check_override(self):
151         """
152         Checks override entries for validity.
153
154         Returns an empty string if there are no problems
155         or the text of a warning if there are
156         """
157
158         summary = ""
159
160         # Abandon the check if it's a non-sourceful upload
161         if not self.changes["architecture"].has_key("source"):
162             return summary
163
164         for name, entry in sorted(self.files.items()):
165             if not entry.has_key("new") and entry["type"] == "deb":
166                 if entry["section"] != "-":
167                     if entry["section"].lower() != entry["override section"].lower():
168                         summary += "%s: package says section is %s, override says %s.\n" % (name,
169                                                                                             entry["section"],
170                                                                                             entry["override section"])
171
172                 if entry["priority"] != "-":
173                     if entry["priority"] != entry["override priority"]:
174                         summary += "%s: package says priority is %s, override says %s.\n" % (name,
175                                                                                              entry["priority"],
176                                                                                              entry["override priority"])
177
178         return summary
179
180     @session_wrapper
181     def remove_known_changes(self, session=None):
182         session.delete(get_dbchange(self.changes_file, session))
183
184     def mark_missing_fields(self):
185         """add "missing" in fields which we will require for the known_changes table"""
186         for key in ['urgency', 'maintainer', 'fingerprint', 'changed-by' ]:
187             if (not self.changes.has_key(key)) or (not self.changes[key]):
188                 self.changes[key]='missing'
189
190     @session_wrapper
191     def add_known_changes(self, dirpath, in_queue=None, session=None):
192         """add "missing" in fields which we will require for the known_changes table"""
193         cnf = Config()
194
195         changesfile = os.path.join(dirpath, self.changes_file)
196         filetime = datetime.datetime.fromtimestamp(os.path.getctime(changesfile))
197
198         self.mark_missing_fields()
199
200         multivalues = {}
201         for key in ("distribution", "architecture", "binary"):
202             if isinstance(self.changes[key], dict):
203                 multivalues[key] = " ".join(self.changes[key].keys())
204             else:
205                 multivalues[key] = self.changes[key]
206
207         chg = DBChange()
208         chg.changesname = self.changes_file
209         chg.seen = filetime
210         chg.in_queue_id = in_queue
211         chg.source = self.changes["source"]
212         chg.binaries = multivalues["binary"]
213         chg.architecture = multivalues["architecture"]
214         chg.version = self.changes["version"]
215         chg.distribution = multivalues["distribution"]
216         chg.urgency = self.changes["urgency"]
217         chg.maintainer = self.changes["maintainer"]
218         chg.fingerprint = self.changes["fingerprint"]
219         chg.changedby = self.changes["changed-by"]
220         chg.date = self.changes["date"]
221
222         session.add(chg)
223
224         files = []
225         for chg_fn, entry in self.files.items():
226             try:
227                 f = open(os.path.join(dirpath, chg_fn))
228                 cpf = ChangePendingFile()
229                 cpf.filename = chg_fn
230                 cpf.size = entry['size']
231                 cpf.md5sum = entry['md5sum']
232
233                 if entry.has_key('sha1sum'):
234                     cpf.sha1sum = entry['sha1sum']
235                 else:
236                     f.seek(0)
237                     cpf.sha1sum = apt_pkg.sha1sum(f)
238
239                 if entry.has_key('sha256sum'):
240                     cpf.sha256sum = entry['sha256sum']
241                 else:
242                     f.seek(0)
243                     cpf.sha256sum = apt_pkg.sha256sum(f)
244
245                 session.add(cpf)
246                 files.append(cpf)
247                 f.close()
248
249             except IOError:
250                 # Can't find the file, try to look it up in the pool
251                 poolname = poolify(entry["source"], entry["component"])
252                 l = get_location(cnf["Dir::Pool"], entry["component"], session=session)
253
254                 found, poolfile = check_poolfile(os.path.join(poolname, chg_fn),
255                                                  entry['size'],
256                                                  entry["md5sum"],
257                                                  l.location_id,
258                                                  session=session)
259
260                 if found is None:
261                     Logger.log(["E: Found multiple files for pool (%s) for %s" % (chg_fn, entry["component"])])
262                 elif found is False and poolfile is not None:
263                     Logger.log(["E: md5sum/size mismatch for %s in pool" % (chg_fn)])
264                 else:
265                     if poolfile is None:
266                         Logger.log(["E: Could not find %s in pool" % (chg_fn)])
267                     else:
268                         chg.poolfiles.append(poolfile)
269
270         chg.files = files
271
272         session.commit()
273         chg = session.query(DBChange).filter_by(changesname = self.changes_file).one();
274
275         return chg
276
277     def unknown_files_fields(self, name):
278         return sorted(list( set(self.files[name].keys()) -
279                             set(CHANGESFIELDS_FILES)))
280
281     def unknown_changes_fields(self):
282         return sorted(list( set(self.changes.keys()) -
283                             set(CHANGESFIELDS_MANDATORY + CHANGESFIELDS_OPTIONAL)))
284
285     def unknown_dsc_fields(self):
286         return sorted(list( set(self.dsc.keys()) -
287                             set(CHANGESFIELDS_DSC)))
288
289     def unknown_dsc_files_fields(self, name):
290         return sorted(list( set(self.dsc_files[name].keys()) -
291                             set(CHANGESFIELDS_DSCFILES_MANDATORY + CHANGESFIELDS_DSCFILES_OPTIONAL)))
292
293     def str_files(self):
294         r = []
295         for name, entry in self.files.items():
296             r.append("  %s:" % (name))
297             for i in CHANGESFIELDS_FILES:
298                 if entry.has_key(i):
299                     r.append("   %s: %s" % (i.capitalize(), entry[i]))
300             xfields = self.unknown_files_fields(name)
301             if len(xfields) > 0:
302                 r.append("files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
303
304         return r
305
306     def str_changes(self):
307         r = []
308         for i in CHANGESFIELDS_MANDATORY:
309             val = self.changes[i]
310             if isinstance(val, list):
311                 val = " ".join(val)
312             elif isinstance(val, dict):
313                 val = " ".join(val.keys())
314             r.append('  %s: %s' % (i.capitalize(), val))
315
316         for i in CHANGESFIELDS_OPTIONAL:
317             if self.changes.has_key(i):
318                 r.append('  %s: %s' % (i.capitalize(), self.changes[i]))
319
320         xfields = self.unknown_changes_fields()
321         if len(xfields) > 0:
322             r.append("Warning: changes still has the following unrecognised fields: %s" % ", ".join(xfields))
323
324         return r
325
326     def str_dsc(self):
327         r = []
328         for i in CHANGESFIELDS_DSC:
329             if self.dsc.has_key(i):
330                 r.append('  %s: %s' % (i.capitalize(), self.dsc[i]))
331
332         xfields = self.unknown_dsc_fields()
333         if len(xfields) > 0:
334             r.append("Warning: dsc still has the following unrecognised fields: %s" % ", ".join(xfields))
335
336         return r
337
338     def str_dsc_files(self):
339         r = []
340         for name, entry in self.dsc_files.items():
341             r.append("  %s:" % (name))
342             for i in CHANGESFIELDS_DSCFILES_MANDATORY:
343                 r.append("   %s: %s" % (i.capitalize(), entry[i]))
344             for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
345                 if entry.has_key(i):
346                     r.append("   %s: %s" % (i.capitalize(), entry[i]))
347             xfields = self.unknown_dsc_files_fields(name)
348             if len(xfields) > 0:
349                 r.append("dsc_files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
350
351         return r
352
353     def __str__(self):
354         r = []
355
356         r.append(" Changes:")
357         r += self.str_changes()
358
359         r.append("")
360
361         r.append(" Dsc:")
362         r += self.str_dsc()
363
364         r.append("")
365
366         r.append(" Files:")
367         r += self.str_files()
368
369         r.append("")
370
371         r.append(" Dsc Files:")
372         r += self.str_dsc_files()
373
374         return "\n".join(r)
375
376 __all__.append('Changes')