daklib/changes.py

   1 #!/usr/bin/env python
   2 # vim:set et sw=4:
   3
   4 """
   5 Changes class for dak
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2001 - 2006 James Troup <james@nocrew.org>
   9 @copyright: 2009  Joerg Jaspert <joerg@debian.org>
  10 @copyright: 2009  Mark Hymers <mhy@debian.org>
  11 @license: GNU General Public License version 2 or later
  12 """
  13
  14 # This program is free software; you can redistribute it and/or modify
  15 # it under the terms of the GNU General Public License as published by
  16 # the Free Software Foundation; either version 2 of the License, or
  17 # (at your option) any later version.
  18
  19 # This program is distributed in the hope that it will be useful,
  20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22 # GNU General Public License for more details.
  23
  24 # You should have received a copy of the GNU General Public License
  25 # along with this program; if not, write to the Free Software
  26 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  27
  28 ###############################################################################
  29
  30 import os
  31 import stat
  32 import time
  33
  34 import datetime
  35 from cPickle import Unpickler, Pickler
  36 from errno import EPERM
  37
  38 from apt_inst import debExtractControl
  39 from apt_pkg import ParseSection
  40
  41 from utils import open_file, fubar, poolify
  42 from config import *
  43 from dbconn import *
  44
  45 ###############################################################################
  46
  47 __all__ = []
  48
  49 ###############################################################################
  50
  51 CHANGESFIELDS_MANDATORY = [ "distribution", "source", "architecture",
  52         "version", "maintainer", "urgency", "fingerprint", "changedby822",
  53         "changedby2047", "changedbyname", "maintainer822", "maintainer2047",
  54         "maintainername", "maintaineremail", "closes", "changes" ]
  55
  56 __all__.append('CHANGESFIELDS_MANDATORY')
  57
  58 CHANGESFIELDS_OPTIONAL = [ "changed-by", "filecontents", "format",
  59         "process-new note", "adv id", "distribution-version", "sponsoremail" ]
  60
  61 __all__.append('CHANGESFIELDS_OPTIONAL')
  62
  63 CHANGESFIELDS_FILES = [ "package", "version", "architecture", "type", "size",
  64         "md5sum", "sha1sum", "sha256sum", "component", "location id",
  65         "source package", "source version", "maintainer", "dbtype", "files id",
  66         "new", "section", "priority", "othercomponents", "pool name",
  67         "original component" ]
  68
  69 __all__.append('CHANGESFIELDS_FILES')
  70
  71 CHANGESFIELDS_DSC = [ "source", "version", "maintainer", "fingerprint",
  72         "uploaders", "bts changelog", "dm-upload-allowed" ]
  73
  74 __all__.append('CHANGESFIELDS_DSC')
  75
  76 CHANGESFIELDS_DSCFILES_MANDATORY = [ "size", "md5sum" ]
  77
  78 __all__.append('CHANGESFIELDS_DSCFILES_MANDATORY')
  79
  80 CHANGESFIELDS_DSCFILES_OPTIONAL = [ "files id" ]
  81
  82 __all__.append('CHANGESFIELDS_DSCFILES_OPTIONAL')
  83
  84 CHANGESFIELDS_ORIGFILES = [ "id", "location" ]
  85
  86 __all__.append('CHANGESFIELDS_ORIGFILES')
  87
  88 ###############################################################################
  89
  90 class Changes(object):
  91     """ Convenience wrapper to carry around all the package information """
  92
  93     def __init__(self, **kwds):
  94         self.reset()
  95
  96     def reset(self):
  97         self.changes_file = ""
  98
  99         self.changes = {}
 100         self.dsc = {}
 101         self.files = {}
 102         self.dsc_files = {}
 103         self.orig_files = {}
 104
 105     def file_summary(self):
 106         # changes["distribution"] may not exist in corner cases
 107         # (e.g. unreadable changes files)
 108         if not self.changes.has_key("distribution") or not \
 109                isinstance(self.changes["distribution"], dict):
 110             self.changes["distribution"] = {}
 111
 112         byhand = False
 113         new = False
 114         summary = ""
 115         override_summary = ""
 116
 117         for name, entry in sorted(self.files.items()):
 118             if entry.has_key("byhand"):
 119                 byhand = True
 120                 summary += name + " byhand\n"
 121
 122             elif entry.has_key("new"):
 123                 new = True
 124                 summary += "(new) %s %s %s\n" % (name, entry["priority"], entry["section"])
 125
 126                 if entry.has_key("othercomponents"):
 127                     summary += "WARNING: Already present in %s distribution.\n" % (entry["othercomponents"])
 128
 129                 if entry["type"] == "deb":
 130                     deb_fh = open_file(name)
 131                     summary += ParseSection(debExtractControl(deb_fh))["Description"] + '\n'
 132                     deb_fh.close()
 133
 134             else:
 135                 entry["pool name"] = poolify(self.changes.get("source", ""), entry["component"])
 136                 destination = entry["pool name"] + name
 137                 summary += name + "\n  to " + destination + "\n"
 138
 139                 if not entry.has_key("type"):
 140                     entry["type"] = "unknown"
 141
 142                 if entry["type"] in ["deb", "udeb", "dsc"]:
 143                     # (queue/unchecked), there we have override entries already, use them
 144                     # (process-new), there we dont have override entries, use the newly generated ones.
 145                     override_prio = entry.get("override priority", entry["priority"])
 146                     override_sect = entry.get("override section", entry["section"])
 147                     override_summary += "%s - %s %s\n" % (name, override_prio, override_sect)
 148
 149         return (byhand, new, summary, override_summary)
 150
 151     def check_override(self):
 152         """
 153         Checks override entries for validity.
 154
 155         Returns an empty string if there are no problems
 156         or the text of a warning if there are
 157         """
 158
 159         summary = ""
 160
 161         # Abandon the check if it's a non-sourceful upload
 162         if not self.changes["architecture"].has_key("source"):
 163             return summary
 164
 165         for name, entry in sorted(self.files.items()):
 166             if not entry.has_key("new") and entry["type"] == "deb":
 167                 if entry["section"] != "-":
 168                     if entry["section"].lower() != entry["override section"].lower():
 169                         summary += "%s: package says section is %s, override says %s.\n" % (name,
 170                                                                                             entry["section"],
 171                                                                                             entry["override section"])
 172
 173                 if entry["priority"] != "-":
 174                     if entry["priority"] != entry["override priority"]:
 175                         summary += "%s: package says priority is %s, override says %s.\n" % (name,
 176                                                                                              entry["priority"],
 177                                                                                              entry["override priority"])
 178
 179         return summary
 180
 181     def remove_known_changes(self, session=None):
 182         if session is None:
 183             session = DBConn().session()
 184             privatetrans = True
 185
 186         session.query(KnownChange).filter(changesfile=self.changes_file).delete()
 187
 188         if privatetrans:
 189             session.commit()
 190             session.close()
 191     def add_known_changes(self, queue, session=None):
 192         cnf = Config()
 193
 194         if session is None:
 195             session = DBConn().session()
 196             privatetrans = True
 197
 198         dirpath = cnf["Dir::Queue::%s" % (queue) ]
 199         changesfile = os.path.join(dirpath, self.changes_file)
 200         filetime = datetime.datetime.fromtimestamp(os.path.getctime(changesfile))
 201
 202         session.execute(
 203             """INSERT INTO known_changes
 204               (changesname, seen, source, binaries, architecture, version,
 205               distribution, urgency, maintainer, fingerprint, changedby, date)
 206               VALUES (:changesfile,:filetime,:source,:binary, :architecture,
 207               :version,:distribution,:urgency,:maintainer,:fingerprint,:changedby,:date)""",
 208               { 'changesfile':changesfile,
 209                 'filetime':filetime,
 210                 'source':self.changes["source"],
 211                 'binary':self.changes["binary"],
 212                 'architecture':self.changes["architecture"],
 213                 'version':self.changes["version"],
 214                 'distribution':self.changes["distribution"],
 215                 'urgency':self.changes["urgency"],
 216                 'maintainer':self.changes["maintainer"],
 217                 'fingerprint':self.changes["fingerprint"],
 218                 'changedby':self.changes["changed-by"],
 219                 'date':self.changes["date"]} )
 220
 221         if privatetrans:
 222             session.commit()
 223             session.close()
 224
 225     def load_dot_dak(self, changesfile):
 226         """
 227         Update ourself by reading a previously created cPickle .dak dumpfile.
 228         """
 229
 230         self.changes_file = changesfile
 231         dump_filename = self.changes_file[:-8]+".dak"
 232         dump_file = open_file(dump_filename)
 233
 234         p = Unpickler(dump_file)
 235
 236         self.changes.update(p.load())
 237         self.dsc.update(p.load())
 238         self.files.update(p.load())
 239         self.dsc_files.update(p.load())
 240
 241         next_obj = p.load()
 242         if isinstance(next_obj, dict):
 243             self.orig_files.update(next_obj)
 244         else:
 245             # Auto-convert old dak files to new format supporting
 246             # multiple tarballs
 247             orig_tar_gz = None
 248             for dsc_file in self.dsc_files.keys():
 249                 if dsc_file.endswith(".orig.tar.gz"):
 250                     orig_tar_gz = dsc_file
 251             self.orig_files[orig_tar_gz] = {}
 252             if next_obj != None:
 253                 self.orig_files[orig_tar_gz]["id"] = next_obj
 254             next_obj = p.load()
 255             if next_obj != None and next_obj != "":
 256                 self.orig_files[orig_tar_gz]["location"] = next_obj
 257             if len(self.orig_files[orig_tar_gz]) == 0:
 258                 del self.orig_files[orig_tar_gz]
 259
 260         dump_file.close()
 261
 262     def sanitised_files(self):
 263         ret = {}
 264         for name, entry in self.files.items():
 265             ret[name] = {}
 266             for i in CHANGESFIELDS_FILES:
 267                 if entry.has_key(i):
 268                     ret[name][i] = entry[i]
 269
 270         return ret
 271
 272     def sanitised_changes(self):
 273         ret = {}
 274         # Mandatory changes fields
 275         for i in CHANGESFIELDS_MANDATORY:
 276             ret[i] = self.changes[i]
 277
 278         # Optional changes fields
 279         for i in CHANGESFIELDS_OPTIONAL:
 280             if self.changes.has_key(i):
 281                 ret[i] = self.changes[i]
 282
 283         return ret
 284
 285     def sanitised_dsc(self):
 286         ret = {}
 287         for i in CHANGESFIELDS_DSC:
 288             if self.dsc.has_key(i):
 289                 ret[i] = self.dsc[i]
 290
 291         return ret
 292
 293     def sanitised_dsc_files(self):
 294         ret = {}
 295         for name, entry in self.dsc_files.items():
 296             ret[name] = {}
 297             # Mandatory dsc_files fields
 298             for i in CHANGESFIELDS_DSCFILES_MANDATORY:
 299                 ret[name][i] = entry[i]
 300
 301             # Optional dsc_files fields
 302             for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
 303                 if entry.has_key(i):
 304                     ret[name][i] = entry[i]
 305
 306         return ret
 307
 308     def sanitised_orig_files(self):
 309         ret = {}
 310         for name, entry in self.orig_files.items():
 311             ret[name] = {}
 312             # Optional orig_files fields
 313             for i in CHANGESFIELDS_ORIGFILES:
 314                 if entry.has_key(i):
 315                     ret[name][i] = entry[i]
 316
 317         return ret
 318
 319     def write_dot_dak(self, dest_dir):
 320         """
 321         Dump ourself into a cPickle file.
 322
 323         @type dest_dir: string
 324         @param dest_dir: Path where the dumpfile should be stored
 325
 326         @note: This could just dump the dictionaries as is, but I'd like to avoid this so
 327                there's some idea of what process-accepted & process-new use from
 328                process-unchecked. (JT)
 329
 330         """
 331
 332         dump_filename = os.path.join(dest_dir, self.changes_file[:-8] + ".dak")
 333         dump_file = open_file(dump_filename, 'w')
 334
 335         try:
 336             os.chmod(dump_filename, 0664)
 337         except OSError, e:
 338             # chmod may fail when the dumpfile is not owned by the user
 339             # invoking dak (like e.g. when NEW is processed by a member
 340             # of ftpteam)
 341             if e.errno == EPERM:
 342                 perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE])
 343                 # security precaution, should never happen unless a weird
 344                 # umask is set anywhere
 345                 if perms & stat.S_IWOTH:
 346                     fubar("%s is world writable and chmod failed." % \
 347                         (dump_filename,))
 348                 # ignore the failed chmod otherwise as the file should
 349                 # already have the right privileges and is just, at worst,
 350                 # unreadable for world
 351             else:
 352                 raise
 353
 354         p = Pickler(dump_file, 1)
 355
 356         p.dump(self.sanitised_changes())
 357         p.dump(self.sanitised_dsc())
 358         p.dump(self.sanitised_files())
 359         p.dump(self.sanitised_dsc_files())
 360         p.dump(self.sanitised_orig_files())
 361
 362         dump_file.close()
 363
 364     def unknown_files_fields(self, name):
 365         return sorted(list( set(self.files[name].keys()) -
 366                             set(CHANGESFIELDS_FILES)))
 367
 368     def unknown_changes_fields(self):
 369         return sorted(list( set(self.changes.keys()) -
 370                             set(CHANGESFIELDS_MANDATORY + CHANGESFIELDS_OPTIONAL)))
 371
 372     def unknown_dsc_fields(self):
 373         return sorted(list( set(self.dsc.keys()) -
 374                             set(CHANGESFIELDS_DSC)))
 375
 376     def unknown_dsc_files_fields(self, name):
 377         return sorted(list( set(self.dsc_files[name].keys()) -
 378                             set(CHANGESFIELDS_DSCFILES_MANDATORY + CHANGESFIELDS_DSCFILES_OPTIONAL)))
 379
 380     def str_files(self):
 381         r = []
 382         for name, entry in self.files.items():
 383             r.append("  %s:" % (name))
 384             for i in CHANGESFIELDS_FILES:
 385                 if entry.has_key(i):
 386                     r.append("   %s: %s" % (i.capitalize(), entry[i]))
 387             xfields = self.unknown_files_fields(name)
 388             if len(xfields) > 0:
 389                 r.append("files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
 390
 391         return r
 392
 393     def str_changes(self):
 394         r = []
 395         for i in CHANGESFIELDS_MANDATORY:
 396             val = self.changes[i]
 397             if isinstance(val, list):
 398                 val = " ".join(val)
 399             elif isinstance(val, dict):
 400                 val = " ".join(val.keys())
 401             r.append('  %s: %s' % (i.capitalize(), val))
 402
 403         for i in CHANGESFIELDS_OPTIONAL:
 404             if self.changes.has_key(i):
 405                 r.append('  %s: %s' % (i.capitalize(), self.changes[i]))
 406
 407         xfields = self.unknown_changes_fields()
 408         if len(xfields) > 0:
 409             r.append("Warning: changes still has the following unrecognised fields: %s" % ", ".join(xfields))
 410
 411         return r
 412
 413     def str_dsc(self):
 414         r = []
 415         for i in CHANGESFIELDS_DSC:
 416             if self.dsc.has_key(i):
 417                 r.append('  %s: %s' % (i.capitalize(), self.dsc[i]))
 418
 419         xfields = self.unknown_dsc_fields()
 420         if len(xfields) > 0:
 421             r.append("Warning: dsc still has the following unrecognised fields: %s" % ", ".join(xfields))
 422
 423         return r
 424
 425     def str_dsc_files(self):
 426         r = []
 427         for name, entry in self.dsc_files.items():
 428             r.append("  %s:" % (name))
 429             for i in CHANGESFIELDS_DSCFILES_MANDATORY:
 430                 r.append("   %s: %s" % (i.capitalize(), entry[i]))
 431             for i in CHANGESFIELDS_DSCFILES_OPTIONAL:
 432                 if entry.has_key(i):
 433                     r.append("   %s: %s" % (i.capitalize(), entry[i]))
 434             xfields = self.unknown_dsc_files_fields(name)
 435             if len(xfields) > 0:
 436                 r.append("dsc_files[%s] still has following unrecognised keys: %s" % (name, ", ".join(xfields)))
 437
 438         return r
 439
 440     def __str__(self):
 441         r = []
 442
 443         r.append(" Changes:")
 444         r += self.str_changes()
 445
 446         r.append("")
 447
 448         r.append(" Dsc:")
 449         r += self.str_dsc()
 450
 451         r.append("")
 452
 453         r.append(" Files:")
 454         r += self.str_files()
 455
 456         r.append("")
 457
 458         r.append(" Dsc Files:")
 459         r += self.str_dsc_files()
 460
 461         return "\n".join(r)
 462
 463 __all__.append('Changes')