dak/import_new_files.py

   1 #!/usr/bin/env python
   2 # coding=utf8
   3
   4 """
   5 Import known_changes files
   6
   7 @contact: Debian FTP Master <ftpmaster@debian.org>
   8 @copyright: 2009  Mike O'Connor <stew@debian.org>
   9 @license: GNU General Public License version 2 or later
  10 """
  11
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; either version 2 of the License, or
  15 # (at your option) any later version.
  16
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25
  26 ################################################################################
  27
  28
  29 ################################################################################
  30
  31 import sys
  32 import os
  33 import logging
  34 import threading
  35 import glob
  36 import apt_pkg
  37 from daklib.dbconn import DBConn, get_dbchange, get_policy_queue, session_wrapper, ChangePendingFile
  38 from daklib.config import Config
  39 from daklib.queue import Upload
  40
  41 # where in dak.conf all of our configuration will be stowed
  42 options_prefix = "NewFiles"
  43 options_prefix = "%s::Options" % options_prefix
  44
  45 log = logging.getLogger()
  46
  47 ################################################################################
  48
  49
  50 def usage (exit_code=0):
  51     print """Usage: dak import-new-files [options]
  52
  53 OPTIONS
  54      -v, --verbose
  55         show verbose information messages
  56
  57      -q, --quiet
  58         supress all output but errors
  59
  60 """
  61     sys.exit(exit_code)
  62
  63 class ImportNewFiles(object):
  64     @session_wrapper
  65     def __init__(self, session=None):
  66         cnf = Config()
  67         try:
  68             newq = get_policy_queue('new', session)
  69             for changes_fn in glob.glob(newq.path + "/*.changes"):
  70                 changes_bn = os.path.basename(changes_fn)
  71                 chg = get_dbchange(changes_bn, session)
  72
  73                 u = Upload()
  74                 success = u.load_changes(changes_fn)
  75                 u.pkg.changes_file = changes_bn
  76                 u.check_hashes()
  77
  78                 if not chg:
  79                     chg = u.pkg.add_known_changes(newq.path, newq.policy_queue_id, session)
  80                     session.add(chg)
  81
  82                 if not success:
  83                     log.critical("failed to load %s" % changes_fn)
  84                     sys.exit(1)
  85                 else:
  86                     log.critical("ACCLAIM: %s" % changes_fn)
  87
  88                 files=[]
  89                 for chg_fn in u.pkg.files.keys():
  90                     try:
  91                         f = open(os.path.join(newq.path, chg_fn))
  92                         cpf = ChangePendingFile()
  93                         cpf.filename = chg_fn
  94                         cpf.size = u.pkg.files[chg_fn]['size']
  95                         cpf.md5sum = u.pkg.files[chg_fn]['md5sum']
  96
  97                         if u.pkg.files[chg_fn].has_key('sha1sum'):
  98                             cpf.sha1sum = u.pkg.files[chg_fn]['sha1sum']
  99                         else:
 100                             log.warning("Having to generate sha1sum for %s" % chg_fn)
 101                             f.seek(0)
 102                             cpf.sha1sum = apt_pkg.sha1sum(f)
 103
 104                         if u.pkg.files[chg_fn].has_key('sha256sum'):
 105                             cpf.sha256sum = u.pkg.files[chg_fn]['sha256sum']
 106                         else:
 107                             log.warning("Having to generate sha256sum for %s" % chg_fn)
 108                             f.seek(0)
 109                             cpf.sha256sum = apt_pkg.sha256sum(f)
 110
 111                         session.add(cpf)
 112                         files.append(cpf)
 113                         f.close()
 114                     except IOError:
 115                         # Can't find the file, try to look it up in the pool
 116                         poolname = utils.poolify(u.pkg.changes["source"], u.pkg.files[chg_fn]["component"])
 117                         l = get_location(cnf["Dir::Pool"], u.pkg.files[chg_fn]["component"], session=session)
 118                         if not l:
 119                             log.critical("ERROR: Can't find location for %s (component %s)" % (chg_fn, u.pkg.files[chg_fn]["component"]))
 120
 121                         found, poolfile = check_poolfile(os.path.join(poolname, chg_fn),
 122                                                          u.pkg.files[chg_fn]['size'],
 123                                                          u.pkg.files[chg_fn]["md5sum"],
 124                                                          l,
 125                                                          session=session)
 126
 127                         if found is None:
 128                             log.critical("ERROR: Found multiple files for %s in pool" % chg_fn)
 129                             sys.exit(1)
 130                         elif found is False and poolfile is not None:
 131                             log.critical("ERROR: md5sum / size mismatch for %s in pool" % chg_fn)
 132                             sys.exit(1)
 133                         else:
 134                             if poolfile is None:
 135                                 log.critical("ERROR: Could not find %s in pool" % chg_fn)
 136                                 sys.exit(1)
 137                             else:
 138                                 chg.changeslinks.append(poolfile)
 139
 140
 141                 chg.files = files
 142
 143
 144             session.commit()
 145
 146         except KeyboardInterrupt:
 147             print("Caught C-c; terminating.")
 148             utils.warn("Caught C-c; terminating.")
 149             self.plsDie()
 150
 151
 152 def main():
 153     cnf = Config()
 154
 155     arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
 156                  ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
 157                  ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
 158                 ]
 159
 160     args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments,sys.argv)
 161
 162     num_threads = 1
 163
 164     if len(args) > 0:
 165         usage(1)
 166
 167     if cnf.has_key("%s::%s" % (options_prefix,"Help")):
 168         usage(0)
 169
 170     level=logging.INFO
 171     if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
 172         level=logging.ERROR
 173
 174     elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
 175         level=logging.DEBUG
 176
 177
 178     logging.basicConfig( level=level,
 179                          format='%(asctime)s %(levelname)s %(message)s',
 180                          stream = sys.stderr )
 181
 182     ImportNewFiles()
 183
 184
 185 if __name__ == '__main__':
 186     main()