#!/usr/bin/env python # Sanity check the database # Copyright (C) 2000, 2001 James Troup # $Id: tea,v 1.15 2002-01-19 18:57:49 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # And, lo, a great and menacing voice rose from the depths, and with # great wrath and vehemence it's voice boomed across the # land... ``hehehehehehe... that *tickles*'' # -- aj on IRC ################################################################################ import pg, sys, os, string, stat, time import utils, db_access import apt_pkg, apt_inst; ################################################################################ Cnf = None; projectB = None; db_files = {}; waste = 0.0; excluded = {}; current_file = None; future_files = {}; current_time = time.time(); ################################################################################ def process_dir (arg, dirname, filenames): global waste, db_files, excluded; if string.find(dirname, '/disks-') != -1 or string.find(dirname, 'upgrade-') != -1: return; # hack; can't handle .changes files if string.find(dirname, 'proposed-updates') != -1: return; for name in filenames: filename = os.path.abspath(dirname+'/'+name); filename = string.replace(filename, 'potato-proposed-updates', 'proposed-updates'); if os.path.isfile(filename) and not os.path.islink(filename) and not db_files.has_key(filename) and not excluded.has_key(filename): waste = waste + os.stat(filename)[stat.ST_SIZE]; print filename ################################################################################ def check_files(): global db_files; print "Building list of Database files..."; q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id") ql = q.getresult(); db_files = {}; for i in ql: filename = os.path.abspath(i[0] + i[1]); db_files[filename] = ""; if os.access(filename, os.R_OK) == 0: utils.warn("'%s' doesn't exist." % (filename)); file = utils.open_file(Cnf["Dir::OverrideDir"]+'override.unreferenced'); for filename in file.readlines(): filename = filename[:-1]; excluded[filename] = ""; print "Checking against existent files..."; os.path.walk(Cnf["Dir::RootDir"]+'dists/', process_dir, None); print print "%s wasted..." % (utils.size_type(waste)); ################################################################################ def check_dscs(): count = 0; suite = 'unstable'; for component in Cnf.SubTree("Component").List(): if component == "mixed": continue; component = string.lower(component); list_filename = '%s%s_%s_source.list' % (Cnf["Dir::ListsDir"], suite, component); list_file = utils.open_file(list_filename); for line in list_file.readlines(): file = line[:-1]; try: utils.parse_changes(file, 1); except utils.invalid_dsc_format_exc, line: utils.warn("syntax error in .dsc file '%s', line %s." % (file, line)); count = count + 1; if count: utils.warn("Found %s invalid .dsc files." % (count)); ################################################################################ def check_override(): for suite in [ "stable", "unstable" ]: print suite print "-------------" print suite_id = db_access.get_suite_id(suite); q = projectB.query(""" SELECT DISTINCT b.package FROM binaries b, bin_associations ba WHERE b.id = ba.bin AND ba.suite = %s AND NOT EXISTS (SELECT * FROM override o WHERE o.suite = %s AND o.package = b.package)""" % (suite_id, suite_id)); print q q = projectB.query(""" SELECT DISTINCT s.source FROM source s, src_associations sa WHERE s.id = sa.source AND sa.suite = %s AND NOT EXISTS (SELECT * FROM override o WHERE o.suite = %s and o.package = s.source)""" % (suite_id, suite_id)); print q ################################################################################ # Ensure that the source files for any given package is all in one # directory so that 'apt-get source' works... def check_source_in_one_dir(): # Not the most enterprising method, but hey... broken_count = 0; q = projectB.query("SELECT id FROM source;"); for i in q.getresult(): source_id = i[0]; q2 = projectB.query(""" SELECT l.path, f.filename FROM files f, dsc_files df, location l WHERE df.source = %s AND f.id = df.file AND l.id = f.location""" % (source_id)); first_path = ""; first_filename = ""; broken = 0; for j in q2.getresult(): filename = j[0]+j[1]; path = os.path.dirname(filename); if first_path == "": first_path = path; first_filename = filename; elif first_path != path: symlink = path + '/' + os.path.basename(first_filename); if not os.path.exists(symlink): broken = 1; print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, source_id, symlink); if broken: broken_count = broken_count + 1; print "Found %d source packages where the source is not all in one directory." % (broken_count); ################################################################################ def check_md5sums(): print "Getting file information from database..."; q = projectB.query("SELECT l.path, f.filename, f.md5sum, f.size FROM files f, location l WHERE f.location = l.id") ql = q.getresult(); print "Checking file md5sums & sizes..."; for i in ql: filename = os.path.abspath(i[0] + i[1]); db_md5sum = i[2]; db_size = int(i[3]); try: file = utils.open_file(filename); except: utils.warn("can't open '%s'." % (filename)); continue; md5sum = apt_pkg.md5sum(file); size = os.stat(filename)[stat.ST_SIZE]; if md5sum != db_md5sum: utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, db_md5sum)); if size != db_size: utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, db_size)); print "Done." ################################################################################ # # Check all files for timestamps in the future; common from hardware # (e.g. alpha) which have far-future dates as their default dates. def Ent(Kind,Name,Link,Mode,UID,GID,Size,MTime,Major,Minor): global future_files; if MTime > current_time: future_files[current_file] = MTime; print "%s: %s '%s','%s',%u,%u,%u,%u,%u" % (current_file, Kind,Name,Link,Mode,UID,GID,Size, MTime); def check_timestamps(): global current_file; q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.deb$'") ql = q.getresult(); db_files = {}; count = 0; for i in ql: filename = os.path.abspath(i[0] + i[1]); if os.access(filename, os.R_OK): file = utils.open_file(filename); current_file = filename; sys.stderr.write("Processing %s.\n" % (filename)); apt_inst.debExtract(file,Ent,"control.tar.gz"); file.seek(0); apt_inst.debExtract(file,Ent,"data.tar.gz"); count = count + 1; print "Checked %d files (out of %d)." % (count, len(db_files.keys())); ################################################################################ def main (): global Cnf, projectB, db_files, waste, excluded; Cnf = utils.get_conf() apt_pkg.ParseCommandLine(Cnf,[],sys.argv); projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); db_access.init(Cnf, projectB); #check_md5sums(); #check_source_in_one_dir(); #check_override(); #check_dscs(); #check_files(); check_timestamps(); ####################################################################################### if __name__ == '__main__': main();