From 30413cf0ff7bc21b8d2b8b4346406357fe55dc19 Mon Sep 17 00:00:00 2001 From: James Troup Date: Mon, 15 May 2006 13:00:10 -0400 Subject: [PATCH] Globally remove trailing semi-colon damage. --- dak/check_archive.py | 410 +++++++------- dak/check_overrides.py | 94 ++-- dak/check_proposed_updates.py | 232 ++++---- dak/clean_proposed_updates.py | 146 ++--- dak/clean_queues.py | 112 ++-- dak/clean_suites.py | 244 ++++----- dak/compare_suites.py | 48 +- dak/control_overrides.py | 200 +++---- dak/control_suite.py | 200 +++---- dak/cruft_report.py | 356 ++++++------ dak/decode_dot_dak.py | 88 +-- dak/examine_package.py | 320 +++++------ dak/find_null_maintainers.py | 96 ++-- dak/generate_index_diffs.py | 32 +- dak/generate_releases.py | 24 +- dak/import_archive.py | 584 ++++++++++---------- dak/import_ldap_fingerprints.py | 162 +++--- dak/import_users_from_passwd.py | 66 +-- dak/init_db.py | 144 ++--- dak/init_dirs.py | 76 +-- dak/lib/database.py | 196 +++---- dak/lib/logging.py | 48 +- dak/lib/queue.py | 724 ++++++++++++------------- dak/lib/utils.py | 838 ++++++++++++++-------------- dak/ls.py | 114 ++-- dak/make_maintainers.py | 88 +-- dak/make_overrides.py | 76 +-- dak/make_suite_file_list.py | 340 ++++++------ dak/mirror_split.py | 208 +++---- dak/override.py | 166 +++--- dak/poolize.py | 116 ++-- dak/process_accepted.py | 508 ++++++++--------- dak/process_new.py | 848 ++++++++++++++--------------- dak/process_unchecked.py | 934 ++++++++++++++++---------------- dak/queue_report.py | 286 +++++----- dak/reject_proposed_updates.py | 192 +++---- dak/rm.py | 324 +++++------ dak/security_install.py | 298 +++++----- dak/shell.py | 8 +- dak/split_done.py | 28 +- dak/stats.py | 204 +++---- dak/symlink_dists.py | 96 ++-- dak/test/001/test.py | 42 +- dak/test/002/test.py | 10 +- dak/test/003/test.py | 10 +- dak/test/004/test.py | 84 +-- dak/test/005/test.py | 12 +- dak/test/006/test.py | 44 +- 48 files changed, 5238 insertions(+), 5238 deletions(-) diff --git a/dak/check_archive.py b/dak/check_archive.py index 228fd81e..6b83cd17 100755 --- a/dak/check_archive.py +++ b/dak/check_archive.py @@ -27,20 +27,20 @@ ################################################################################ -import commands, os, pg, stat, string, sys, time; -import db_access, utils; -import apt_pkg, apt_inst; +import commands, os, pg, stat, string, sys, time +import db_access, utils +import apt_pkg, apt_inst ################################################################################ -Cnf = None; -projectB = None; -db_files = {}; -waste = 0.0; -excluded = {}; -current_file = None; -future_files = {}; -current_time = time.time(); +Cnf = None +projectB = None +db_files = {} +waste = 0.0 +excluded = {} +current_file = None +future_files = {} +current_time = time.time() ################################################################################ @@ -68,71 +68,71 @@ The following MODEs are available: ################################################################################ def process_dir (unused, dirname, filenames): - global waste, db_files, excluded; + global waste, db_files, excluded if dirname.find('/disks-') != -1 or dirname.find('upgrade-') != -1: - return; + return # hack; can't handle .changes files if dirname.find('proposed-updates') != -1: - return; + return for name in filenames: - filename = os.path.abspath(dirname+'/'+name); - filename = filename.replace('potato-proposed-updates', 'proposed-updates'); + filename = os.path.abspath(dirname+'/'+name) + filename = filename.replace('potato-proposed-updates', 'proposed-updates') if os.path.isfile(filename) and not os.path.islink(filename) and not db_files.has_key(filename) and not excluded.has_key(filename): - waste += os.stat(filename)[stat.ST_SIZE]; + waste += os.stat(filename)[stat.ST_SIZE] print filename ################################################################################ def check_files(): - global db_files; + global db_files - print "Building list of database files..."; + print "Building list of database files..." q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id") - ql = q.getresult(); + ql = q.getresult() - db_files.clear(); + db_files.clear() for i in ql: - filename = os.path.abspath(i[0] + i[1]); - db_files[filename] = ""; + filename = os.path.abspath(i[0] + i[1]) + db_files[filename] = "" if os.access(filename, os.R_OK) == 0: - utils.warn("'%s' doesn't exist." % (filename)); + utils.warn("'%s' doesn't exist." % (filename)) - filename = Cnf["Dir::Override"]+'override.unreferenced'; + filename = Cnf["Dir::Override"]+'override.unreferenced' if os.path.exists(filename): - file = utils.open_file(filename); + file = utils.open_file(filename) for filename in file.readlines(): - filename = filename[:-1]; - excluded[filename] = ""; + filename = filename[:-1] + excluded[filename] = "" - print "Checking against existent files..."; + print "Checking against existent files..." - os.path.walk(Cnf["Dir::Root"]+'pool/', process_dir, None); + os.path.walk(Cnf["Dir::Root"]+'pool/', process_dir, None) print - print "%s wasted..." % (utils.size_type(waste)); + print "%s wasted..." % (utils.size_type(waste)) ################################################################################ def check_dscs(): - count = 0; - suite = 'unstable'; + count = 0 + suite = 'unstable' for component in Cnf.SubTree("Component").List(): if component == "mixed": - continue; - component = component.lower(); - list_filename = '%s%s_%s_source.list' % (Cnf["Dir::Lists"], suite, component); - list_file = utils.open_file(list_filename); + continue + component = component.lower() + list_filename = '%s%s_%s_source.list' % (Cnf["Dir::Lists"], suite, component) + list_file = utils.open_file(list_filename) for line in list_file.readlines(): - file = line[:-1]; + file = line[:-1] try: - utils.parse_changes(file, signing_rules=1); + utils.parse_changes(file, signing_rules=1) except utils.invalid_dsc_format_exc, line: - utils.warn("syntax error in .dsc file '%s', line %s." % (file, line)); - count += 1; + utils.warn("syntax error in .dsc file '%s', line %s." % (file, line)) + count += 1 if count: - utils.warn("Found %s invalid .dsc files." % (count)); + utils.warn("Found %s invalid .dsc files." % (count)) ################################################################################ @@ -141,18 +141,18 @@ def check_override(): print suite print "-"*len(suite) print - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) q = projectB.query(""" SELECT DISTINCT b.package FROM binaries b, bin_associations ba WHERE b.id = ba.bin AND ba.suite = %s AND NOT EXISTS (SELECT 1 FROM override o WHERE o.suite = %s AND o.package = b.package)""" - % (suite_id, suite_id)); + % (suite_id, suite_id)) print q q = projectB.query(""" SELECT DISTINCT s.source FROM source s, src_associations sa WHERE s.id = sa.source AND sa.suite = %s AND NOT EXISTS (SELECT 1 FROM override o WHERE o.suite = %s and o.package = s.source)""" - % (suite_id, suite_id)); + % (suite_id, suite_id)) print q ################################################################################ @@ -162,54 +162,54 @@ SELECT DISTINCT s.source FROM source s, src_associations sa def check_source_in_one_dir(): # Not the most enterprising method, but hey... - broken_count = 0; - q = projectB.query("SELECT id FROM source;"); + broken_count = 0 + q = projectB.query("SELECT id FROM source;") for i in q.getresult(): - source_id = i[0]; + source_id = i[0] q2 = projectB.query(""" SELECT l.path, f.filename FROM files f, dsc_files df, location l WHERE df.source = %s AND f.id = df.file AND l.id = f.location""" - % (source_id)); - first_path = ""; - first_filename = ""; - broken = 0; + % (source_id)) + first_path = "" + first_filename = "" + broken = 0 for j in q2.getresult(): - filename = j[0] + j[1]; - path = os.path.dirname(filename); + filename = j[0] + j[1] + path = os.path.dirname(filename) if first_path == "": - first_path = path; - first_filename = filename; + first_path = path + first_filename = filename elif first_path != path: - symlink = path + '/' + os.path.basename(first_filename); + symlink = path + '/' + os.path.basename(first_filename) if not os.path.exists(symlink): - broken = 1; - print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, source_id, symlink); + broken = 1 + print "WOAH, we got a live one here... %s [%s] {%s}" % (filename, source_id, symlink) if broken: - broken_count += 1; - print "Found %d source packages where the source is not all in one directory." % (broken_count); + broken_count += 1 + print "Found %d source packages where the source is not all in one directory." % (broken_count) ################################################################################ def check_md5sums(): - print "Getting file information from database..."; + print "Getting file information from database..." q = projectB.query("SELECT l.path, f.filename, f.md5sum, f.size FROM files f, location l WHERE f.location = l.id") - ql = q.getresult(); + ql = q.getresult() - print "Checking file md5sums & sizes..."; + print "Checking file md5sums & sizes..." for i in ql: - filename = os.path.abspath(i[0] + i[1]); - db_md5sum = i[2]; - db_size = int(i[3]); + filename = os.path.abspath(i[0] + i[1]) + db_md5sum = i[2] + db_size = int(i[3]) try: - file = utils.open_file(filename); + file = utils.open_file(filename) except: - utils.warn("can't open '%s'." % (filename)); - continue; - md5sum = apt_pkg.md5sum(file); - size = os.stat(filename)[stat.ST_SIZE]; + utils.warn("can't open '%s'." % (filename)) + continue + md5sum = apt_pkg.md5sum(file) + size = os.stat(filename)[stat.ST_SIZE] if md5sum != db_md5sum: - utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, db_md5sum)); + utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, db_md5sum)) if size != db_size: - utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, db_size)); + utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, db_size)) print "Done." @@ -219,259 +219,259 @@ def check_md5sums(): # (e.g. alpha) which have far-future dates as their default dates. def Ent(Kind,Name,Link,Mode,UID,GID,Size,MTime,Major,Minor): - global future_files; + global future_files if MTime > current_time: - future_files[current_file] = MTime; - print "%s: %s '%s','%s',%u,%u,%u,%u,%u,%u,%u" % (current_file, Kind,Name,Link,Mode,UID,GID,Size, MTime, Major, Minor); + future_files[current_file] = MTime + print "%s: %s '%s','%s',%u,%u,%u,%u,%u,%u,%u" % (current_file, Kind,Name,Link,Mode,UID,GID,Size, MTime, Major, Minor) def check_timestamps(): - global current_file; + global current_file q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.deb$'") - ql = q.getresult(); - db_files.clear(); - count = 0; + ql = q.getresult() + db_files.clear() + count = 0 for i in ql: - filename = os.path.abspath(i[0] + i[1]); + filename = os.path.abspath(i[0] + i[1]) if os.access(filename, os.R_OK): - file = utils.open_file(filename); - current_file = filename; - sys.stderr.write("Processing %s.\n" % (filename)); - apt_inst.debExtract(file,Ent,"control.tar.gz"); - file.seek(0); - apt_inst.debExtract(file,Ent,"data.tar.gz"); - count += 1; - print "Checked %d files (out of %d)." % (count, len(db_files.keys())); + file = utils.open_file(filename) + current_file = filename + sys.stderr.write("Processing %s.\n" % (filename)) + apt_inst.debExtract(file,Ent,"control.tar.gz") + file.seek(0) + apt_inst.debExtract(file,Ent,"data.tar.gz") + count += 1 + print "Checked %d files (out of %d)." % (count, len(db_files.keys())) ################################################################################ def check_missing_tar_gz_in_dsc(): - count = 0; + count = 0 - print "Building list of database files..."; - q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.dsc$'"); - ql = q.getresult(); + print "Building list of database files..." + q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id AND f.filename ~ '.dsc$'") + ql = q.getresult() if ql: - print "Checking %d files..." % len(ql); + print "Checking %d files..." % len(ql) else: print "No files to check." for i in ql: - filename = os.path.abspath(i[0] + i[1]); + filename = os.path.abspath(i[0] + i[1]) try: # NB: don't enforce .dsc syntax - dsc = utils.parse_changes(filename); + dsc = utils.parse_changes(filename) except: - utils.fubar("error parsing .dsc file '%s'." % (filename)); - dsc_files = utils.build_file_list(dsc, is_a_dsc=1); - has_tar = 0; + utils.fubar("error parsing .dsc file '%s'." % (filename)) + dsc_files = utils.build_file_list(dsc, is_a_dsc=1) + has_tar = 0 for file in dsc_files.keys(): - m = utils.re_issource.match(file); + m = utils.re_issource.match(file) if not m: - utils.fubar("%s not recognised as source." % (file)); - type = m.group(3); + utils.fubar("%s not recognised as source." % (file)) + type = m.group(3) if type == "orig.tar.gz" or type == "tar.gz": - has_tar = 1; + has_tar = 1 if not has_tar: - utils.warn("%s has no .tar.gz in the .dsc file." % (file)); - count += 1; + utils.warn("%s has no .tar.gz in the .dsc file." % (file)) + count += 1 if count: - utils.warn("Found %s invalid .dsc files." % (count)); + utils.warn("Found %s invalid .dsc files." % (count)) ################################################################################ def validate_sources(suite, component): - filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component); - print "Processing %s..." % (filename); + filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component) + print "Processing %s..." % (filename) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)); + temp_filename = utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): - sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)); - sys.exit(result); - sources = utils.open_file(temp_filename); - Sources = apt_pkg.ParseTagFile(sources); + sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) + sys.exit(result) + sources = utils.open_file(temp_filename) + Sources = apt_pkg.ParseTagFile(sources) while Sources.Step(): - source = Sources.Section.Find('Package'); - directory = Sources.Section.Find('Directory'); - files = Sources.Section.Find('Files'); + source = Sources.Section.Find('Package') + directory = Sources.Section.Find('Directory') + files = Sources.Section.Find('Files') for i in files.split('\n'): - (md5, size, name) = i.split(); - filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name); + (md5, size, name) = i.split() + filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name) if not os.path.exists(filename): if directory.find("potato") == -1: - print "W: %s missing." % (filename); + print "W: %s missing." % (filename) else: - pool_location = utils.poolify (source, component); - pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"], pool_location, name); + pool_location = utils.poolify (source, component) + pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"], pool_location, name) if not os.path.exists(pool_filename): - print "E: %s missing (%s)." % (filename, pool_filename); + print "E: %s missing (%s)." % (filename, pool_filename) else: # Create symlink - pool_filename = os.path.normpath(pool_filename); - filename = os.path.normpath(filename); - src = utils.clean_symlink(pool_filename, filename, Cnf["Dir::Root"]); - print "Symlinking: %s -> %s" % (filename, src); - #os.symlink(src, filename); - sources.close(); - os.unlink(temp_filename); + pool_filename = os.path.normpath(pool_filename) + filename = os.path.normpath(filename) + src = utils.clean_symlink(pool_filename, filename, Cnf["Dir::Root"]) + print "Symlinking: %s -> %s" % (filename, src) + #os.symlink(src, filename) + sources.close() + os.unlink(temp_filename) ######################################## def validate_packages(suite, component, architecture): filename = "%s/dists/%s/%s/binary-%s/Packages.gz" \ - % (Cnf["Dir::Root"], suite, component, architecture); - print "Processing %s..." % (filename); + % (Cnf["Dir::Root"], suite, component, architecture) + print "Processing %s..." % (filename) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)); + temp_filename = utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): - sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)); - sys.exit(result); - packages = utils.open_file(temp_filename); - Packages = apt_pkg.ParseTagFile(packages); + sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) + sys.exit(result) + packages = utils.open_file(temp_filename) + Packages = apt_pkg.ParseTagFile(packages) while Packages.Step(): - filename = "%s/%s" % (Cnf["Dir::Root"], Packages.Section.Find('Filename')); + filename = "%s/%s" % (Cnf["Dir::Root"], Packages.Section.Find('Filename')) if not os.path.exists(filename): - print "W: %s missing." % (filename); - packages.close(); - os.unlink(temp_filename); + print "W: %s missing." % (filename) + packages.close() + os.unlink(temp_filename) ######################################## def check_indices_files_exist(): for suite in [ "stable", "testing", "unstable" ]: for component in Cnf.ValueList("Suite::%s::Components" % (suite)): - architectures = Cnf.ValueList("Suite::%s::Architectures" % (suite)); + architectures = Cnf.ValueList("Suite::%s::Architectures" % (suite)) for arch in map(string.lower, architectures): if arch == "source": - validate_sources(suite, component); + validate_sources(suite, component) elif arch == "all": - continue; + continue else: - validate_packages(suite, component, arch); + validate_packages(suite, component, arch) ################################################################################ def check_files_not_symlinks(): - print "Building list of database files... ",; - before = time.time(); + print "Building list of database files... ", + before = time.time() q = projectB.query("SELECT l.path, f.filename, f.id FROM files f, location l WHERE f.location = l.id") - print "done. (%d seconds)" % (int(time.time()-before)); - q_files = q.getresult(); + print "done. (%d seconds)" % (int(time.time()-before)) + q_files = q.getresult() -# locations = {}; -# q = projectB.query("SELECT l.path, c.name, l.id FROM location l, component c WHERE l.component = c.id"); +# locations = {} +# q = projectB.query("SELECT l.path, c.name, l.id FROM location l, component c WHERE l.component = c.id") # for i in q.getresult(): -# path = os.path.normpath(i[0] + i[1]); -# locations[path] = (i[0], i[2]); +# path = os.path.normpath(i[0] + i[1]) +# locations[path] = (i[0], i[2]) -# q = projectB.query("BEGIN WORK"); +# q = projectB.query("BEGIN WORK") for i in q_files: - filename = os.path.normpath(i[0] + i[1]); -# file_id = i[2]; + filename = os.path.normpath(i[0] + i[1]) +# file_id = i[2] if os.access(filename, os.R_OK) == 0: - utils.warn("%s: doesn't exist." % (filename)); + utils.warn("%s: doesn't exist." % (filename)) else: if os.path.islink(filename): - utils.warn("%s: is a symlink." % (filename)); + utils.warn("%s: is a symlink." % (filename)) # You probably don't want to use the rest of this... -# print "%s: is a symlink." % (filename); -# dest = os.readlink(filename); +# print "%s: is a symlink." % (filename) +# dest = os.readlink(filename) # if not os.path.isabs(dest): -# dest = os.path.normpath(os.path.join(os.path.dirname(filename), dest)); -# print "--> %s" % (dest); +# dest = os.path.normpath(os.path.join(os.path.dirname(filename), dest)) +# print "--> %s" % (dest) # # Determine suitable location ID # # [in what must be the suckiest way possible?] -# location_id = None; +# location_id = None # for path in locations.keys(): # if dest.find(path) == 0: -# (location, location_id) = locations[path]; -# break; +# (location, location_id) = locations[path] +# break # if not location_id: -# utils.fubar("Can't find location for %s (%s)." % (dest, filename)); -# new_filename = dest.replace(location, ""); -# q = projectB.query("UPDATE files SET filename = '%s', location = %s WHERE id = %s" % (new_filename, location_id, file_id)); -# q = projectB.query("COMMIT WORK"); +# utils.fubar("Can't find location for %s (%s)." % (dest, filename)) +# new_filename = dest.replace(location, "") +# q = projectB.query("UPDATE files SET filename = '%s', location = %s WHERE id = %s" % (new_filename, location_id, file_id)) +# q = projectB.query("COMMIT WORK") ################################################################################ def chk_bd_process_dir (unused, dirname, filenames): for name in filenames: if not name.endswith(".dsc"): - continue; - filename = os.path.abspath(dirname+'/'+name); - dsc = utils.parse_changes(filename); + continue + filename = os.path.abspath(dirname+'/'+name) + dsc = utils.parse_changes(filename) for field_name in [ "build-depends", "build-depends-indep" ]: - field = dsc.get(field_name); + field = dsc.get(field_name) if field: try: - apt_pkg.ParseSrcDepends(field); + apt_pkg.ParseSrcDepends(field) except: - print "E: [%s] %s: %s" % (filename, field_name, field); - pass; + print "E: [%s] %s: %s" % (filename, field_name, field) + pass ################################################################################ def check_build_depends(): - os.path.walk(Cnf["Dir::Root"], chk_bd_process_dir, None); + os.path.walk(Cnf["Dir::Root"], chk_bd_process_dir, None) ################################################################################ def main (): - global Cnf, projectB, db_files, waste, excluded; + global Cnf, projectB, db_files, waste, excluded - Cnf = utils.get_conf(); - Arguments = [('h',"help","Tea::Options::Help")]; + Cnf = utils.get_conf() + Arguments = [('h',"help","Tea::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Tea::Options::%s" % (i)): - Cnf["Tea::Options::%s" % (i)] = ""; + Cnf["Tea::Options::%s" % (i)] = "" - args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Tea::Options") if Options["Help"]: - usage(); + usage() if len(args) < 1: - utils.warn("tea requires at least one argument"); - usage(1); + utils.warn("tea requires at least one argument") + usage(1) elif len(args) > 1: - utils.warn("tea accepts only one argument"); - usage(1); - mode = args[0].lower(); + utils.warn("tea accepts only one argument") + usage(1) + mode = args[0].lower() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) if mode == "md5sums": - check_md5sums(); + check_md5sums() elif mode == "files": - check_files(); + check_files() elif mode == "dsc-syntax": - check_dscs(); + check_dscs() elif mode == "missing-overrides": - check_override(); + check_override() elif mode == "source-in-one-dir": - check_source_in_one_dir(); + check_source_in_one_dir() elif mode == "timestamps": - check_timestamps(); + check_timestamps() elif mode == "tar-gz-in-dsc": - check_missing_tar_gz_in_dsc(); + check_missing_tar_gz_in_dsc() elif mode == "validate-indices": - check_indices_files_exist(); + check_indices_files_exist() elif mode == "files-not-symlinks": - check_files_not_symlinks(); + check_files_not_symlinks() elif mode == "validate-builddeps": - check_build_depends(); + check_build_depends() else: - utils.warn("unknown mode '%s'" % (mode)); - usage(1); + utils.warn("unknown mode '%s'" % (mode)) + usage(1) ################################################################################ if __name__ == '__main__': - main(); + main() diff --git a/dak/check_overrides.py b/dak/check_overrides.py index b95997f7..282bce08 100755 --- a/dak/check_overrides.py +++ b/dak/check_overrides.py @@ -47,14 +47,14 @@ ################################################################################ -import pg, sys, os; -import utils, db_access, logging; -import apt_pkg; +import pg, sys, os +import utils, db_access, logging +import apt_pkg ################################################################################ -Options = None; -projectB = None; +Options = None +projectB = None Logger = None sections = {} priorities = {} @@ -79,58 +79,58 @@ def gen_blacklist(dir): blacklist[entry] = 1 def process(osuite, affected_suites, originosuite, component, type): - global Logger, Options, projectB, sections, priorities; + global Logger, Options, projectB, sections, priorities - osuite_id = db_access.get_suite_id(osuite); + osuite_id = db_access.get_suite_id(osuite) if osuite_id == -1: - utils.fubar("Suite '%s' not recognised." % (osuite)); + utils.fubar("Suite '%s' not recognised." % (osuite)) originosuite_id = None if originosuite: - originosuite_id = db_access.get_suite_id(originosuite); + originosuite_id = db_access.get_suite_id(originosuite) if originosuite_id == -1: - utils.fubar("Suite '%s' not recognised." % (originosuite)); + utils.fubar("Suite '%s' not recognised." % (originosuite)) - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) if component_id == -1: - utils.fubar("Component '%s' not recognised." % (component)); + utils.fubar("Component '%s' not recognised." % (component)) - type_id = db_access.get_override_type_id(type); + type_id = db_access.get_override_type_id(type) if type_id == -1: - utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)); - dsc_type_id = db_access.get_override_type_id("dsc"); + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)) + dsc_type_id = db_access.get_override_type_id("dsc") deb_type_id = db_access.get_override_type_id("deb") source_priority_id = db_access.get_priority_id("source") if type == "deb" or type == "udeb": - packages = {}; + packages = {} q = projectB.query(""" SELECT b.package FROM binaries b, bin_associations ba, files f, location l, component c WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s -""" % (type, ",".join(map(str,affected_suites)), component_id)); +""" % (type, ",".join(map(str,affected_suites)), component_id)) for i in q.getresult(): - packages[i[0]] = 0; + packages[i[0]] = 0 - src_packages = {}; + src_packages = {} q = projectB.query(""" SELECT s.source FROM source s, src_associations sa, files f, location l, component c WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s -""" % (",".join(map(str,affected_suites)), component_id)); +""" % (",".join(map(str,affected_suites)), component_id)) for i in q.getresult(): - src_packages[i[0]] = 0; + src_packages[i[0]] = 0 # ----------- # Drop unused overrides - q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)); - projectB.query("BEGIN WORK"); + q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)) + projectB.query("BEGIN WORK") if type == "dsc": for i in q.getresult(): - package = i[0]; + package = i[0] if src_packages.has_key(package): src_packages[package] = 1 else: @@ -142,12 +142,12 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, if not Options["No-Action"]: projectB.query("""DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)); + %s""" % (package, osuite_id, component_id, type_id)) # create source overrides based on binary overrides, as source # overrides not always get created q = projectB.query(""" SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s - """ % (osuite_id, component_id)); + """ % (osuite_id, component_id)) for i in q.getresult(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: @@ -161,7 +161,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, component, priority, section, type, maintainer) VALUES ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])); + dsc_type_id, i[3])) # Check whether originosuite has an override for us we can # copy if originosuite: @@ -172,7 +172,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, target.suite=%s AND origin.component = target.component AND origin.type = target.type) WHERE origin.suite = %s AND origin.component = %s AND origin.type = %s""" % - (osuite_id, originosuite_id, component_id, type_id)); + (osuite_id, originosuite_id, component_id, type_id)) for i in q.getresult(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: @@ -184,7 +184,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, maintainer='%s' WHERE package='%s' AND suite=%s AND component=%s AND type=%s""" % (i[2], i[3], package, osuite_id, component_id, - dsc_type_id)); + dsc_type_id)) continue # we can copy src_packages[package] = 1 @@ -195,7 +195,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, component, priority, section, type, maintainer) VALUES ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])); + dsc_type_id, i[3])) for package, hasoverride in src_packages.items(): if not hasoverride: @@ -203,7 +203,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, else: # binary override for i in q.getresult(): - package = i[0]; + package = i[0] if packages.has_key(package): packages[package] = 1 else: @@ -215,7 +215,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, if not Options["No-Action"]: projectB.query("""DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)); + %s""" % (package, osuite_id, component_id, type_id)) # Check whether originosuite has an override for us we can # copy @@ -227,7 +227,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, target.suite=%s AND origin.component = target.component AND origin.type = target.type) WHERE origin.suite = %s AND origin.component = %s AND origin.type = %s""" % (osuite_id, - originosuite_id, component_id, type_id)); + originosuite_id, component_id, type_id)) for i in q.getresult(): package = i[0] if not packages.has_key(package) or packages[package]: @@ -240,7 +240,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, maintainer='%s' WHERE package='%s' AND suite=%s AND component=%s AND type=%s""" % (i[1], i[2], i[3], package, osuite_id, - component_id, type_id)); + component_id, type_id)) continue # we can copy packages[package] = 1 @@ -249,36 +249,36 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, if not Options["No-Action"]: projectB.query("""INSERT INTO override (package, suite, component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])); + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])) for package, hasoverride in packages.items(): if not hasoverride: utils.warn("%s has no override!" % package) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") sys.stdout.flush() ################################################################################ def main (): - global Logger, Options, projectB, sections, priorities; + global Logger, Options, projectB, sections, priorities Cnf = utils.get_conf() Arguments = [('h',"help","Cindy::Options::Help"), - ('n',"no-action", "Cindy::Options::No-Action")]; + ('n',"no-action", "Cindy::Options::No-Action")] for i in [ "help", "no-action" ]: if not Cnf.has_key("Cindy::Options::%s" % (i)): - Cnf["Cindy::Options::%s" % (i)] = ""; - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + Cnf["Cindy::Options::%s" % (i)] = "" + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Cindy::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) # init sections, priorities: q = projectB.query("SELECT id, section FROM section") @@ -304,13 +304,13 @@ def main (): originosuite = None originremark = "" try: - originosuite = Cnf["Cindy::OverrideSuites::%s::OriginSuite" % osuite]; + originosuite = Cnf["Cindy::OverrideSuites::%s::OriginSuite" % osuite] originosuite = originosuite.lower() originremark = " taking missing from %s" % originosuite except KeyError: pass - print "Processing %s%s..." % (osuite, originremark); + print "Processing %s%s..." % (osuite, originremark) # Get a list of all suites that use the override file of 'osuite' ocodename = Cnf["Suite::%s::codename" % osuite] suites = [] @@ -338,9 +338,9 @@ def main (): otypes = ["dsc"] + otypes for otype in otypes: print "Processing %s [%s - %s] using %s..." \ - % (osuite, component, otype, suites); + % (osuite, component, otype, suites) sys.stdout.flush() - process(osuite, suiteids, originosuite, component, otype); + process(osuite, suiteids, originosuite, component, otype) Logger.close() diff --git a/dak/check_proposed_updates.py b/dak/check_proposed_updates.py index 91f51930..d935399e 100755 --- a/dak/check_proposed_updates.py +++ b/dak/check_proposed_updates.py @@ -29,18 +29,18 @@ ################################################################################ -import pg, sys, os; +import pg, sys, os import utils, db_access -import apt_pkg, apt_inst; +import apt_pkg, apt_inst ################################################################################ -Cnf = None; -projectB = None; -Options = None; -stable = {}; -stable_virtual = {}; -architectures = None; +Cnf = None +projectB = None +Options = None +stable = {} +stable_virtual = {} +architectures = None ################################################################################ @@ -59,105 +59,105 @@ Need either changes files, deb files or an admin.txt file with a '.joey' suffix. def d_test (dict, key, positive, negative): if not dict: - return negative; + return negative if dict.has_key(key): - return positive; + return positive else: - return negative; + return negative ################################################################################ def check_dep (depends, dep_type, check_archs, filename, files): - pkg_unsat = 0; + pkg_unsat = 0 for arch in check_archs: for parsed_dep in apt_pkg.ParseDepends(depends): - unsat = []; + unsat = [] for atom in parsed_dep: - (dep, version, constraint) = atom; + (dep, version, constraint) = atom # As a real package? if stable.has_key(dep): if stable[dep].has_key(arch): if apt_pkg.CheckDep(stable[dep][arch], constraint, version): if Options["debug"]: - print "Found %s as a real package." % (utils.pp_deps(parsed_dep)); - unsat = 0; - break; + print "Found %s as a real package." % (utils.pp_deps(parsed_dep)) + unsat = 0 + break # As a virtual? if stable_virtual.has_key(dep): if stable_virtual[dep].has_key(arch): if not constraint and not version: if Options["debug"]: - print "Found %s as a virtual package." % (utils.pp_deps(parsed_dep)); - unsat = 0; - break; + print "Found %s as a virtual package." % (utils.pp_deps(parsed_dep)) + unsat = 0 + break # As part of the same .changes? epochless_version = utils.re_no_epoch.sub('', version) - dep_filename = "%s_%s_%s.deb" % (dep, epochless_version, arch); + dep_filename = "%s_%s_%s.deb" % (dep, epochless_version, arch) if files.has_key(dep_filename): if Options["debug"]: - print "Found %s in the same upload." % (utils.pp_deps(parsed_dep)); - unsat = 0; - break; + print "Found %s in the same upload." % (utils.pp_deps(parsed_dep)) + unsat = 0 + break # Not found... # [FIXME: must be a better way ... ] error = "%s not found. [Real: " % (utils.pp_deps(parsed_dep)) if stable.has_key(dep): if stable[dep].has_key(arch): - error += "%s:%s:%s" % (dep, arch, stable[dep][arch]); + error += "%s:%s:%s" % (dep, arch, stable[dep][arch]) else: - error += "%s:-:-" % (dep); + error += "%s:-:-" % (dep) else: - error += "-:-:-"; - error += ", Virtual: "; + error += "-:-:-" + error += ", Virtual: " if stable_virtual.has_key(dep): if stable_virtual[dep].has_key(arch): - error += "%s:%s" % (dep, arch); + error += "%s:%s" % (dep, arch) else: - error += "%s:-"; + error += "%s:-" else: - error += "-:-"; - error += ", Upload: "; + error += "-:-" + error += ", Upload: " if files.has_key(dep_filename): - error += "yes"; + error += "yes" else: - error += "no"; - error += "]"; - unsat.append(error); + error += "no" + error += "]" + unsat.append(error) if unsat: - sys.stderr.write("MWAAP! %s: '%s' %s can not be satisifed:\n" % (filename, utils.pp_deps(parsed_dep), dep_type)); + sys.stderr.write("MWAAP! %s: '%s' %s can not be satisifed:\n" % (filename, utils.pp_deps(parsed_dep), dep_type)) for error in unsat: - sys.stderr.write(" %s\n" % (error)); - pkg_unsat = 1; + sys.stderr.write(" %s\n" % (error)) + pkg_unsat = 1 - return pkg_unsat; + return pkg_unsat def check_package(filename, files): try: - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(filename))); + control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(filename))) except: - utils.warn("%s: debExtractControl() raised %s." % (filename, sys.exc_type)); - return 1; - Depends = control.Find("Depends"); - Pre_Depends = control.Find("Pre-Depends"); - #Recommends = control.Find("Recommends"); - pkg_arch = control.Find("Architecture"); - base_file = os.path.basename(filename); + utils.warn("%s: debExtractControl() raised %s." % (filename, sys.exc_type)) + return 1 + Depends = control.Find("Depends") + Pre_Depends = control.Find("Pre-Depends") + #Recommends = control.Find("Recommends") + pkg_arch = control.Find("Architecture") + base_file = os.path.basename(filename) if pkg_arch == "all": - check_archs = architectures; + check_archs = architectures else: - check_archs = [pkg_arch]; + check_archs = [pkg_arch] - pkg_unsat = 0; + pkg_unsat = 0 if Pre_Depends: - pkg_unsat += check_dep(Pre_Depends, "pre-dependency", check_archs, base_file, files); + pkg_unsat += check_dep(Pre_Depends, "pre-dependency", check_archs, base_file, files) if Depends: - pkg_unsat += check_dep(Depends, "dependency", check_archs, base_file, files); + pkg_unsat += check_dep(Depends, "dependency", check_archs, base_file, files) #if Recommends: - #pkg_unsat += check_dep(Recommends, "recommendation", check_archs, base_file, files); + #pkg_unsat += check_dep(Recommends, "recommendation", check_archs, base_file, files) - return pkg_unsat; + return pkg_unsat ################################################################################ @@ -165,142 +165,142 @@ def pass_fail (filename, result): if not Options["quiet"]: print "%s:" % (os.path.basename(filename)), if result: - print "FAIL"; + print "FAIL" else: - print "ok"; + print "ok" ################################################################################ def check_changes (filename): try: - changes = utils.parse_changes(filename); - files = utils.build_file_list(changes); + changes = utils.parse_changes(filename) + files = utils.build_file_list(changes) except: - utils.warn("Error parsing changes file '%s'" % (filename)); - return; + utils.warn("Error parsing changes file '%s'" % (filename)) + return - result = 0; + result = 0 # Move to the pool directory - cwd = os.getcwd(); - file = files.keys()[0]; - pool_dir = Cnf["Dir::Pool"] + '/' + utils.poolify(changes["source"], files[file]["component"]); - os.chdir(pool_dir); + cwd = os.getcwd() + file = files.keys()[0] + pool_dir = Cnf["Dir::Pool"] + '/' + utils.poolify(changes["source"], files[file]["component"]) + os.chdir(pool_dir) - changes_result = 0; + changes_result = 0 for file in files.keys(): if file.endswith(".deb"): - result = check_package(file, files); + result = check_package(file, files) if Options["verbose"]: - pass_fail(file, result); - changes_result += result; + pass_fail(file, result) + changes_result += result - pass_fail (filename, changes_result); + pass_fail (filename, changes_result) # Move back - os.chdir(cwd); + os.chdir(cwd) ################################################################################ def check_deb (filename): - result = check_package(filename, {}); - pass_fail(filename, result); + result = check_package(filename, {}) + pass_fail(filename, result) ################################################################################ def check_joey (filename): - file = utils.open_file(filename); + file = utils.open_file(filename) - cwd = os.getcwd(); - os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"])); + cwd = os.getcwd() + os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"])) for line in file.readlines(): - line = line.rstrip(); + line = line.rstrip() if line.find('install') != -1: - split_line = line.split(); + split_line = line.split() if len(split_line) != 2: - utils.fubar("Parse error (not exactly 2 elements): %s" % (line)); - install_type = split_line[0]; + utils.fubar("Parse error (not exactly 2 elements): %s" % (line)) + install_type = split_line[0] if install_type not in [ "install", "install-u", "sync-install" ]: - utils.fubar("Unknown install type ('%s') from: %s" % (install_type, line)); + utils.fubar("Unknown install type ('%s') from: %s" % (install_type, line)) changes_filename = split_line[1] if Options["debug"]: - print "Processing %s..." % (changes_filename); - check_changes(changes_filename); - file.close(); + print "Processing %s..." % (changes_filename) + check_changes(changes_filename) + file.close() - os.chdir(cwd); + os.chdir(cwd) ################################################################################ def parse_packages(): - global stable, stable_virtual, architectures; + global stable, stable_virtual, architectures # Parse the Packages files (since it's a sub-second operation on auric) - suite = "stable"; - stable = {}; - components = Cnf.ValueList("Suite::%s::Components" % (suite)); - architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))); + suite = "stable" + stable = {} + components = Cnf.ValueList("Suite::%s::Components" % (suite)) + architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))) for component in components: for architecture in architectures: - filename = "%s/dists/%s/%s/binary-%s/Packages" % (Cnf["Dir::Root"], suite, component, architecture); - packages = utils.open_file(filename, 'r'); - Packages = apt_pkg.ParseTagFile(packages); + filename = "%s/dists/%s/%s/binary-%s/Packages" % (Cnf["Dir::Root"], suite, component, architecture) + packages = utils.open_file(filename, 'r') + Packages = apt_pkg.ParseTagFile(packages) while Packages.Step(): - package = Packages.Section.Find('Package'); - version = Packages.Section.Find('Version'); - provides = Packages.Section.Find('Provides'); + package = Packages.Section.Find('Package') + version = Packages.Section.Find('Version') + provides = Packages.Section.Find('Provides') if not stable.has_key(package): - stable[package] = {}; - stable[package][architecture] = version; + stable[package] = {} + stable[package][architecture] = version if provides: for virtual_pkg in provides.split(","): - virtual_pkg = virtual_pkg.strip(); + virtual_pkg = virtual_pkg.strip() if not stable_virtual.has_key(virtual_pkg): - stable_virtual[virtual_pkg] = {}; - stable_virtual[virtual_pkg][architecture] = "NA"; + stable_virtual[virtual_pkg] = {} + stable_virtual[virtual_pkg][architecture] = "NA" packages.close() ################################################################################ def main (): - global Cnf, projectB, Options; + global Cnf, projectB, Options Cnf = utils.get_conf() Arguments = [('d', "debug", "Jeri::Options::Debug"), ('q',"quiet","Jeri::Options::Quiet"), ('v',"verbose","Jeri::Options::Verbose"), - ('h',"help","Jeri::Options::Help")]; + ('h',"help","Jeri::Options::Help")] for i in [ "debug", "quiet", "verbose", "help" ]: if not Cnf.has_key("Jeri::Options::%s" % (i)): - Cnf["Jeri::Options::%s" % (i)] = ""; + Cnf["Jeri::Options::%s" % (i)] = "" - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Jeri::Options") if Options["Help"]: - usage(0); + usage(0) if not arguments: - utils.fubar("need at least one package name as an argument."); + utils.fubar("need at least one package name as an argument.") - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) print "Parsing packages files...", - parse_packages(); - print "done."; + parse_packages() + print "done." for file in arguments: if file.endswith(".changes"): - check_changes(file); + check_changes(file) elif file.endswith(".deb"): - check_deb(file); + check_deb(file) elif file.endswith(".joey"): - check_joey(file); + check_joey(file) else: - utils.fubar("Unrecognised file type: '%s'." % (file)); + utils.fubar("Unrecognised file type: '%s'." % (file)) ####################################################################################### diff --git a/dak/clean_proposed_updates.py b/dak/clean_proposed_updates.py index 29fd9240..9faf7eb1 100755 --- a/dak/clean_proposed_updates.py +++ b/dak/clean_proposed_updates.py @@ -20,18 +20,18 @@ ################################################################################ -import os, pg, re, sys; -import utils, db_access; -import apt_pkg; +import os, pg, re, sys +import utils, db_access +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; -Options = None; -pu = {}; +Cnf = None +projectB = None +Options = None +pu = {} -re_isdeb = re.compile (r"^(.+)_(.+?)_(.+?).u?deb$"); +re_isdeb = re.compile (r"^(.+)_(.+?)_(.+?).u?deb$") ################################################################################ @@ -49,91 +49,91 @@ Need either changes files or an admin.txt file with a '.joey' suffix.""" def check_changes (filename): try: - changes = utils.parse_changes(filename); - files = utils.build_file_list(changes); + changes = utils.parse_changes(filename) + files = utils.build_file_list(changes) except: - utils.warn("Couldn't read changes file '%s'." % (filename)); - return; - num_files = len(files.keys()); + utils.warn("Couldn't read changes file '%s'." % (filename)) + return + num_files = len(files.keys()) for file in files.keys(): if utils.re_isadeb.match(file): - m = re_isdeb.match(file); - pkg = m.group(1); - version = m.group(2); - arch = m.group(3); + m = re_isdeb.match(file) + pkg = m.group(1) + version = m.group(2) + arch = m.group(3) if Options["debug"]: - print "BINARY: %s ==> %s_%s_%s" % (file, pkg, version, arch); + print "BINARY: %s ==> %s_%s_%s" % (file, pkg, version, arch) else: m = utils.re_issource.match(file) if m: - pkg = m.group(1); - version = m.group(2); - type = m.group(3); + pkg = m.group(1) + version = m.group(2) + type = m.group(3) if type != "dsc": - del files[file]; - num_files -= 1; - continue; - arch = "source"; + del files[file] + num_files -= 1 + continue + arch = "source" if Options["debug"]: - print "SOURCE: %s ==> %s_%s_%s" % (file, pkg, version, arch); + print "SOURCE: %s ==> %s_%s_%s" % (file, pkg, version, arch) else: - utils.fubar("unknown type, fix me"); + utils.fubar("unknown type, fix me") if not pu.has_key(pkg): # FIXME - utils.warn("%s doesn't seem to exist in p-u?? (from %s [%s])" % (pkg, file, filename)); - continue; + utils.warn("%s doesn't seem to exist in p-u?? (from %s [%s])" % (pkg, file, filename)) + continue if not pu[pkg].has_key(arch): # FIXME - utils.warn("%s doesn't seem to exist for %s in p-u?? (from %s [%s])" % (pkg, arch, file, filename)); - continue; - pu_version = utils.re_no_epoch.sub('', pu[pkg][arch]); + utils.warn("%s doesn't seem to exist for %s in p-u?? (from %s [%s])" % (pkg, arch, file, filename)) + continue + pu_version = utils.re_no_epoch.sub('', pu[pkg][arch]) if pu_version == version: if Options["verbose"]: - print "%s: ok" % (file); + print "%s: ok" % (file) else: if Options["verbose"]: - print "%s: superseded, removing. [%s]" % (file, pu_version); - del files[file]; + print "%s: superseded, removing. [%s]" % (file, pu_version) + del files[file] - new_num_files = len(files.keys()); + new_num_files = len(files.keys()) if new_num_files == 0: - print "%s: no files left, superseded by %s" % (filename, pu_version); - dest = Cnf["Dir::Morgue"] + "/misc/"; - utils.move(filename, dest); + print "%s: no files left, superseded by %s" % (filename, pu_version) + dest = Cnf["Dir::Morgue"] + "/misc/" + utils.move(filename, dest) elif new_num_files < num_files: - print "%s: lost files, MWAAP." % (filename); + print "%s: lost files, MWAAP." % (filename) else: if Options["verbose"]: - print "%s: ok" % (filename); + print "%s: ok" % (filename) ################################################################################ def check_joey (filename): - file = utils.open_file(filename); + file = utils.open_file(filename) - cwd = os.getcwd(); - os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"])); + cwd = os.getcwd() + os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"])) for line in file.readlines(): - line = line.rstrip(); + line = line.rstrip() if line.find('install') != -1: - split_line = line.split(); + split_line = line.split() if len(split_line) != 2: - utils.fubar("Parse error (not exactly 2 elements): %s" % (line)); - install_type = split_line[0]; + utils.fubar("Parse error (not exactly 2 elements): %s" % (line)) + install_type = split_line[0] if install_type not in [ "install", "install-u", "sync-install" ]: - utils.fubar("Unknown install type ('%s') from: %s" % (install_type, line)); + utils.fubar("Unknown install type ('%s') from: %s" % (install_type, line)) changes_filename = split_line[1] if Options["debug"]: - print "Processing %s..." % (changes_filename); - check_changes(changes_filename); + print "Processing %s..." % (changes_filename) + check_changes(changes_filename) - os.chdir(cwd); + os.chdir(cwd) ################################################################################ def init_pu (): - global pu; + global pu q = projectB.query(""" SELECT b.package, b.version, a.arch_string @@ -144,49 +144,49 @@ UNION SELECT s.source, s.version, 'source' FROM src_associations sa, source s, suite su WHERE s.id = sa.source AND sa.suite = su.id AND su.suite_name = 'proposed-updates' -ORDER BY package, version, arch_string; -"""); - ql = q.getresult(); +ORDER BY package, version, arch_string +""") + ql = q.getresult() for i in ql: - pkg = i[0]; - version = i[1]; - arch = i[2]; + pkg = i[0] + version = i[1] + arch = i[2] if not pu.has_key(pkg): - pu[pkg] = {}; - pu[pkg][arch] = version; + pu[pkg] = {} + pu[pkg][arch] = version def main (): - global Cnf, projectB, Options; + global Cnf, projectB, Options Cnf = utils.get_conf() Arguments = [('d', "debug", "Halle::Options::Debug"), ('v',"verbose","Halle::Options::Verbose"), - ('h',"help","Halle::Options::Help")]; + ('h',"help","Halle::Options::Help")] for i in [ "debug", "verbose", "help" ]: if not Cnf.has_key("Halle::Options::%s" % (i)): - Cnf["Halle::Options::%s" % (i)] = ""; + Cnf["Halle::Options::%s" % (i)] = "" - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Halle::Options") if Options["Help"]: - usage(0); + usage(0) if not arguments: - utils.fubar("need at least one package name as an argument."); + utils.fubar("need at least one package name as an argument.") - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) - init_pu(); + init_pu() for file in arguments: if file.endswith(".changes"): - check_changes(file); + check_changes(file) elif file.endswith(".joey"): - check_joey(file); + check_joey(file) else: - utils.fubar("Unrecognised file type: '%s'." % (file)); + utils.fubar("Unrecognised file type: '%s'." % (file)) ####################################################################################### diff --git a/dak/clean_queues.py b/dak/clean_queues.py index 74a76055..32d8cd7e 100755 --- a/dak/clean_queues.py +++ b/dak/clean_queues.py @@ -34,16 +34,16 @@ ################################################################################ -import os, stat, sys, time; -import utils; -import apt_pkg; +import os, stat, sys, time +import utils +import apt_pkg ################################################################################ -Cnf = None; -Options = None; -del_dir = None; -delete_date = None; +Cnf = None +Options = None +del_dir = None +delete_date = None ################################################################################ @@ -62,35 +62,35 @@ Clean out incoming directories. ################################################################################ def init (): - global delete_date, del_dir; + global delete_date, del_dir - delete_date = int(time.time())-(int(Options["Days"])*84600); + delete_date = int(time.time())-(int(Options["Days"])*84600) # Ensure a directory exists to remove files to if not Options["No-Action"]: - date = time.strftime("%Y-%m-%d"); - del_dir = Cnf["Dir::Morgue"] + '/' + Cnf["Shania::MorgueSubDir"] + '/' + date; + date = time.strftime("%Y-%m-%d") + del_dir = Cnf["Dir::Morgue"] + '/' + Cnf["Shania::MorgueSubDir"] + '/' + date if not os.path.exists(del_dir): - os.makedirs(del_dir, 02775); + os.makedirs(del_dir, 02775) if not os.path.isdir(del_dir): - utils.fubar("%s must be a directory." % (del_dir)); + utils.fubar("%s must be a directory." % (del_dir)) # Move to the directory to clean - incoming = Options["Incoming"]; + incoming = Options["Incoming"] if incoming == "": - incoming = Cnf["Dir::Queue::Unchecked"]; - os.chdir(incoming); + incoming = Cnf["Dir::Queue::Unchecked"] + os.chdir(incoming) # Remove a file to the morgue def remove (file): if os.access(file, os.R_OK): - dest_filename = del_dir + '/' + os.path.basename(file); + dest_filename = del_dir + '/' + os.path.basename(file) # If the destination file exists; try to find another filename to use if os.path.exists(dest_filename): - dest_filename = utils.find_next_free(dest_filename, 10); - utils.move(file, dest_filename, 0660); + dest_filename = utils.find_next_free(dest_filename, 10) + utils.move(file, dest_filename, 0660) else: - utils.warn("skipping '%s', permission denied." % (os.path.basename(file))); + utils.warn("skipping '%s', permission denied." % (os.path.basename(file))) # Removes any old files. # [Used for Incoming/REJECT] @@ -100,111 +100,111 @@ def flush_old (): if os.path.isfile(file): if os.stat(file)[stat.ST_MTIME] < delete_date: if Options["No-Action"]: - print "I: Would delete '%s'." % (os.path.basename(file)); + print "I: Would delete '%s'." % (os.path.basename(file)) else: if Options["Verbose"]: - print "Removing '%s' (to '%s')." % (os.path.basename(file), del_dir); - remove(file); + print "Removing '%s' (to '%s')." % (os.path.basename(file), del_dir) + remove(file) else: if Options["Verbose"]: - print "Skipping, too new, '%s'." % (os.path.basename(file)); + print "Skipping, too new, '%s'." % (os.path.basename(file)) # Removes any files which are old orphans (not associated with a valid .changes file). # [Used for Incoming] # def flush_orphans (): - all_files = {}; - changes_files = []; + all_files = {} + changes_files = [] # Build up the list of all files in the directory for i in os.listdir('.'): if os.path.isfile(i): - all_files[i] = 1; + all_files[i] = 1 if i.endswith(".changes"): - changes_files.append(i); + changes_files.append(i) # Proces all .changes and .dsc files. for changes_filename in changes_files: try: - changes = utils.parse_changes(changes_filename); - files = utils.build_file_list(changes); + changes = utils.parse_changes(changes_filename) + files = utils.build_file_list(changes) except: - utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_type)); - continue; + utils.warn("error processing '%s'; skipping it. [Got %s]" % (changes_filename, sys.exc_type)) + continue - dsc_files = {}; + dsc_files = {} for file in files.keys(): if file.endswith(".dsc"): try: - dsc = utils.parse_changes(file); - dsc_files = utils.build_file_list(dsc, is_a_dsc=1); + dsc = utils.parse_changes(file) + dsc_files = utils.build_file_list(dsc, is_a_dsc=1) except: - utils.warn("error processing '%s'; skipping it. [Got %s]" % (file, sys.exc_type)); - continue; + utils.warn("error processing '%s'; skipping it. [Got %s]" % (file, sys.exc_type)) + continue # Ensure all the files we've seen aren't deleted - keys = []; + keys = [] for i in (files.keys(), dsc_files.keys(), [changes_filename]): - keys.extend(i); + keys.extend(i) for key in keys: if all_files.has_key(key): if Options["Verbose"]: - print "Skipping, has parents, '%s'." % (key); - del all_files[key]; + print "Skipping, has parents, '%s'." % (key) + del all_files[key] # Anthing left at this stage is not referenced by a .changes (or # a .dsc) and should be deleted if old enough. for file in all_files.keys(): if os.stat(file)[stat.ST_MTIME] < delete_date: if Options["No-Action"]: - print "I: Would delete '%s'." % (os.path.basename(file)); + print "I: Would delete '%s'." % (os.path.basename(file)) else: if Options["Verbose"]: - print "Removing '%s' (to '%s')." % (os.path.basename(file), del_dir); - remove(file); + print "Removing '%s' (to '%s')." % (os.path.basename(file), del_dir) + remove(file) else: if Options["Verbose"]: - print "Skipping, too new, '%s'." % (os.path.basename(file)); + print "Skipping, too new, '%s'." % (os.path.basename(file)) ################################################################################ def main (): - global Cnf, Options; + global Cnf, Options Cnf = utils.get_conf() for i in ["Help", "Incoming", "No-Action", "Verbose" ]: if not Cnf.has_key("Shania::Options::%s" % (i)): - Cnf["Shania::Options::%s" % (i)] = ""; + Cnf["Shania::Options::%s" % (i)] = "" if not Cnf.has_key("Shania::Options::Days"): - Cnf["Shania::Options::Days"] = "14"; + Cnf["Shania::Options::Days"] = "14" Arguments = [('h',"help","Shania::Options::Help"), ('d',"days","Shania::Options::Days", "IntLevel"), ('i',"incoming","Shania::Options::Incoming", "HasArg"), ('n',"no-action","Shania::Options::No-Action"), - ('v',"verbose","Shania::Options::Verbose")]; + ('v',"verbose","Shania::Options::Verbose")] - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Shania::Options") if Options["Help"]: - usage(); + usage() - init(); + init() if Options["Verbose"]: print "Processing incoming..." - flush_orphans(); + flush_orphans() reject = Cnf["Dir::Queue::Reject"] if os.path.exists(reject) and os.path.isdir(reject): if Options["Verbose"]: print "Processing incoming/REJECT..." - os.chdir(reject); - flush_old(); + os.chdir(reject) + flush_old() ####################################################################################### if __name__ == '__main__': - main(); + main() diff --git a/dak/clean_suites.py b/dak/clean_suites.py index fdbcf26d..fbeacb67 100755 --- a/dak/clean_suites.py +++ b/dak/clean_suites.py @@ -35,9 +35,9 @@ import utils ################################################################################ -projectB = None; -Cnf = None; -Options = None; +projectB = None +Cnf = None +Options = None now_date = None; # mark newly "deleted" things as deleted "now" delete_date = None; # delete things marked "deleted" earler than this @@ -54,7 +54,7 @@ Clean old packages from suites. ################################################################################ def check_binaries(): - global delete_date, now_date; + global delete_date, now_date print "Checking for orphaned binary packages..." @@ -63,33 +63,33 @@ def check_binaries(): q = projectB.query(""" SELECT b.file FROM binaries b, files f WHERE f.last_used IS NULL AND b.file = f.id - AND NOT EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)"""); - ql = q.getresult(); + AND NOT EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""") + ql = q.getresult() - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for i in ql: - file_id = i[0]; + file_id = i[0] projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id)) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") # Check for any binaries which are marked for eventual deletion # but are now used again. q = projectB.query(""" SELECT b.file FROM binaries b, files f WHERE f.last_used IS NOT NULL AND f.id = b.file - AND EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)"""); - ql = q.getresult(); + AND EXISTS (SELECT 1 FROM bin_associations ba WHERE ba.bin = b.id)""") + ql = q.getresult() - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for i in ql: - file_id = i[0]; - projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id)); - projectB.query("COMMIT WORK"); + file_id = i[0] + projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id)) + projectB.query("COMMIT WORK") ######################################## def check_sources(): - global delete_date, now_date; + global delete_date, now_date print "Checking for orphaned source packages..." @@ -99,29 +99,29 @@ def check_sources(): SELECT s.id, s.file FROM source s, files f WHERE f.last_used IS NULL AND s.file = f.id AND NOT EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id) - AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)"""); + AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)""") #### XXX: this should ignore cases where the files for the binary b #### have been marked for deletion (so the delay between bins go #### byebye and sources go byebye is 0 instead of StayOfExecution) - ql = q.getresult(); + ql = q.getresult() - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for i in ql: - source_id = i[0]; - dsc_file_id = i[1]; + source_id = i[0] + dsc_file_id = i[1] # Mark the .dsc file for deletion projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, dsc_file_id)) # Mark all other files references by .dsc too if they're not used by anyone else - x = projectB.query("SELECT f.id FROM files f, dsc_files d WHERE d.source = %s AND d.file = f.id" % (source_id)); + x = projectB.query("SELECT f.id FROM files f, dsc_files d WHERE d.source = %s AND d.file = f.id" % (source_id)) for j in x.getresult(): - file_id = j[0]; - y = projectB.query("SELECT id FROM dsc_files d WHERE d.file = %s" % (file_id)); + file_id = j[0] + y = projectB.query("SELECT id FROM dsc_files d WHERE d.file = %s" % (file_id)) if len(y.getresult()) == 1: - projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id)); - projectB.query("COMMIT WORK"); + projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s AND last_used IS NULL" % (now_date, file_id)) + projectB.query("COMMIT WORK") # Check for any sources which are marked for deletion but which # are now used again. @@ -130,44 +130,44 @@ SELECT s.id, s.file FROM source s, files f SELECT f.id FROM source s, files f, dsc_files df WHERE f.last_used IS NOT NULL AND s.id = df.source AND df.file = f.id AND ((EXISTS (SELECT 1 FROM src_associations sa WHERE sa.source = s.id)) - OR (EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)))"""); + OR (EXISTS (SELECT 1 FROM binaries b WHERE b.source = s.id)))""") #### XXX: this should also handle deleted binaries specially (ie, not #### reinstate sources because of them - ql = q.getresult(); + ql = q.getresult() # Could be done in SQL; but left this way for hysterical raisins # [and freedom to innovate don'cha know?] - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for i in ql: - file_id = i[0]; - projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id)); - projectB.query("COMMIT WORK"); + file_id = i[0] + projectB.query("UPDATE files SET last_used = NULL WHERE id = %s" % (file_id)) + projectB.query("COMMIT WORK") ######################################## def check_files(): - global delete_date, now_date; + global delete_date, now_date # FIXME: this is evil; nothing should ever be in this state. if # they are, it's a bug and the files should not be auto-deleted. - return; + return print "Checking for unused files..." q = projectB.query(""" SELECT id FROM files f WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.file = f.id) - AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)"""); + AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = f.id)""") - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for i in q.getresult(): - file_id = i[0]; - projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s" % (now_date, file_id)); - projectB.query("COMMIT WORK"); + file_id = i[0] + projectB.query("UPDATE files SET last_used = '%s' WHERE id = %s" % (now_date, file_id)) + projectB.query("COMMIT WORK") def clean_binaries(): - global delete_date, now_date; + global delete_date, now_date # We do this here so that the binaries we remove will have their # source also removed (if possible). @@ -176,72 +176,72 @@ def clean_binaries(): # buys anything keeping this separate print "Cleaning binaries from the DB..." if not Options["No-Action"]: - before = time.time(); - sys.stdout.write("[Deleting from binaries table... "); - sys.stderr.write("DELETE FROM binaries WHERE EXISTS (SELECT 1 FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')\n" % (delete_date)); - projectB.query("DELETE FROM binaries WHERE EXISTS (SELECT 1 FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')" % (delete_date)); - sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))); + before = time.time() + sys.stdout.write("[Deleting from binaries table... ") + sys.stderr.write("DELETE FROM binaries WHERE EXISTS (SELECT 1 FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')\n" % (delete_date)) + projectB.query("DELETE FROM binaries WHERE EXISTS (SELECT 1 FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')" % (delete_date)) + sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))) ######################################## def clean(): - global delete_date, now_date; - count = 0; - size = 0; + global delete_date, now_date + count = 0 + size = 0 print "Cleaning out packages..." - date = time.strftime("%Y-%m-%d"); - dest = Cnf["Dir::Morgue"] + '/' + Cnf["Rhona::MorgueSubDir"] + '/' + date; + date = time.strftime("%Y-%m-%d") + dest = Cnf["Dir::Morgue"] + '/' + Cnf["Rhona::MorgueSubDir"] + '/' + date if not os.path.exists(dest): - os.mkdir(dest); + os.mkdir(dest) # Delete from source if not Options["No-Action"]: - before = time.time(); - sys.stdout.write("[Deleting from source table... "); - projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date)); - projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date)); - sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))); + before = time.time() + sys.stdout.write("[Deleting from source table... ") + projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date)) + projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date)) + sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))) # Delete files from the pool - q = projectB.query("SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)); + q = projectB.query("SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)) for i in q.getresult(): - filename = i[0] + i[1]; + filename = i[0] + i[1] if not os.path.exists(filename): - utils.warn("can not find '%s'." % (filename)); - continue; + utils.warn("can not find '%s'." % (filename)) + continue if os.path.isfile(filename): if os.path.islink(filename): - count += 1; + count += 1 if Options["No-Action"]: - print "Removing symlink %s..." % (filename); + print "Removing symlink %s..." % (filename) else: - os.unlink(filename); + os.unlink(filename) else: - size += os.stat(filename)[stat.ST_SIZE]; - count += 1; + size += os.stat(filename)[stat.ST_SIZE] + count += 1 - dest_filename = dest + '/' + os.path.basename(filename); + dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.exists(dest_filename): - dest_filename = utils.find_next_free(dest_filename); + dest_filename = utils.find_next_free(dest_filename) if Options["No-Action"]: - print "Cleaning %s -> %s ..." % (filename, dest_filename); + print "Cleaning %s -> %s ..." % (filename, dest_filename) else: - utils.move(filename, dest_filename); + utils.move(filename, dest_filename) else: - utils.fubar("%s is neither symlink nor file?!" % (filename)); + utils.fubar("%s is neither symlink nor file?!" % (filename)) # Delete from the 'files' table if not Options["No-Action"]: - before = time.time(); - sys.stdout.write("[Deleting from files table... "); - projectB.query("DELETE FROM files WHERE last_used <= '%s'" % (delete_date)); - sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))); + before = time.time() + sys.stdout.write("[Deleting from files table... ") + projectB.query("DELETE FROM files WHERE last_used <= '%s'" % (delete_date)) + sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))) if count > 0: - sys.stderr.write("Cleaned %d files, %s.\n" % (count, utils.size_type(size))); + sys.stderr.write("Cleaned %d files, %s.\n" % (count, utils.size_type(size))) ################################################################################ @@ -251,20 +251,20 @@ def clean_maintainers(): q = projectB.query(""" SELECT m.id FROM maintainer m WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id) - AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id)"""); - ql = q.getresult(); + AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id)""") + ql = q.getresult() - count = 0; - projectB.query("BEGIN WORK"); + count = 0 + projectB.query("BEGIN WORK") for i in ql: - maintainer_id = i[0]; + maintainer_id = i[0] if not Options["No-Action"]: - projectB.query("DELETE FROM maintainer WHERE id = %s" % (maintainer_id)); - count += 1; - projectB.query("COMMIT WORK"); + projectB.query("DELETE FROM maintainer WHERE id = %s" % (maintainer_id)) + count += 1 + projectB.query("COMMIT WORK") if count > 0: - sys.stderr.write("Cleared out %d maintainer entries.\n" % (count)); + sys.stderr.write("Cleared out %d maintainer entries.\n" % (count)) ################################################################################ @@ -274,81 +274,81 @@ def clean_fingerprints(): q = projectB.query(""" SELECT f.id FROM fingerprint f WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id) - AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)"""); - ql = q.getresult(); + AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""") + ql = q.getresult() - count = 0; - projectB.query("BEGIN WORK"); + count = 0 + projectB.query("BEGIN WORK") for i in ql: - fingerprint_id = i[0]; + fingerprint_id = i[0] if not Options["No-Action"]: - projectB.query("DELETE FROM fingerprint WHERE id = %s" % (fingerprint_id)); - count += 1; - projectB.query("COMMIT WORK"); + projectB.query("DELETE FROM fingerprint WHERE id = %s" % (fingerprint_id)) + count += 1 + projectB.query("COMMIT WORK") if count > 0: - sys.stderr.write("Cleared out %d fingerprint entries.\n" % (count)); + sys.stderr.write("Cleared out %d fingerprint entries.\n" % (count)) ################################################################################ def clean_queue_build(): - global now_date; + global now_date if not Cnf.ValueList("Dinstall::QueueBuildSuites") or Options["No-Action"]: - return; + return print "Cleaning out queue build symlinks..." - our_delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Rhona::QueueBuildStayOfExecution"]))); - count = 0; + our_delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Rhona::QueueBuildStayOfExecution"]))) + count = 0 - q = projectB.query("SELECT filename FROM queue_build WHERE last_used <= '%s'" % (our_delete_date)); + q = projectB.query("SELECT filename FROM queue_build WHERE last_used <= '%s'" % (our_delete_date)) for i in q.getresult(): - filename = i[0]; + filename = i[0] if not os.path.exists(filename): - utils.warn("%s (from queue_build) doesn't exist." % (filename)); - continue; + utils.warn("%s (from queue_build) doesn't exist." % (filename)) + continue if not Cnf.FindB("Dinstall::SecurityQueueBuild") and not os.path.islink(filename): - utils.fubar("%s (from queue_build) should be a symlink but isn't." % (filename)); - os.unlink(filename); - count += 1; - projectB.query("DELETE FROM queue_build WHERE last_used <= '%s'" % (our_delete_date)); + utils.fubar("%s (from queue_build) should be a symlink but isn't." % (filename)) + os.unlink(filename) + count += 1 + projectB.query("DELETE FROM queue_build WHERE last_used <= '%s'" % (our_delete_date)) if count: - sys.stderr.write("Cleaned %d queue_build files.\n" % (count)); + sys.stderr.write("Cleaned %d queue_build files.\n" % (count)) ################################################################################ def main(): - global Cnf, Options, projectB, delete_date, now_date; + global Cnf, Options, projectB, delete_date, now_date Cnf = utils.get_conf() for i in ["Help", "No-Action" ]: if not Cnf.has_key("Rhona::Options::%s" % (i)): - Cnf["Rhona::Options::%s" % (i)] = ""; + Cnf["Rhona::Options::%s" % (i)] = "" Arguments = [('h',"help","Rhona::Options::Help"), - ('n',"no-action","Rhona::Options::No-Action")]; + ('n',"no-action","Rhona::Options::No-Action")] - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Rhona::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - now_date = time.strftime("%Y-%m-%d %H:%M"); - delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Rhona::StayOfExecution"]))); + now_date = time.strftime("%Y-%m-%d %H:%M") + delete_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time()-int(Cnf["Rhona::StayOfExecution"]))) - check_binaries(); - clean_binaries(); - check_sources(); - check_files(); - clean(); - clean_maintainers(); - clean_fingerprints(); - clean_queue_build(); + check_binaries() + clean_binaries() + check_sources() + check_files() + clean() + clean_maintainers() + clean_fingerprints() + clean_queue_build() ################################################################################ diff --git a/dak/compare_suites.py b/dak/compare_suites.py index ea0bbc21..cbceeb11 100755 --- a/dak/compare_suites.py +++ b/dak/compare_suites.py @@ -21,14 +21,14 @@ ################################################################################ -import pg, sys; -import utils, db_access; -import apt_pkg; +import pg, sys +import utils, db_access +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ @@ -42,42 +42,42 @@ Looks for fixable descrepancies between stable and unstable. ################################################################################ def main (): - global Cnf, projectB; + global Cnf, projectB - Cnf = utils.get_conf(); - Arguments = [('h',"help","Andrea::Options::Help")]; + Cnf = utils.get_conf() + Arguments = [('h',"help","Andrea::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Andrea::Options::%s" % (i)): - Cnf["Andrea::Options::%s" % (i)] = ""; + Cnf["Andrea::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Andrea::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) - src_suite = "stable"; - dst_suite = "unstable"; + src_suite = "stable" + dst_suite = "unstable" - src_suite_id = db_access.get_suite_id(src_suite); - dst_suite_id = db_access.get_suite_id(dst_suite); - arch_all_id = db_access.get_architecture_id("all"); - dsc_type_id = db_access.get_override_type_id("dsc"); + src_suite_id = db_access.get_suite_id(src_suite) + dst_suite_id = db_access.get_suite_id(dst_suite) + arch_all_id = db_access.get_architecture_id("all") + dsc_type_id = db_access.get_override_type_id("dsc") for arch in Cnf.ValueList("Suite::%s::Architectures" % (src_suite)): if arch == "source": - continue; + continue # Arch: all doesn't work; consider packages which go from # arch: all to arch: any, e.g. debconf... needs more checks # and thought later. if arch == "all": - continue; - arch_id = db_access.get_architecture_id(arch); + continue + arch_id = db_access.get_architecture_id(arch) q = projectB.query(""" SELECT b_src.package, b_src.version, a.arch_string FROM binaries b_src, bin_associations ba, override o, architecture a @@ -91,9 +91,9 @@ SELECT b_src.package, b_src.version, a.arch_string (SELECT 1 FROM bin_associations ba3, binaries b2 WHERE ba3.bin = b2.id AND ba3.suite = %s AND b2.package = b_dst.package)) ORDER BY b_src.package;""" - % (src_suite_id, arch_id, dst_suite_id, dsc_type_id, arch_id, arch_all_id, dst_suite_id, dst_suite_id)); + % (src_suite_id, arch_id, dst_suite_id, dsc_type_id, arch_id, arch_all_id, dst_suite_id, dst_suite_id)) for i in q.getresult(): - print " ".join(i); + print " ".join(i) ####################################################################################### diff --git a/dak/control_overrides.py b/dak/control_overrides.py index 4a5d62bc..4015f2ae 100755 --- a/dak/control_overrides.py +++ b/dak/control_overrides.py @@ -50,15 +50,15 @@ ################################################################################ -import pg, sys, time; -import utils, db_access, logging; -import apt_pkg; +import pg, sys, time +import utils, db_access, logging +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; -Logger = None; +Cnf = None +projectB = None +Logger = None ################################################################################ @@ -85,162 +85,162 @@ def usage (exit_code=0): ################################################################################ def process_file (file, suite, component, type, action): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id == -1: - utils.fubar("Suite '%s' not recognised." % (suite)); + utils.fubar("Suite '%s' not recognised." % (suite)) - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) if component_id == -1: - utils.fubar("Component '%s' not recognised." % (component)); + utils.fubar("Component '%s' not recognised." % (component)) - type_id = db_access.get_override_type_id(type); + type_id = db_access.get_override_type_id(type) if type_id == -1: - utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (type)); + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)" % (type)) # --set is done mostly internal for performance reasons; most # invocations of --set will be updates and making people wait 2-3 # minutes while 6000 select+inserts are run needlessly isn't cool. - original = {}; - new = {}; - c_skipped = 0; - c_added = 0; - c_updated = 0; - c_removed = 0; - c_error = 0; + original = {} + new = {} + c_skipped = 0 + c_added = 0 + c_updated = 0 + c_removed = 0 + c_error = 0 q = projectB.query("SELECT o.package, o.priority, o.section, o.maintainer, p.priority, s.section FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s and o.priority = p.id and o.section = s.id" - % (suite_id, component_id, type_id)); + % (suite_id, component_id, type_id)) for i in q.getresult(): - original[i[0]] = i[1:]; + original[i[0]] = i[1:] - start_time = time.time(); - projectB.query("BEGIN WORK"); + start_time = time.time() + projectB.query("BEGIN WORK") for line in file.readlines(): - line = utils.re_comments.sub('', line).strip(); + line = utils.re_comments.sub('', line).strip() if line == "": - continue; + continue - maintainer_override = None; + maintainer_override = None if type == "dsc": - split_line = line.split(None, 2); + split_line = line.split(None, 2) if len(split_line) == 2: - (package, section) = split_line; + (package, section) = split_line elif len(split_line) == 3: - (package, section, maintainer_override) = split_line; + (package, section, maintainer_override) = split_line else: - utils.warn("'%s' does not break into 'package section [maintainer-override]'." % (line)); - c_error += 1; - continue; - priority = "source"; + utils.warn("'%s' does not break into 'package section [maintainer-override]'." % (line)) + c_error += 1 + continue + priority = "source" else: # binary or udeb - split_line = line.split(None, 3); + split_line = line.split(None, 3) if len(split_line) == 3: - (package, priority, section) = split_line; + (package, priority, section) = split_line elif len(split_line) == 4: - (package, priority, section, maintainer_override) = split_line; + (package, priority, section, maintainer_override) = split_line else: - utils.warn("'%s' does not break into 'package priority section [maintainer-override]'." % (line)); - c_error += 1; - continue; + utils.warn("'%s' does not break into 'package priority section [maintainer-override]'." % (line)) + c_error += 1 + continue - section_id = db_access.get_section_id(section); + section_id = db_access.get_section_id(section) if section_id == -1: - utils.warn("'%s' is not a valid section. ['%s' in suite %s, component %s]." % (section, package, suite, component)); - c_error += 1; - continue; - priority_id = db_access.get_priority_id(priority); + utils.warn("'%s' is not a valid section. ['%s' in suite %s, component %s]." % (section, package, suite, component)) + c_error += 1 + continue + priority_id = db_access.get_priority_id(priority) if priority_id == -1: - utils.warn("'%s' is not a valid priority. ['%s' in suite %s, component %s]." % (priority, package, suite, component)); - c_error += 1; - continue; + utils.warn("'%s' is not a valid priority. ['%s' in suite %s, component %s]." % (priority, package, suite, component)) + c_error += 1 + continue if new.has_key(package): - utils.warn("Can't insert duplicate entry for '%s'; ignoring all but the first. [suite %s, component %s]" % (package, suite, component)); - c_error += 1; - continue; - new[package] = ""; + utils.warn("Can't insert duplicate entry for '%s'; ignoring all but the first. [suite %s, component %s]" % (package, suite, component)) + c_error += 1 + continue + new[package] = "" if original.has_key(package): - (old_priority_id, old_section_id, old_maintainer_override, old_priority, old_section) = original[package]; + (old_priority_id, old_section_id, old_maintainer_override, old_priority, old_section) = original[package] if action == "add" or old_priority_id == priority_id and \ old_section_id == section_id and \ ((old_maintainer_override == maintainer_override) or \ (old_maintainer_override == "" and maintainer_override == None)): # If it's unchanged or we're in 'add only' mode, ignore it - c_skipped += 1; - continue; + c_skipped += 1 + continue else: # If it's changed, delete the old one so we can # reinsert it with the new information - c_updated += 1; + c_updated += 1 projectB.query("DELETE FROM override WHERE suite = %s AND component = %s AND package = '%s' AND type = %s" - % (suite_id, component_id, package, type_id)); + % (suite_id, component_id, package, type_id)) # Log changes if old_priority_id != priority_id: - Logger.log(["changed priority",package,old_priority,priority]); + Logger.log(["changed priority",package,old_priority,priority]) if old_section_id != section_id: - Logger.log(["changed section",package,old_section,section]); + Logger.log(["changed section",package,old_section,section]) if old_maintainer_override != maintainer_override: - Logger.log(["changed maintainer override",package,old_maintainer_override,maintainer_override]); - update_p = 1; + Logger.log(["changed maintainer override",package,old_maintainer_override,maintainer_override]) + update_p = 1 else: - c_added += 1; - update_p = 0; + c_added += 1 + update_p = 0 if maintainer_override: projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '%s')" - % (suite_id, component_id, type_id, package, priority_id, section_id, maintainer_override)); + % (suite_id, component_id, type_id, package, priority_id, section_id, maintainer_override)) else: projectB.query("INSERT INTO override (suite, component, type, package, priority, section,maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" - % (suite_id, component_id, type_id, package, priority_id, section_id)); + % (suite_id, component_id, type_id, package, priority_id, section_id)) if not update_p: - Logger.log(["new override",suite,component,type,package,priority,section,maintainer_override]); + Logger.log(["new override",suite,component,type,package,priority,section,maintainer_override]) if not action == "add": # Delete any packages which were removed for package in original.keys(): if not new.has_key(package): projectB.query("DELETE FROM override WHERE suite = %s AND component = %s AND package = '%s' AND type = %s" - % (suite_id, component_id, package, type_id)); - c_removed += 1; - Logger.log(["removed override",suite,component,type,package]); + % (suite_id, component_id, package, type_id)) + c_removed += 1 + Logger.log(["removed override",suite,component,type,package]) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") if not Cnf["Natalie::Options::Quiet"]: - print "Done in %d seconds. [Updated = %d, Added = %d, Removed = %d, Skipped = %d, Errors = %d]" % (int(time.time()-start_time), c_updated, c_added, c_removed, c_skipped, c_error); - Logger.log(["set complete",c_updated, c_added, c_removed, c_skipped, c_error]); + print "Done in %d seconds. [Updated = %d, Added = %d, Removed = %d, Skipped = %d, Errors = %d]" % (int(time.time()-start_time), c_updated, c_added, c_removed, c_skipped, c_error) + Logger.log(["set complete",c_updated, c_added, c_removed, c_skipped, c_error]) ################################################################################ def list(suite, component, type): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id == -1: - utils.fubar("Suite '%s' not recognised." % (suite)); + utils.fubar("Suite '%s' not recognised." % (suite)) - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) if component_id == -1: - utils.fubar("Component '%s' not recognised." % (component)); + utils.fubar("Component '%s' not recognised." % (component)) - type_id = db_access.get_override_type_id(type); + type_id = db_access.get_override_type_id(type) if type_id == -1: - utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)); + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)) if type == "dsc": - q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, type_id)); + q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, type_id)) for i in q.getresult(): - print utils.result_join(i); + print utils.result_join(i) else: - q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, type_id)); + q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, type_id)) for i in q.getresult(): - print utils.result_join(i[:-1]); + print utils.result_join(i[:-1]) ################################################################################ def main (): - global Cnf, projectB, Logger; + global Cnf, projectB, Logger - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('a', "add", "Natalie::Options::Add"), ('c', "component", "Natalie::Options::Component", "HasArg"), ('h', "help", "Natalie::Options::Help"), @@ -248,46 +248,46 @@ def main (): ('q', "quiet", "Natalie::Options::Quiet"), ('s', "suite", "Natalie::Options::Suite", "HasArg"), ('S', "set", "Natalie::Options::Set"), - ('t', "type", "Natalie::Options::Type", "HasArg")]; + ('t', "type", "Natalie::Options::Type", "HasArg")] # Default arguments for i in [ "add", "help", "list", "quiet", "set" ]: if not Cnf.has_key("Natalie::Options::%s" % (i)): - Cnf["Natalie::Options::%s" % (i)] = ""; + Cnf["Natalie::Options::%s" % (i)] = "" if not Cnf.has_key("Natalie::Options::Component"): - Cnf["Natalie::Options::Component"] = "main"; + Cnf["Natalie::Options::Component"] = "main" if not Cnf.has_key("Natalie::Options::Suite"): - Cnf["Natalie::Options::Suite"] = "unstable"; + Cnf["Natalie::Options::Suite"] = "unstable" if not Cnf.has_key("Natalie::Options::Type"): - Cnf["Natalie::Options::Type"] = "deb"; + Cnf["Natalie::Options::Type"] = "deb" - file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) if Cnf["Natalie::Options::Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) - action = None; + action = None for i in [ "add", "list", "set" ]: if Cnf["Natalie::Options::%s" % (i)]: if action: - utils.fubar("Can not perform more than one action at once."); - action = i; + utils.fubar("Can not perform more than one action at once.") + action = i (suite, component, type) = (Cnf["Natalie::Options::Suite"], Cnf["Natalie::Options::Component"], Cnf["Natalie::Options::Type"]) if action == "list": - list(suite, component, type); + list(suite, component, type) else: - Logger = logging.Logger(Cnf, "natalie"); + Logger = logging.Logger(Cnf, "natalie") if file_list: for file in file_list: - process_file(utils.open_file(file), suite, component, type, action); + process_file(utils.open_file(file), suite, component, type, action) else: - process_file(sys.stdin, suite, component, type, action); - Logger.close(); + process_file(sys.stdin, suite, component, type, action) + Logger.close() ####################################################################################### diff --git a/dak/control_suite.py b/dak/control_suite.py index 2454b234..fb3f5fcc 100755 --- a/dak/control_suite.py +++ b/dak/control_suite.py @@ -42,15 +42,15 @@ ####################################################################################### -import pg, sys; -import apt_pkg; -import utils, db_access, logging; +import pg, sys +import apt_pkg +import utils, db_access, logging ####################################################################################### -Cnf = None; -projectB = None; -Logger = None; +Cnf = None +projectB = None +Logger = None ################################################################################ @@ -74,163 +74,163 @@ def get_id (package, version, architecture): else: q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) - ql = q.getresult(); + ql = q.getresult() if not ql: - utils.warn("Couldn't find '%s~%s~%s'." % (package, version, architecture)); - return None; + utils.warn("Couldn't find '%s~%s~%s'." % (package, version, architecture)) + return None if len(ql) > 1: - utils.warn("Found more than one match for '%s~%s~%s'." % (package, version, architecture)); - return None; - id = ql[0][0]; - return id; + utils.warn("Found more than one match for '%s~%s~%s'." % (package, version, architecture)) + return None + id = ql[0][0] + return id ####################################################################################### def set_suite (file, suite_id): - lines = file.readlines(); + lines = file.readlines() - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") # Build up a dictionary of what is currently in the suite - current = {}; - q = projectB.query("SELECT b.package, b.version, a.arch_string, ba.id FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id)); - ql = q.getresult(); + current = {} + q = projectB.query("SELECT b.package, b.version, a.arch_string, ba.id FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id)) + ql = q.getresult() for i in ql: - key = " ".join(i[:3]); - current[key] = i[3]; - q = projectB.query("SELECT s.source, s.version, sa.id FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id)); - ql = q.getresult(); + key = " ".join(i[:3]) + current[key] = i[3] + q = projectB.query("SELECT s.source, s.version, sa.id FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id)) + ql = q.getresult() for i in ql: - key = " ".join(i[:2]) + " source"; - current[key] = i[2]; + key = " ".join(i[:2]) + " source" + current[key] = i[2] # Build up a dictionary of what should be in the suite - desired = {}; + desired = {} for line in lines: - split_line = line.strip().split(); + split_line = line.strip().split() if len(split_line) != 3: - utils.warn("'%s' does not break into 'package version architecture'." % (line[:-1])); - continue; - key = " ".join(split_line); - desired[key] = ""; + utils.warn("'%s' does not break into 'package version architecture'." % (line[:-1])) + continue + key = " ".join(split_line) + desired[key] = "" # Check to see which packages need removed and remove them for key in current.keys(): if not desired.has_key(key): - (package, version, architecture) = key.split(); - id = current[key]; + (package, version, architecture) = key.split() + id = current[key] if architecture == "source": - q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (id)); + q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (id)) else: - q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (id)); - Logger.log(["removed",key,id]); + q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (id)) + Logger.log(["removed",key,id]) # Check to see which packages need added and add them for key in desired.keys(): if not current.has_key(key): - (package, version, architecture) = key.split(); - id = get_id (package, version, architecture); + (package, version, architecture) = key.split() + id = get_id (package, version, architecture) if not id: - continue; + continue if architecture == "source": - q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, id)); + q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, id)) else: - q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, id)); - Logger.log(["added",key,id]); + q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, id)) + Logger.log(["added",key,id]) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") ####################################################################################### def process_file (file, suite, action): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if action == "set": - set_suite (file, suite_id); - return; + set_suite (file, suite_id) + return - lines = file.readlines(); + lines = file.readlines() - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for line in lines: - split_line = line.strip().split(); + split_line = line.strip().split() if len(split_line) != 3: - utils.warn("'%s' does not break into 'package version architecture'." % (line[:-1])); - continue; + utils.warn("'%s' does not break into 'package version architecture'." % (line[:-1])) + continue - (package, version, architecture) = split_line; + (package, version, architecture) = split_line - id = get_id(package, version, architecture); + id = get_id(package, version, architecture) if not id: - continue; + continue if architecture == "source": # Find the existing assoications ID, if any - q = projectB.query("SELECT id FROM src_associations WHERE suite = %s and source = %s" % (suite_id, id)); - ql = q.getresult(); + q = projectB.query("SELECT id FROM src_associations WHERE suite = %s and source = %s" % (suite_id, id)) + ql = q.getresult() if not ql: - assoication_id = None; + assoication_id = None else: - assoication_id = ql[0][0]; + assoication_id = ql[0][0] # Take action if action == "add": if assoication_id: - utils.warn("'%s~%s~%s' already exists in suite %s." % (package, version, architecture, suite)); - continue; + utils.warn("'%s~%s~%s' already exists in suite %s." % (package, version, architecture, suite)) + continue else: - q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, id)); + q = projectB.query("INSERT INTO src_associations (suite, source) VALUES (%s, %s)" % (suite_id, id)) elif action == "remove": if assoication_id == None: - utils.warn("'%s~%s~%s' doesn't exist in suite %s." % (package, version, architecture, suite)); - continue; + utils.warn("'%s~%s~%s' doesn't exist in suite %s." % (package, version, architecture, suite)) + continue else: - q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (assoication_id)); + q = projectB.query("DELETE FROM src_associations WHERE id = %s" % (assoication_id)) else: # Find the existing assoications ID, if any - q = projectB.query("SELECT id FROM bin_associations WHERE suite = %s and bin = %s" % (suite_id, id)); - ql = q.getresult(); + q = projectB.query("SELECT id FROM bin_associations WHERE suite = %s and bin = %s" % (suite_id, id)) + ql = q.getresult() if not ql: - assoication_id = None; + assoication_id = None else: - assoication_id = ql[0][0]; + assoication_id = ql[0][0] # Take action if action == "add": if assoication_id: - utils.warn("'%s~%s~%s' already exists in suite %s." % (package, version, architecture, suite)); - continue; + utils.warn("'%s~%s~%s' already exists in suite %s." % (package, version, architecture, suite)) + continue else: - q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, id)); + q = projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%s, %s)" % (suite_id, id)) elif action == "remove": if assoication_id == None: - utils.warn("'%s~%s~%s' doesn't exist in suite %s." % (package, version, architecture, suite)); - continue; + utils.warn("'%s~%s~%s' doesn't exist in suite %s." % (package, version, architecture, suite)) + continue else: - q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (assoication_id)); + q = projectB.query("DELETE FROM bin_associations WHERE id = %s" % (assoication_id)) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") ####################################################################################### def get_list (suite): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) # List binaries - q = projectB.query("SELECT b.package, b.version, a.arch_string FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id)); - ql = q.getresult(); + q = projectB.query("SELECT b.package, b.version, a.arch_string FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id" % (suite_id)) + ql = q.getresult() for i in ql: - print " ".join(i); + print " ".join(i) # List source - q = projectB.query("SELECT s.source, s.version FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id)); - ql = q.getresult(); + q = projectB.query("SELECT s.source, s.version FROM source s, src_associations sa WHERE sa.suite = %s AND sa.source = s.id" % (suite_id)) + ql = q.getresult() for i in ql: - print " ".join(i) + " source"; + print " ".join(i) + " source" ####################################################################################### def main (): - global Cnf, projectB, Logger; + global Cnf, projectB, Logger Cnf = utils.get_conf() @@ -238,52 +238,52 @@ def main (): ('h',"help","Heidi::Options::Help"), ('l',"list","Heidi::Options::List","HasArg"), ('r',"remove", "Heidi::Options::Remove", "HasArg"), - ('s',"set", "Heidi::Options::Set", "HasArg")]; + ('s',"set", "Heidi::Options::Set", "HasArg")] for i in ["add", "help", "list", "remove", "set", "version" ]: if not Cnf.has_key("Heidi::Options::%s" % (i)): - Cnf["Heidi::Options::%s" % (i)] = ""; + Cnf["Heidi::Options::%s" % (i)] = "" - file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + file_list = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Heidi::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"],int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"],int(Cnf["DB::Port"])) - db_access.init(Cnf, projectB); + db_access.init(Cnf, projectB) - action = None; + action = None for i in ("add", "list", "remove", "set"): if Cnf["Heidi::Options::%s" % (i)] != "": - suite = Cnf["Heidi::Options::%s" % (i)]; + suite = Cnf["Heidi::Options::%s" % (i)] if db_access.get_suite_id(suite) == -1: - utils.fubar("Unknown suite '%s'." %(suite)); + utils.fubar("Unknown suite '%s'." %(suite)) else: if action: - utils.fubar("Can only perform one action at a time."); - action = i; + utils.fubar("Can only perform one action at a time.") + action = i # Need an action... if action == None: - utils.fubar("No action specified."); + utils.fubar("No action specified.") # Safety/Sanity check if action == "set" and suite != "testing": - utils.fubar("Will not reset a suite other than testing."); + utils.fubar("Will not reset a suite other than testing.") if action == "list": - get_list(suite); + get_list(suite) else: - Logger = logging.Logger(Cnf, "heidi"); + Logger = logging.Logger(Cnf, "heidi") if file_list: for file in file_list: - process_file(utils.open_file(file), suite, action); + process_file(utils.open_file(file), suite, action) else: - process_file(sys.stdin, suite, action); - Logger.close(); + process_file(sys.stdin, suite, action) + Logger.close() ####################################################################################### diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 4d588a8b..803a7752 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -28,19 +28,19 @@ ################################################################################ -import commands, pg, os, string, sys, time; -import utils, db_access; -import apt_pkg; +import commands, pg, os, string, sys, time +import utils, db_access +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; -suite_id = None; +Cnf = None +projectB = None +suite_id = None no_longer_in_suite = {}; # Really should be static to add_nbs, but I'm lazy -source_binaries = {}; -source_versions = {}; +source_binaries = {} +source_versions = {} ################################################################################ @@ -58,67 +58,67 @@ Check for obsolete or duplicated packages. def add_nbs(nbs_d, source, version, package): # Ensure the package is still in the suite (someone may have already removed it) if no_longer_in_suite.has_key(package): - return; + return else: - q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package)); + q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package)) if not q.getresult(): - no_longer_in_suite[package] = ""; - return; + no_longer_in_suite[package] = "" + return nbs_d.setdefault(source, {}) nbs_d[source].setdefault(version, {}) - nbs_d[source][version][package] = ""; + nbs_d[source][version][package] = "" ################################################################################ # Check for packages built on architectures they shouldn't be. def do_anais(architecture, binaries_list, source): if architecture == "any" or architecture == "all": - return ""; + return "" - anais_output = ""; - architectures = {}; + anais_output = "" + architectures = {} for arch in architecture.split(): - architectures[arch.strip()] = ""; + architectures[arch.strip()] = "" for binary in binaries_list: - q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary)); - ql = q.getresult(); - versions = []; + q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary)) + ql = q.getresult() + versions = [] for i in ql: - arch = i[0]; - version = i[1]; + arch = i[0] + version = i[1] if architectures.has_key(arch): - versions.append(version); - versions.sort(apt_pkg.VersionCompare); + versions.append(version) + versions.sort(apt_pkg.VersionCompare) if versions: latest_version = versions.pop() else: - latest_version = None; + latest_version = None # Check for 'invalid' architectures versions_d = {} for i in ql: - arch = i[0]; - version = i[1]; + arch = i[0] + version = i[1] if not architectures.has_key(arch): versions_d.setdefault(version, []) versions_d[version].append(arch) if versions_d != {}: - anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture); - versions = versions_d.keys(); - versions.sort(apt_pkg.VersionCompare); + anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture) + versions = versions_d.keys() + versions.sort(apt_pkg.VersionCompare) for version in versions: - arches = versions_d[version]; - arches.sort(); - anais_output += " o %s: %s\n" % (version, ", ".join(arches)); - return anais_output; + arches = versions_d[version] + arches.sort() + anais_output += " o %s: %s\n" % (version, ", ".join(arches)) + return anais_output ################################################################################ def do_nviu(): - experimental_id = db_access.get_suite_id("experimental"); + experimental_id = db_access.get_suite_id("experimental") if experimental_id == -1: - return; + return # Check for packages in experimental obsoleted by versions in unstable q = projectB.query(""" SELECT s.source, s.version AS experimental, s2.version AS unstable @@ -126,76 +126,76 @@ SELECT s.source, s.version AS experimental, s2.version AS unstable WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id AND sa2.source = s2.id AND s.source = s2.source AND versioncmp(s.version, s2.version) < 0""" % (experimental_id, - db_access.get_suite_id("unstable"))); - ql = q.getresult(); + db_access.get_suite_id("unstable"))) + ql = q.getresult() if ql: - nviu_to_remove = []; - print "Newer version in unstable"; - print "-------------------------"; - print ; + nviu_to_remove = [] + print "Newer version in unstable" + print "-------------------------" + print for i in ql: - (source, experimental_version, unstable_version) = i; - print " o %s (%s, %s)" % (source, experimental_version, unstable_version); - nviu_to_remove.append(source); + (source, experimental_version, unstable_version) = i + print " o %s (%s, %s)" % (source, experimental_version, unstable_version) + nviu_to_remove.append(source) print print "Suggested command:" - print " melanie -m \"[rene] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove)); + print " melanie -m \"[rene] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove)) print ################################################################################ def do_nbs(real_nbs): - output = "Not Built from Source\n"; - output += "---------------------\n\n"; + output = "Not Built from Source\n" + output += "---------------------\n\n" - nbs_to_remove = []; - nbs_keys = real_nbs.keys(); - nbs_keys.sort(); + nbs_to_remove = [] + nbs_keys = real_nbs.keys() + nbs_keys.sort() for source in nbs_keys: output += " * %s_%s builds: %s\n" % (source, source_versions.get(source, "??"), - source_binaries.get(source, "(source does not exist)")); + source_binaries.get(source, "(source does not exist)")) output += " but no longer builds:\n" - versions = real_nbs[source].keys(); - versions.sort(apt_pkg.VersionCompare); + versions = real_nbs[source].keys() + versions.sort(apt_pkg.VersionCompare) for version in versions: - packages = real_nbs[source][version].keys(); - packages.sort(); + packages = real_nbs[source][version].keys() + packages.sort() for pkg in packages: - nbs_to_remove.append(pkg); - output += " o %s: %s\n" % (version, ", ".join(packages)); + nbs_to_remove.append(pkg) + output += " o %s: %s\n" % (version, ", ".join(packages)) - output += "\n"; + output += "\n" if nbs_to_remove: - print output; + print output print "Suggested command:" - print " melanie -m \"[rene] NBS\" -b %s" % (" ".join(nbs_to_remove)); + print " melanie -m \"[rene] NBS\" -b %s" % (" ".join(nbs_to_remove)) print ################################################################################ def do_dubious_nbs(dubious_nbs): - print "Dubious NBS"; - print "-----------"; - print ; + print "Dubious NBS" + print "-----------" + print - dubious_nbs_keys = dubious_nbs.keys(); - dubious_nbs_keys.sort(); + dubious_nbs_keys = dubious_nbs.keys() + dubious_nbs_keys.sort() for source in dubious_nbs_keys: print " * %s_%s builds: %s" % (source, source_versions.get(source, "??"), - source_binaries.get(source, "(source does not exist)")); + source_binaries.get(source, "(source does not exist)")) print " won't admit to building:" - versions = dubious_nbs[source].keys(); - versions.sort(apt_pkg.VersionCompare); + versions = dubious_nbs[source].keys() + versions.sort(apt_pkg.VersionCompare) for version in versions: - packages = dubious_nbs[source][version].keys(); - packages.sort(); - print " o %s: %s" % (version, ", ".join(packages)); + packages = dubious_nbs[source][version].keys() + packages.sort() + print " o %s: %s" % (version, ", ".join(packages)) - print ; + print ################################################################################ @@ -207,7 +207,7 @@ def do_obsolete_source(duplicate_bins, bin2source): if not obsolete.has_key(source): if not source_binaries.has_key(source): # Source has already been removed - continue; + continue else: obsolete[source] = map(string.strip, source_binaries[source].split(',')) @@ -236,140 +236,140 @@ def do_obsolete_source(duplicate_bins, bin2source): output += "\n" if to_remove: - print output; + print output print "Suggested command:" - print " melanie -S -p -m \"[rene] obsolete source package\" %s" % (" ".join(to_remove)); + print " melanie -S -p -m \"[rene] obsolete source package\" %s" % (" ".join(to_remove)) print ################################################################################ def main (): - global Cnf, projectB, suite_id, source_binaries, source_versions; + global Cnf, projectB, suite_id, source_binaries, source_versions - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('h',"help","Rene::Options::Help"), ('m',"mode","Rene::Options::Mode", "HasArg"), - ('s',"suite","Rene::Options::Suite","HasArg")]; + ('s',"suite","Rene::Options::Suite","HasArg")] for i in [ "help" ]: if not Cnf.has_key("Rene::Options::%s" % (i)): - Cnf["Rene::Options::%s" % (i)] = ""; - Cnf["Rene::Options::Suite"] = Cnf["Dinstall::DefaultSuite"]; + Cnf["Rene::Options::%s" % (i)] = "" + Cnf["Rene::Options::Suite"] = Cnf["Dinstall::DefaultSuite"] if not Cnf.has_key("Rene::Options::Mode"): - Cnf["Rene::Options::Mode"] = "daily"; + Cnf["Rene::Options::Mode"] = "daily" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Rene::Options") if Options["Help"]: - usage(); + usage() # Set up checks based on mode if Options["Mode"] == "daily": - checks = [ "nbs", "nviu", "obsolete source" ]; + checks = [ "nbs", "nviu", "obsolete source" ] elif Options["Mode"] == "full": - checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ]; + checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ] else: - utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"])); - usage(1); + utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"])) + usage(1) - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) - bin_pkgs = {}; - src_pkgs = {}; + bin_pkgs = {} + src_pkgs = {} bin2source = {} - bins_in_suite = {}; - nbs = {}; - source_versions = {}; + bins_in_suite = {} + nbs = {} + source_versions = {} - anais_output = ""; - duplicate_bins = {}; + anais_output = "" + duplicate_bins = {} suite = Options["Suite"] - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) - bin_not_built = {}; + bin_not_built = {} if "bnb" in checks: # Initalize a large hash table of all binary packages - before = time.time(); - sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)); - q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id)); - ql = q.getresult(); - sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); + before = time.time() + sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)) + q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id)) + ql = q.getresult() + sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) for i in ql: - bins_in_suite[i[0]] = ""; + bins_in_suite[i[0]] = "" # Checks based on the Sources files - components = Cnf.ValueList("Suite::%s::Components" % (suite)); + components = Cnf.ValueList("Suite::%s::Components" % (suite)) for component in components: - filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component); + filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)); + temp_filename = utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): - sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)); - sys.exit(result); - sources = utils.open_file(temp_filename); - Sources = apt_pkg.ParseTagFile(sources); + sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) + sys.exit(result) + sources = utils.open_file(temp_filename) + Sources = apt_pkg.ParseTagFile(sources) while Sources.Step(): - source = Sources.Section.Find('Package'); - source_version = Sources.Section.Find('Version'); - architecture = Sources.Section.Find('Architecture'); - binaries = Sources.Section.Find('Binary'); - binaries_list = map(string.strip, binaries.split(',')); + source = Sources.Section.Find('Package') + source_version = Sources.Section.Find('Version') + architecture = Sources.Section.Find('Architecture') + binaries = Sources.Section.Find('Binary') + binaries_list = map(string.strip, binaries.split(',')) if "bnb" in checks: # Check for binaries not built on any architecture. for binary in binaries_list: if not bins_in_suite.has_key(binary): bin_not_built.setdefault(source, {}) - bin_not_built[source][binary] = ""; + bin_not_built[source][binary] = "" if "anais" in checks: - anais_output += do_anais(architecture, binaries_list, source); + anais_output += do_anais(architecture, binaries_list, source) # Check for duplicated packages and build indices for checking "no source" later - source_index = component + '/' + source; + source_index = component + '/' + source if src_pkgs.has_key(source): - print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]); - src_pkgs[source] = source_index; + print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]) + src_pkgs[source] = source_index for binary in binaries_list: if bin_pkgs.has_key(binary): key_list = [ source, bin_pkgs[binary] ] key_list.sort() key = '~'.join(key_list) duplicate_bins.setdefault(key, []) - duplicate_bins[key].append(binary); - bin_pkgs[binary] = source; - source_binaries[source] = binaries; - source_versions[source] = source_version; + duplicate_bins[key].append(binary) + bin_pkgs[binary] = source + source_binaries[source] = binaries + source_versions[source] = source_version - sources.close(); - os.unlink(temp_filename); + sources.close() + os.unlink(temp_filename) # Checks based on the Packages files for component in components + ['main/debian-installer']: - architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))); + architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))) for architecture in architectures: - filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture); + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture) # apt_pkg.ParseTagFile needs a real file handle - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)); + temp_filename = utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): - sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)); - sys.exit(result); - packages = utils.open_file(temp_filename); - Packages = apt_pkg.ParseTagFile(packages); + sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) + sys.exit(result) + packages = utils.open_file(temp_filename) + Packages = apt_pkg.ParseTagFile(packages) while Packages.Step(): - package = Packages.Section.Find('Package'); - source = Packages.Section.Find('Source', ""); - version = Packages.Section.Find('Version'); + package = Packages.Section.Find('Package') + source = Packages.Section.Find('Source', "") + version = Packages.Section.Find('Version') if source == "": - source = package; + source = package if bin2source.has_key(package) and \ apt_pkg.VersionCompare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version @@ -379,13 +379,13 @@ def main (): bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: - m = utils.re_extract_src_version.match(source); - source = m.group(1); - version = m.group(2); + m = utils.re_extract_src_version.match(source) + source = m.group(1) + version = m.group(2) if not bin_pkgs.has_key(package): nbs.setdefault(source,{}) nbs[source].setdefault(package, {}) - nbs[source][package][version] = ""; + nbs[source][package][version] = "" else: previous_source = bin_pkgs[package] if previous_source != source: @@ -395,31 +395,31 @@ def main (): duplicate_bins.setdefault(key, []) if package not in duplicate_bins[key]: duplicate_bins[key].append(package) - packages.close(); - os.unlink(temp_filename); + packages.close() + os.unlink(temp_filename) if "obsolete source" in checks: do_obsolete_source(duplicate_bins, bin2source) # Distinguish dubious (version numbers match) and 'real' NBS (they don't) - dubious_nbs = {}; - real_nbs = {}; + dubious_nbs = {} + real_nbs = {} for source in nbs.keys(): for package in nbs[source].keys(): - versions = nbs[source][package].keys(); - versions.sort(apt_pkg.VersionCompare); - latest_version = versions.pop(); - source_version = source_versions.get(source,"0"); + versions = nbs[source][package].keys() + versions.sort(apt_pkg.VersionCompare) + latest_version = versions.pop() + source_version = source_versions.get(source,"0") if apt_pkg.VersionCompare(latest_version, source_version) == 0: - add_nbs(dubious_nbs, source, latest_version, package); + add_nbs(dubious_nbs, source, latest_version, package) else: - add_nbs(real_nbs, source, latest_version, package); + add_nbs(real_nbs, source, latest_version, package) if "nviu" in checks: - do_nviu(); + do_nviu() if "nbs" in checks: - do_nbs(real_nbs); + do_nbs(real_nbs) ### @@ -428,36 +428,36 @@ def main (): print if "bnb" in checks: - print "Unbuilt binary packages"; - print "-----------------------"; + print "Unbuilt binary packages" + print "-----------------------" print - keys = bin_not_built.keys(); - keys.sort(); + keys = bin_not_built.keys() + keys.sort() for source in keys: - binaries = bin_not_built[source].keys(); - binaries.sort(); - print " o %s: %s" % (source, ", ".join(binaries)); - print ; + binaries = bin_not_built[source].keys() + binaries.sort() + print " o %s: %s" % (source, ", ".join(binaries)) + print if "bms" in checks: - print "Built from multiple source packages"; - print "-----------------------------------"; - print ; - keys = duplicate_bins.keys(); - keys.sort(); + print "Built from multiple source packages" + print "-----------------------------------" + print + keys = duplicate_bins.keys() + keys.sort() for key in keys: - (source_a, source_b) = key.split("~"); - print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key])); - print ; + (source_a, source_b) = key.split("~") + print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key])) + print if "anais" in checks: - print "Architecture Not Allowed In Source"; - print "----------------------------------"; - print anais_output; - print ; + print "Architecture Not Allowed In Source" + print "----------------------------------" + print anais_output + print if "dubious nbs" in checks: - do_dubious_nbs(dubious_nbs); + do_dubious_nbs(dubious_nbs) ################################################################################ diff --git a/dak/decode_dot_dak.py b/dak/decode_dot_dak.py index e832851f..51620ca9 100755 --- a/dak/decode_dot_dak.py +++ b/dak/decode_dot_dak.py @@ -27,9 +27,9 @@ ################################################################################ -import sys; -import katie, utils; -import apt_pkg; +import sys +import katie, utils +import apt_pkg ################################################################################ @@ -45,88 +45,88 @@ Dumps the info in .katie FILE(s). def main(): Cnf = utils.get_conf() - Arguments = [('h',"help","Ashley::Options::Help")]; + Arguments = [('h',"help","Ashley::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Ashley::Options::%s" % (i)): - Cnf["Ashley::Options::%s" % (i)] = ""; + Cnf["Ashley::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Ashley::Options") if Options["Help"]: - usage(); + usage() - k = katie.Katie(Cnf); + k = katie.Katie(Cnf) for arg in sys.argv[1:]: - arg = utils.validate_changes_file_arg(arg,require_changes=-1); - k.pkg.changes_file = arg; - print "%s:" % (arg); - k.init_vars(); - k.update_vars(); - - changes = k.pkg.changes; - print " Changes:"; + arg = utils.validate_changes_file_arg(arg,require_changes=-1) + k.pkg.changes_file = arg + print "%s:" % (arg) + k.init_vars() + k.update_vars() + + changes = k.pkg.changes + print " Changes:" # Mandatory changes fields for i in [ "source", "version", "maintainer", "urgency", "changedby822", "changedby2047", "changedbyname", "maintainer822", "maintainer2047", "maintainername", "maintaineremail", "fingerprint", "changes" ]: - print " %s: %s" % (i.capitalize(), changes[i]); - del changes[i]; + print " %s: %s" % (i.capitalize(), changes[i]) + del changes[i] # Mandatory changes lists for i in [ "distribution", "architecture", "closes" ]: - print " %s: %s" % (i.capitalize(), " ".join(changes[i].keys())); - del changes[i]; + print " %s: %s" % (i.capitalize(), " ".join(changes[i].keys())) + del changes[i] # Optional changes fields for i in [ "changed-by", "filecontents", "format" ]: if changes.has_key(i): - print " %s: %s" % (i.capitalize(), changes[i]); - del changes[i]; - print; + print " %s: %s" % (i.capitalize(), changes[i]) + del changes[i] + print if changes: - utils.warn("changes still has following unrecognised keys: %s" % (changes.keys())); + utils.warn("changes still has following unrecognised keys: %s" % (changes.keys())) - dsc = k.pkg.dsc; - print " Dsc:"; + dsc = k.pkg.dsc + print " Dsc:" for i in [ "source", "version", "maintainer", "fingerprint", "uploaders", "bts changelog" ]: if dsc.has_key(i): - print " %s: %s" % (i.capitalize(), dsc[i]); - del dsc[i]; - print; + print " %s: %s" % (i.capitalize(), dsc[i]) + del dsc[i] + print if dsc: - utils.warn("dsc still has following unrecognised keys: %s" % (dsc.keys())); + utils.warn("dsc still has following unrecognised keys: %s" % (dsc.keys())) - files = k.pkg.files; + files = k.pkg.files print " Files:" for file in files.keys(): - print " %s:" % (file); + print " %s:" % (file) for i in [ "package", "version", "architecture", "type", "size", "md5sum", "component", "location id", "source package", "source version", "maintainer", "dbtype", "files id", "new", "section", "priority", "pool name" ]: if files[file].has_key(i): - print " %s: %s" % (i.capitalize(), files[file][i]); - del files[file][i]; + print " %s: %s" % (i.capitalize(), files[file][i]) + del files[file][i] if files[file]: - utils.warn("files[%s] still has following unrecognised keys: %s" % (file, files[file].keys())); - print; + utils.warn("files[%s] still has following unrecognised keys: %s" % (file, files[file].keys())) + print - dsc_files = k.pkg.dsc_files; - print " Dsc Files:"; + dsc_files = k.pkg.dsc_files + print " Dsc Files:" for file in dsc_files.keys(): - print " %s:" % (file); + print " %s:" % (file) # Mandatory fields for i in [ "size", "md5sum" ]: - print " %s: %s" % (i.capitalize(), dsc_files[file][i]); - del dsc_files[file][i]; + print " %s: %s" % (i.capitalize(), dsc_files[file][i]) + del dsc_files[file][i] # Optional fields for i in [ "files id" ]: if dsc_files[file].has_key(i): - print " %s: %s" % (i.capitalize(), dsc_files[file][i]); - del dsc_files[file][i]; + print " %s: %s" % (i.capitalize(), dsc_files[file][i]) + del dsc_files[file][i] if dsc_files[file]: - utils.warn("dsc_files[%s] still has following unrecognised keys: %s" % (file, dsc_files[file].keys())); + utils.warn("dsc_files[%s] still has following unrecognised keys: %s" % (file, dsc_files[file].keys())) ################################################################################ diff --git a/dak/examine_package.py b/dak/examine_package.py index be6d767a..1f4633a9 100755 --- a/dak/examine_package.py +++ b/dak/examine_package.py @@ -40,8 +40,8 @@ import pg, db_access ################################################################################ -re_package = re.compile(r"^(.+?)_.*"); -re_doc_directory = re.compile(r".*/doc/([^/]*).*"); +re_package = re.compile(r"^(.+?)_.*") +re_doc_directory = re.compile(r".*/doc/([^/]*).*") re_contrib = re.compile('^contrib/') re_nonfree = re.compile('^non\-free/') @@ -61,28 +61,28 @@ re_spacestrip = re.compile('(\s)') # Colour definitions # Main -main_colour = "\033[36m"; +main_colour = "\033[36m" # Contrib -contrib_colour = "\033[33m"; +contrib_colour = "\033[33m" # Non-Free -nonfree_colour = "\033[31m"; +nonfree_colour = "\033[31m" # Arch -arch_colour = "\033[32m"; +arch_colour = "\033[32m" # End -end_colour = "\033[0m"; +end_colour = "\033[0m" # Bold -bold_colour = "\033[1m"; +bold_colour = "\033[1m" # Bad maintainer -maintainer_colour = arch_colour; +maintainer_colour = arch_colour ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None Cnf = utils.get_conf() projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) -db_access.init(Cnf, projectB); +db_access.init(Cnf, projectB) ################################################################################ @@ -107,67 +107,67 @@ def get_depends_parts(depend) : return d_parts def get_or_list(depend) : - or_list = depend.split("|"); + or_list = depend.split("|") return or_list def get_comma_list(depend) : - dep_list = depend.split(","); + dep_list = depend.split(",") return dep_list def split_depends (d_str) : # creates a list of lists of dictionaries of depends (package,version relation) - d_str = re_spacestrip.sub('',d_str); - depends_tree = []; + d_str = re_spacestrip.sub('',d_str) + depends_tree = [] # first split depends string up amongs comma delimiter - dep_list = get_comma_list(d_str); - d = 0; + dep_list = get_comma_list(d_str) + d = 0 while d < len(dep_list): # put depends into their own list - depends_tree.append([dep_list[d]]); - d += 1; - d = 0; + depends_tree.append([dep_list[d]]) + d += 1 + d = 0 while d < len(depends_tree): - k = 0; + k = 0 # split up Or'd depends into a multi-item list - depends_tree[d] = get_or_list(depends_tree[d][0]); + depends_tree[d] = get_or_list(depends_tree[d][0]) while k < len(depends_tree[d]): # split depends into {package, version relation} - depends_tree[d][k] = get_depends_parts(depends_tree[d][k]); - k += 1; - d += 1; - return depends_tree; + depends_tree[d][k] = get_depends_parts(depends_tree[d][k]) + k += 1 + d += 1 + return depends_tree def read_control (filename): - recommends = []; - depends = []; - section = ''; - maintainer = ''; - arch = ''; + recommends = [] + depends = [] + section = '' + maintainer = '' + arch = '' - deb_file = utils.open_file(filename); + deb_file = utils.open_file(filename) try: - extracts = apt_inst.debExtractControl(deb_file); - control = apt_pkg.ParseSection(extracts); + extracts = apt_inst.debExtractControl(deb_file) + control = apt_pkg.ParseSection(extracts) except: - print "can't parse control info"; - control = ''; + print "can't parse control info" + control = '' - deb_file.close(); + deb_file.close() - control_keys = control.keys(); + control_keys = control.keys() if control.has_key("Depends"): - depends_str = control.Find("Depends"); + depends_str = control.Find("Depends") # create list of dependancy lists - depends = split_depends(depends_str); + depends = split_depends(depends_str) if control.has_key("Recommends"): - recommends_str = control.Find("Recommends"); - recommends = split_depends(recommends_str); + recommends_str = control.Find("Recommends") + recommends = split_depends(recommends_str) if control.has_key("Section"): - section_str = control.Find("Section"); + section_str = control.Find("Section") c_match = re_contrib.search(section_str) nf_match = re_nonfree.search(section_str) @@ -189,78 +189,78 @@ def read_control (filename): localhost = re_localhost.search(maintainer) if localhost: #highlight bad email - maintainer = maintainer_colour + maintainer + end_colour; + maintainer = maintainer_colour + maintainer + end_colour return (control, control_keys, section, depends, recommends, arch, maintainer) def read_dsc (dsc_filename): - dsc = {}; + dsc = {} - dsc_file = utils.open_file(dsc_filename); + dsc_file = utils.open_file(dsc_filename) try: - dsc = utils.parse_changes(dsc_filename); + dsc = utils.parse_changes(dsc_filename) except: print "can't parse control info" - dsc_file.close(); + dsc_file.close() - filecontents = strip_pgp_signature(dsc_filename); + filecontents = strip_pgp_signature(dsc_filename) if dsc.has_key("build-depends"): - builddep = split_depends(dsc["build-depends"]); - builddepstr = create_depends_string(builddep); - filecontents = re_builddep.sub("Build-Depends: "+builddepstr, filecontents); + builddep = split_depends(dsc["build-depends"]) + builddepstr = create_depends_string(builddep) + filecontents = re_builddep.sub("Build-Depends: "+builddepstr, filecontents) if dsc.has_key("build-depends-indep"): - builddepindstr = create_depends_string(split_depends(dsc["build-depends-indep"])); - filecontents = re_builddepind.sub("Build-Depends-Indep: "+builddepindstr, filecontents); + builddepindstr = create_depends_string(split_depends(dsc["build-depends-indep"])) + filecontents = re_builddepind.sub("Build-Depends-Indep: "+builddepindstr, filecontents) if dsc.has_key("architecture") : if (dsc["architecture"] != "any"): - newarch = arch_colour + dsc["architecture"] + end_colour; - filecontents = re_arch.sub("Architecture: " + newarch, filecontents); + newarch = arch_colour + dsc["architecture"] + end_colour + filecontents = re_arch.sub("Architecture: " + newarch, filecontents) - return filecontents; + return filecontents def create_depends_string (depends_tree): # just look up unstable for now. possibly pull from .changes later - suite = "unstable"; - result = ""; - comma_count = 1; + suite = "unstable" + result = "" + comma_count = 1 for l in depends_tree: if (comma_count >= 2): - result += ", "; + result += ", " or_count = 1 for d in l: if (or_count >= 2 ): result += " | " # doesn't do version lookup yet. - q = projectB.query("SELECT DISTINCT(b.package), b.version, c.name, su.suite_name FROM binaries b, files fi, location l, component c, bin_associations ba, suite su WHERE b.package='%s' AND b.file = fi.id AND fi.location = l.id AND l.component = c.id AND ba.bin=b.id AND ba.suite = su.id AND su.suite_name='%s' ORDER BY b.version desc" % (d['name'], suite)); - ql = q.getresult(); + q = projectB.query("SELECT DISTINCT(b.package), b.version, c.name, su.suite_name FROM binaries b, files fi, location l, component c, bin_associations ba, suite su WHERE b.package='%s' AND b.file = fi.id AND fi.location = l.id AND l.component = c.id AND ba.bin=b.id AND ba.suite = su.id AND su.suite_name='%s' ORDER BY b.version desc" % (d['name'], suite)) + ql = q.getresult() if ql: - i = ql[0]; + i = ql[0] if i[2] == "contrib": - result += contrib_colour + d['name']; + result += contrib_colour + d['name'] elif i[2] == "non-free": - result += nonfree_colour + d['name']; + result += nonfree_colour + d['name'] else : - result += main_colour + d['name']; + result += main_colour + d['name'] if d['version'] != '' : - result += " (%s)" % (d['version']); - result += end_colour; + result += " (%s)" % (d['version']) + result += end_colour else: - result += bold_colour + d['name']; + result += bold_colour + d['name'] if d['version'] != '' : - result += " (%s)" % (d['version']); - result += end_colour; - or_count += 1; - comma_count += 1; - return result; + result += " (%s)" % (d['version']) + result += end_colour + or_count += 1 + comma_count += 1 + return result def output_deb_info(filename): - (control, control_keys, section, depends, recommends, arch, maintainer) = read_control(filename); + (control, control_keys, section, depends, recommends, arch, maintainer) = read_control(filename) if control == '': print "no control info" @@ -268,174 +268,174 @@ def output_deb_info(filename): for key in control_keys : output = " " + key + ": " if key == 'Depends': - output += create_depends_string(depends); + output += create_depends_string(depends) elif key == 'Recommends': - output += create_depends_string(recommends); + output += create_depends_string(recommends) elif key == 'Section': - output += section; + output += section elif key == 'Architecture': - output += arch; + output += arch elif key == 'Maintainer': - output += maintainer; + output += maintainer elif key == 'Description': - desc = control.Find(key); - desc = re_newlinespace.sub('\n ', desc); - output += desc; + desc = control.Find(key) + desc = re_newlinespace.sub('\n ', desc) + output += desc else: - output += control.Find(key); - print output; + output += control.Find(key) + print output def do_command (command, filename): - o = os.popen("%s %s" % (command, filename)); - print o.read(); + o = os.popen("%s %s" % (command, filename)) + print o.read() def print_copyright (deb_filename): - package = re_package.sub(r'\1', deb_filename); - o = os.popen("ar p %s data.tar.gz | tar tzvf - | egrep 'usr(/share)?/doc/[^/]*/copyright' | awk '{ print $6 }' | head -n 1" % (deb_filename)); - copyright = o.read()[:-1]; + package = re_package.sub(r'\1', deb_filename) + o = os.popen("ar p %s data.tar.gz | tar tzvf - | egrep 'usr(/share)?/doc/[^/]*/copyright' | awk '{ print $6 }' | head -n 1" % (deb_filename)) + copyright = o.read()[:-1] if copyright == "": print "WARNING: No copyright found, please check package manually." - return; + return - doc_directory = re_doc_directory.sub(r'\1', copyright); + doc_directory = re_doc_directory.sub(r'\1', copyright) if package != doc_directory: - print "WARNING: wrong doc directory (expected %s, got %s)." % (package, doc_directory); - return; + print "WARNING: wrong doc directory (expected %s, got %s)." % (package, doc_directory) + return - o = os.popen("ar p %s data.tar.gz | tar xzOf - %s" % (deb_filename, copyright)); - print o.read(); + o = os.popen("ar p %s data.tar.gz | tar xzOf - %s" % (deb_filename, copyright)) + print o.read() def check_dsc (dsc_filename): - print "---- .dsc file for %s ----" % (dsc_filename); + print "---- .dsc file for %s ----" % (dsc_filename) (dsc) = read_dsc(dsc_filename) print dsc def check_deb (deb_filename): - filename = os.path.basename(deb_filename); + filename = os.path.basename(deb_filename) if filename.endswith(".udeb"): - is_a_udeb = 1; + is_a_udeb = 1 else: - is_a_udeb = 0; + is_a_udeb = 0 - print "---- control file for %s ----" % (filename); - #do_command ("dpkg -I", deb_filename); + print "---- control file for %s ----" % (filename) + #do_command ("dpkg -I", deb_filename) output_deb_info(deb_filename) if is_a_udeb: - print "---- skipping lintian check for µdeb ----"; - print ; + print "---- skipping lintian check for µdeb ----" + print else: - print "---- lintian check for %s ----" % (filename); - do_command ("lintian", deb_filename); - print "---- linda check for %s ----" % (filename); - do_command ("linda", deb_filename); + print "---- lintian check for %s ----" % (filename) + do_command ("lintian", deb_filename) + print "---- linda check for %s ----" % (filename) + do_command ("linda", deb_filename) - print "---- contents of %s ----" % (filename); - do_command ("dpkg -c", deb_filename); + print "---- contents of %s ----" % (filename) + do_command ("dpkg -c", deb_filename) if is_a_udeb: - print "---- skipping copyright for µdeb ----"; + print "---- skipping copyright for µdeb ----" else: - print "---- copyright of %s ----" % (filename); - print_copyright(deb_filename); + print "---- copyright of %s ----" % (filename) + print_copyright(deb_filename) - print "---- file listing of %s ----" % (filename); - do_command ("ls -l", deb_filename); + print "---- file listing of %s ----" % (filename) + do_command ("ls -l", deb_filename) # Read a file, strip the signature and return the modified contents as # a string. def strip_pgp_signature (filename): - file = utils.open_file (filename); - contents = ""; - inside_signature = 0; - skip_next = 0; + file = utils.open_file (filename) + contents = "" + inside_signature = 0 + skip_next = 0 for line in file.readlines(): if line[:-1] == "": - continue; + continue if inside_signature: - continue; + continue if skip_next: - skip_next = 0; - continue; + skip_next = 0 + continue if line.startswith("-----BEGIN PGP SIGNED MESSAGE"): - skip_next = 1; - continue; + skip_next = 1 + continue if line.startswith("-----BEGIN PGP SIGNATURE"): - inside_signature = 1; - continue; + inside_signature = 1 + continue if line.startswith("-----END PGP SIGNATURE"): - inside_signature = 0; - continue; - contents += line; - file.close(); - return contents; + inside_signature = 0 + continue + contents += line + file.close() + return contents # Display the .changes [without the signature] def display_changes (changes_filename): - print "---- .changes file for %s ----" % (changes_filename); - print strip_pgp_signature(changes_filename); + print "---- .changes file for %s ----" % (changes_filename) + print strip_pgp_signature(changes_filename) def check_changes (changes_filename): - display_changes(changes_filename); + display_changes(changes_filename) - changes = utils.parse_changes (changes_filename); - files = utils.build_file_list(changes); + changes = utils.parse_changes (changes_filename) + files = utils.build_file_list(changes) for file in files.keys(): if file.endswith(".deb") or file.endswith(".udeb"): - check_deb(file); + check_deb(file) if file.endswith(".dsc"): - check_dsc(file); + check_dsc(file) # else: => byhand def main (): - global Cnf, projectB, db_files, waste, excluded; + global Cnf, projectB, db_files, waste, excluded # Cnf = utils.get_conf() - Arguments = [('h',"help","Fernanda::Options::Help")]; + Arguments = [('h',"help","Fernanda::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Frenanda::Options::%s" % (i)): - Cnf["Fernanda::Options::%s" % (i)] = ""; + Cnf["Fernanda::Options::%s" % (i)] = "" - args = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + args = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Fernanda::Options") if Options["Help"]: - usage(); + usage() - stdout_fd = sys.stdout; + stdout_fd = sys.stdout for file in args: try: # Pipe output for each argument through less - less_fd = os.popen("less -R -", 'w', 0); + less_fd = os.popen("less -R -", 'w', 0) # -R added to display raw control chars for colour - sys.stdout = less_fd; + sys.stdout = less_fd try: if file.endswith(".changes"): - check_changes(file); + check_changes(file) elif file.endswith(".deb") or file.endswith(".udeb"): - check_deb(file); + check_deb(file) elif file.endswith(".dsc"): - check_dsc(file); + check_dsc(file) else: - utils.fubar("Unrecognised file type: '%s'." % (file)); + utils.fubar("Unrecognised file type: '%s'." % (file)) finally: # Reset stdout here so future less invocations aren't FUBAR - less_fd.close(); - sys.stdout = stdout_fd; + less_fd.close() + sys.stdout = stdout_fd except IOError, e: if errno.errorcode[e.errno] == 'EPIPE': - utils.warn("[fernanda] Caught EPIPE; skipping."); - pass; + utils.warn("[fernanda] Caught EPIPE; skipping.") + pass else: - raise; + raise except KeyboardInterrupt: - utils.warn("[fernanda] Caught C-c; skipping."); - pass; + utils.warn("[fernanda] Caught C-c; skipping.") + pass ####################################################################################### diff --git a/dak/find_null_maintainers.py b/dak/find_null_maintainers.py index d133064a..82eed196 100755 --- a/dak/find_null_maintainers.py +++ b/dak/find_null_maintainers.py @@ -20,14 +20,14 @@ ################################################################################ -import ldap, pg, sys, time; -import apt_pkg; -import utils; +import ldap, pg, sys, time +import apt_pkg +import utils ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ @@ -41,54 +41,54 @@ Checks for users with no packages in the archive ################################################################################ def get_ldap_value(entry, value): - ret = entry.get(value); + ret = entry.get(value) if not ret: - return ""; + return "" else: # FIXME: what about > 0 ? - return ret[0]; + return ret[0] def main(): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() - Arguments = [('h',"help","Rosamund::Options::Help")]; + Arguments = [('h',"help","Rosamund::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Rosamund::Options::%s" % (i)): - Cnf["Rosamund::Options::%s" % (i)] = ""; + Cnf["Rosamund::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Rosamund::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - before = time.time(); - sys.stderr.write("[Getting info from the LDAP server..."); - LDAPDn = Cnf["Emilie::LDAPDn"]; - LDAPServer = Cnf["Emilie::LDAPServer"]; - l = ldap.open(LDAPServer); - l.simple_bind_s("",""); + before = time.time() + sys.stderr.write("[Getting info from the LDAP server...") + LDAPDn = Cnf["Emilie::LDAPDn"] + LDAPServer = Cnf["Emilie::LDAPServer"] + l = ldap.open(LDAPServer) + l.simple_bind_s("","") Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Julia::ValidGID"]), - ["uid", "cn", "mn", "sn", "createtimestamp"]); - sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); + ["uid", "cn", "mn", "sn", "createtimestamp"]) + sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) - db_uid = {}; - db_unstable_uid = {}; + db_uid = {} + db_unstable_uid = {} - before = time.time(); - sys.stderr.write("[Getting UID info for entire archive..."); - q = projectB.query("SELECT DISTINCT u.uid FROM uid u, fingerprint f WHERE f.uid = u.id;"); - sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); + before = time.time() + sys.stderr.write("[Getting UID info for entire archive...") + q = projectB.query("SELECT DISTINCT u.uid FROM uid u, fingerprint f WHERE f.uid = u.id;") + sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) for i in q.getresult(): - db_uid[i[0]] = ""; + db_uid[i[0]] = "" - before = time.time(); - sys.stderr.write("[Getting UID info for unstable..."); + before = time.time() + sys.stderr.write("[Getting UID info for unstable...") q = projectB.query(""" SELECT DISTINCT u.uid FROM suite su, src_associations sa, source s, fingerprint f, uid u WHERE f.uid = u.id AND sa.source = s.id AND sa.suite = su.id @@ -96,39 +96,39 @@ SELECT DISTINCT u.uid FROM suite su, src_associations sa, source s, fingerprint UNION SELECT DISTINCT u.uid FROM suite su, bin_associations ba, binaries b, fingerprint f, uid u WHERE f.uid = u.id AND ba.bin = b.id AND ba.suite = su.id - AND su.suite_name = 'unstable' AND b.sig_fpr = f.id"""); - sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); + AND su.suite_name = 'unstable' AND b.sig_fpr = f.id""") + sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) for i in q.getresult(): - db_unstable_uid[i[0]] = ""; + db_unstable_uid[i[0]] = "" - now = time.time(); + now = time.time() for i in Attrs: - entry = i[1]; - uid = entry["uid"][0]; - created = time.mktime(time.strptime(entry["createtimestamp"][0][:8], '%Y%m%d')); - diff = now - created; + entry = i[1] + uid = entry["uid"][0] + created = time.mktime(time.strptime(entry["createtimestamp"][0][:8], '%Y%m%d')) + diff = now - created # 31536000 is 1 year in seconds, i.e. 60 * 60 * 24 * 365 if diff < 31536000 / 2: - when = "Less than 6 months ago"; + when = "Less than 6 months ago" elif diff < 31536000: - when = "Less than 1 year ago"; + when = "Less than 1 year ago" elif diff < 31536000 * 1.5: - when = "Less than 18 months ago"; + when = "Less than 18 months ago" elif diff < 31536000 * 2: - when = "Less than 2 years ago"; + when = "Less than 2 years ago" elif diff < 31536000 * 3: - when = "Less than 3 years ago"; + when = "Less than 3 years ago" else: - when = "More than 3 years ago"; + when = "More than 3 years ago" name = " ".join([get_ldap_value(entry, "cn"), get_ldap_value(entry, "mn"), - get_ldap_value(entry, "sn")]); + get_ldap_value(entry, "sn")]) if not db_uid.has_key(uid): - print "NONE %s (%s) %s" % (uid, name, when); + print "NONE %s (%s) %s" % (uid, name, when) else: if not db_unstable_uid.has_key(uid): - print "NOT_UNSTABLE %s (%s) %s" % (uid, name, when); + print "NOT_UNSTABLE %s (%s) %s" % (uid, name, when) ############################################################ diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index db856778..ee602e05 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -36,10 +36,10 @@ import utils ################################################################################ -projectB = None; -Cnf = None; -Logger = None; -Options = None; +projectB = None +Cnf = None +Logger = None +Options = None ################################################################################ @@ -52,7 +52,7 @@ Write out ed-style diffs to Packages/Source lists -p name for the patch (defaults to current time) -n take no action """ - sys.exit(exit_code); + sys.exit(exit_code) def tryunlink(file): @@ -286,9 +286,9 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 14): def main(): global Cnf, Options, Logger - os.umask(0002); + os.umask(0002) - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [ ('h', "help", "Tiffani::Options::Help"), ('c', None, "Tiffani::Options::CanonicalPath", "hasArg"), ('p', "patchname", "Tiffani::Options::PatchName", "hasArg"), @@ -296,10 +296,10 @@ def main(): ('d', "tmpdir", "Tiffani::Options::TempDir", "hasArg"), ('m', "maxdiffs", "Tiffani::Options::MaxDiffs", "hasArg"), ('n', "n-act", "Tiffani::Options::NoAct"), - ]; - suites = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Options = Cnf.SubTree("Tiffani::Options"); - if Options.has_key("Help"): usage(); + ] + suites = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + Options = Cnf.SubTree("Tiffani::Options") + if Options.has_key("Help"): usage() maxdiffs = Options.get("MaxDiffs::Default", "14") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) @@ -340,11 +340,11 @@ def main(): else: components = [] - suite_suffix = Cnf.Find("Dinstall::SuiteSuffix"); + suite_suffix = Cnf.Find("Dinstall::SuiteSuffix") if components and suite_suffix: - longsuite = suite + "/" + suite_suffix; + longsuite = suite + "/" + suite_suffix else: - longsuite = suite; + longsuite = suite tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite)) @@ -353,8 +353,8 @@ def main(): elif AptCnf.has_key("bindirectory::%s" % (tree)): sections = AptCnf["bindirectory::%s::Sections" % (tree)].split() else: - aptcnf_filename = os.path.basename(utils.which_apt_conf_file()); - print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename); + aptcnf_filename = os.path.basename(utils.which_apt_conf_file()) + print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename) continue for architecture in architectures: diff --git a/dak/generate_releases.py b/dak/generate_releases.py index 49a19fb7..4dba71f0 100755 --- a/dak/generate_releases.py +++ b/dak/generate_releases.py @@ -123,20 +123,20 @@ def print_sha1_files (tree, files): def main (): global Cnf, AptCnf, projectB, out - out = sys.stdout; + out = sys.stdout Cnf = utils.get_conf() - Arguments = [('h',"help","Ziyi::Options::Help")]; + Arguments = [('h',"help","Ziyi::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Ziyi::Options::%s" % (i)): - Cnf["Ziyi::Options::%s" % (i)] = ""; + Cnf["Ziyi::Options::%s" % (i)] = "" suites = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Ziyi::Options") if Options["Help"]: - usage(); + usage() AptCnf = apt_pkg.newConfiguration() apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file()) @@ -169,11 +169,11 @@ def main (): else: components = [] - suite_suffix = Cnf.Find("Dinstall::SuiteSuffix"); + suite_suffix = Cnf.Find("Dinstall::SuiteSuffix") if components and suite_suffix: - longsuite = suite + "/" + suite_suffix; + longsuite = suite + "/" + suite_suffix else: - longsuite = suite; + longsuite = suite tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite)) @@ -182,8 +182,8 @@ def main (): elif AptCnf.has_key("bindirectory::%s" % (tree)): pass else: - aptcnf_filename = os.path.basename(utils.which_apt_conf_file()); - print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename); + aptcnf_filename = os.path.basename(utils.which_apt_conf_file()) + print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename) continue print Cnf["Dir::Root"] + tree + "/Release" @@ -238,15 +238,15 @@ def main (): release = open(relpath, "w") #release = open(longsuite.replace("/","_") + "_" + arch + "_" + sec + "_Release", "w") except IOError: - utils.fubar("Couldn't write to " + relpath); + utils.fubar("Couldn't write to " + relpath) release.write("Archive: %s\n" % (suite)) if version != "": release.write("Version: %s\n" % (version)) if suite_suffix: - release.write("Component: %s/%s\n" % (suite_suffix,sec)); + release.write("Component: %s/%s\n" % (suite_suffix,sec)) else: - release.write("Component: %s\n" % (sec)); + release.write("Component: %s\n" % (sec)) release.write("Origin: %s\n" % (origin)) release.write("Label: %s\n" % (label)) if notautomatic != "": diff --git a/dak/import_archive.py b/dak/import_archive.py index a23abb70..86bcbec5 100755 --- a/dak/import_archive.py +++ b/dak/import_archive.py @@ -37,9 +37,9 @@ ############################################################################### -import commands, os, pg, re, sys, time; -import apt_pkg; -import db_access, utils; +import commands, os, pg, re, sys, time +import apt_pkg +import db_access, utils ############################################################################### @@ -47,31 +47,31 @@ re_arch_from_filename = re.compile(r"binary-[^/]+") ############################################################################### -Cnf = None; -projectB = None; -files_id_cache = {}; -source_cache = {}; -arch_all_cache = {}; -binary_cache = {}; -location_path_cache = {}; +Cnf = None +projectB = None +files_id_cache = {} +source_cache = {} +arch_all_cache = {} +binary_cache = {} +location_path_cache = {} # -files_id_serial = 0; -source_id_serial = 0; -src_associations_id_serial = 0; -dsc_files_id_serial = 0; -files_query_cache = None; -source_query_cache = None; -src_associations_query_cache = None; -dsc_files_query_cache = None; -orig_tar_gz_cache = {}; +files_id_serial = 0 +source_id_serial = 0 +src_associations_id_serial = 0 +dsc_files_id_serial = 0 +files_query_cache = None +source_query_cache = None +src_associations_query_cache = None +dsc_files_query_cache = None +orig_tar_gz_cache = {} # -binaries_id_serial = 0; -binaries_query_cache = None; -bin_associations_id_serial = 0; -bin_associations_query_cache = None; +binaries_id_serial = 0 +binaries_query_cache = None +bin_associations_id_serial = 0 +bin_associations_query_cache = None # -source_cache_for_binaries = {}; -reject_message = ""; +source_cache_for_binaries = {} +reject_message = "" ################################################################################ @@ -86,124 +86,124 @@ Initializes a projectB database from an existing archive ############################################################################### def reject (str, prefix="Rejected: "): - global reject_message; + global reject_message if str: - reject_message += prefix + str + "\n"; + reject_message += prefix + str + "\n" ############################################################################### def check_signature (filename): if not utils.re_taint_free.match(os.path.basename(filename)): - reject("!!WARNING!! tainted filename: '%s'." % (filename)); - return None; + reject("!!WARNING!! tainted filename: '%s'." % (filename)) + return None - status_read, status_write = os.pipe(); + status_read, status_write = os.pipe() cmd = "gpgv --status-fd %s --keyring %s --keyring %s %s" \ - % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename); - (output, status, exit_status) = utils.gpgv_get_status_output(cmd, status_read, status_write); + % (status_write, Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename) + (output, status, exit_status) = utils.gpgv_get_status_output(cmd, status_read, status_write) # Process the status-fd output - keywords = {}; - bad = internal_error = ""; + keywords = {} + bad = internal_error = "" for line in status.split('\n'): - line = line.strip(); + line = line.strip() if line == "": - continue; - split = line.split(); + continue + split = line.split() if len(split) < 2: - internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line); - continue; - (gnupg, keyword) = split[:2]; + internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line) + continue + (gnupg, keyword) = split[:2] if gnupg != "[GNUPG:]": - internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg); - continue; - args = split[2:]; + internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg) + continue + args = split[2:] if keywords.has_key(keyword) and keyword != "NODATA" and keyword != "SIGEXPIRED": - internal_error += "found duplicate status token ('%s').\n" % (keyword); - continue; + internal_error += "found duplicate status token ('%s').\n" % (keyword) + continue else: - keywords[keyword] = args; + keywords[keyword] = args # If we failed to parse the status-fd output, let's just whine and bail now if internal_error: - reject("internal error while performing signature check on %s." % (filename)); - reject(internal_error, ""); - reject("Please report the above errors to the Archive maintainers by replying to this mail.", ""); - return None; + reject("internal error while performing signature check on %s." % (filename)) + reject(internal_error, "") + reject("Please report the above errors to the Archive maintainers by replying to this mail.", "") + return None # Now check for obviously bad things in the processed output if keywords.has_key("SIGEXPIRED"): - utils.warn("%s: signing key has expired." % (filename)); + utils.warn("%s: signing key has expired." % (filename)) if keywords.has_key("KEYREVOKED"): - reject("key used to sign %s has been revoked." % (filename)); - bad = 1; + reject("key used to sign %s has been revoked." % (filename)) + bad = 1 if keywords.has_key("BADSIG"): - reject("bad signature on %s." % (filename)); - bad = 1; + reject("bad signature on %s." % (filename)) + bad = 1 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"): - reject("failed to check signature on %s." % (filename)); - bad = 1; + reject("failed to check signature on %s." % (filename)) + bad = 1 if keywords.has_key("NO_PUBKEY"): - args = keywords["NO_PUBKEY"]; + args = keywords["NO_PUBKEY"] if len(args) < 1: - reject("internal error while checking signature on %s." % (filename)); - bad = 1; + reject("internal error while checking signature on %s." % (filename)) + bad = 1 else: - fingerprint = args[0]; + fingerprint = args[0] if keywords.has_key("BADARMOR"): - reject("ascii armour of signature was corrupt in %s." % (filename)); - bad = 1; + reject("ascii armour of signature was corrupt in %s." % (filename)) + bad = 1 if keywords.has_key("NODATA"): - utils.warn("no signature found for %s." % (filename)); - return "NOSIG"; - #reject("no signature found in %s." % (filename)); - #bad = 1; + utils.warn("no signature found for %s." % (filename)) + return "NOSIG" + #reject("no signature found in %s." % (filename)) + #bad = 1 if bad: - return None; + return None # Next check gpgv exited with a zero return code if exit_status and not keywords.has_key("NO_PUBKEY"): - reject("gpgv failed while checking %s." % (filename)); + reject("gpgv failed while checking %s." % (filename)) if status.strip(): - reject(utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), ""); + reject(utils.prefix_multi_line_string(status, " [GPG status-fd output:] "), "") else: - reject(utils.prefix_multi_line_string(output, " [GPG output:] "), ""); - return None; + reject(utils.prefix_multi_line_string(output, " [GPG output:] "), "") + return None # Sanity check the good stuff we expect if not keywords.has_key("VALIDSIG"): if not keywords.has_key("NO_PUBKEY"): - reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No VALIDSIG]." % (filename)) + bad = 1 else: - args = keywords["VALIDSIG"]; + args = keywords["VALIDSIG"] if len(args) < 1: - reject("internal error while checking signature on %s." % (filename)); - bad = 1; + reject("internal error while checking signature on %s." % (filename)) + bad = 1 else: - fingerprint = args[0]; + fingerprint = args[0] if not keywords.has_key("GOODSIG") and not keywords.has_key("NO_PUBKEY"): - reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No GOODSIG]." % (filename)) + bad = 1 if not keywords.has_key("SIG_ID") and not keywords.has_key("NO_PUBKEY"): - reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No SIG_ID]." % (filename)) + bad = 1 # Finally ensure there's not something we don't recognise known_keywords = utils.Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", - NODATA=""); + NODATA="") for keyword in keywords.keys(): if not known_keywords.has_key(keyword): - reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename)); - bad = 1; + reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], filename)) + bad = 1 if bad: - return None; + return None else: - return fingerprint; + return fingerprint ################################################################################ @@ -211,36 +211,36 @@ def check_signature (filename): def poolify (s, sub): for i in xrange(len(sub)): if sub[i:] == s[0:len(sub)-i]: - return s[len(sub)-i:]; - return s; + return s[len(sub)-i:] + return s def update_archives (): projectB.query("DELETE FROM archive") for archive in Cnf.SubTree("Archive").List(): - SubSec = Cnf.SubTree("Archive::%s" % (archive)); + SubSec = Cnf.SubTree("Archive::%s" % (archive)) projectB.query("INSERT INTO archive (name, origin_server, description) VALUES ('%s', '%s', '%s')" - % (archive, SubSec["OriginServer"], SubSec["Description"])); + % (archive, SubSec["OriginServer"], SubSec["Description"])) def update_components (): projectB.query("DELETE FROM component") for component in Cnf.SubTree("Component").List(): - SubSec = Cnf.SubTree("Component::%s" % (component)); + SubSec = Cnf.SubTree("Component::%s" % (component)) projectB.query("INSERT INTO component (name, description, meets_dfsg) VALUES ('%s', '%s', '%s')" % - (component, SubSec["Description"], SubSec["MeetsDFSG"])); + (component, SubSec["Description"], SubSec["MeetsDFSG"])) def update_locations (): projectB.query("DELETE FROM location") for location in Cnf.SubTree("Location").List(): - SubSec = Cnf.SubTree("Location::%s" % (location)); - archive_id = db_access.get_archive_id(SubSec["archive"]); - type = SubSec.Find("type"); + SubSec = Cnf.SubTree("Location::%s" % (location)) + archive_id = db_access.get_archive_id(SubSec["archive"]) + type = SubSec.Find("type") if type == "legacy-mixed": - projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, SubSec["type"])); + projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, SubSec["type"])) else: for component in Cnf.SubTree("Component").List(): - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" % - (location, component_id, archive_id, SubSec["type"])); + (location, component_id, archive_id, SubSec["type"])) def update_architectures (): projectB.query("DELETE FROM architecture") @@ -251,66 +251,66 @@ def update_suites (): projectB.query("DELETE FROM suite") for suite in Cnf.SubTree("Suite").List(): SubSec = Cnf.SubTree("Suite::%s" %(suite)) - projectB.query("INSERT INTO suite (suite_name) VALUES ('%s')" % suite.lower()); + projectB.query("INSERT INTO suite (suite_name) VALUES ('%s')" % suite.lower()) for i in ("Version", "Origin", "Description"): if SubSec.has_key(i): projectB.query("UPDATE suite SET %s = '%s' WHERE suite_name = '%s'" % (i.lower(), SubSec[i], suite.lower())) for architecture in Cnf.ValueList("Suite::%s::Architectures" % (suite)): - architecture_id = db_access.get_architecture_id (architecture); - projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)); + architecture_id = db_access.get_architecture_id (architecture) + projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)) def update_override_type(): - projectB.query("DELETE FROM override_type"); + projectB.query("DELETE FROM override_type") for type in Cnf.ValueList("OverrideType"): - projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)); + projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)) def update_priority(): - projectB.query("DELETE FROM priority"); + projectB.query("DELETE FROM priority") for priority in Cnf.SubTree("Priority").List(): - projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])); + projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])) def update_section(): - projectB.query("DELETE FROM section"); + projectB.query("DELETE FROM section") for component in Cnf.SubTree("Component").List(): if Cnf["Natalie::ComponentPosition"] == "prefix": - suffix = ""; + suffix = "" if component != 'main': - prefix = component + '/'; + prefix = component + '/' else: - prefix = ""; + prefix = "" else: - prefix = ""; - component = component.replace("non-US/", ""); + prefix = "" + component = component.replace("non-US/", "") if component != 'main': - suffix = '/' + component; + suffix = '/' + component else: - suffix = ""; + suffix = "" for section in Cnf.ValueList("Section"): - projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)); + projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)) def get_location_path(directory): - global location_path_cache; + global location_path_cache if location_path_cache.has_key(directory): - return location_path_cache[directory]; + return location_path_cache[directory] - q = projectB.query("SELECT DISTINCT path FROM location WHERE path ~ '%s'" % (directory)); + q = projectB.query("SELECT DISTINCT path FROM location WHERE path ~ '%s'" % (directory)) try: - path = q.getresult()[0][0]; + path = q.getresult()[0][0] except: - utils.fubar("[neve] get_location_path(): Couldn't get path for %s" % (directory)); - location_path_cache[directory] = path; - return path; + utils.fubar("[neve] get_location_path(): Couldn't get path for %s" % (directory)) + location_path_cache[directory] = path + return path ################################################################################ def get_or_set_files_id (filename, size, md5sum, location_id): - global files_id_cache, files_id_serial, files_query_cache; + global files_id_cache, files_id_serial, files_query_cache - cache_key = "~".join((filename, size, md5sum, repr(location_id))); + cache_key = "~".join((filename, size, md5sum, repr(location_id))) if not files_id_cache.has_key(cache_key): files_id_serial += 1 - files_query_cache.write("%d\t%s\t%s\t%s\t%d\t\\N\n" % (files_id_serial, filename, size, md5sum, location_id)); + files_query_cache.write("%d\t%s\t%s\t%s\t%d\t\\N\n" % (files_id_serial, filename, size, md5sum, location_id)) files_id_cache[cache_key] = files_id_serial return files_id_cache[cache_key] @@ -318,109 +318,109 @@ def get_or_set_files_id (filename, size, md5sum, location_id): ############################################################################### def process_sources (filename, suite, component, archive): - global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message; + global source_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, source_id_serial, src_associations_id_serial, dsc_files_id_serial, source_cache_for_binaries, orig_tar_gz_cache, reject_message - suite = suite.lower(); - suite_id = db_access.get_suite_id(suite); + suite = suite.lower() + suite_id = db_access.get_suite_id(suite) try: - file = utils.open_file (filename); + file = utils.open_file (filename) except utils.cant_open_exc: - utils.warn("can't open '%s'" % (filename)); - return; - Scanner = apt_pkg.ParseTagFile(file); + utils.warn("can't open '%s'" % (filename)) + return + Scanner = apt_pkg.ParseTagFile(file) while Scanner.Step() != 0: - package = Scanner.Section["package"]; - version = Scanner.Section["version"]; - directory = Scanner.Section["directory"]; - dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + package = Scanner.Section["package"] + version = Scanner.Section["version"] + directory = Scanner.Section["directory"] + dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))) # Sometimes the Directory path is a lie; check in the pool if not os.path.exists(dsc_file): if directory.split('/')[0] == "dists": - directory = Cnf["Dir::PoolRoot"] + utils.poolify(package, component); - dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))); + directory = Cnf["Dir::PoolRoot"] + utils.poolify(package, component) + dsc_file = os.path.join(Cnf["Dir::Root"], directory, "%s_%s.dsc" % (package, utils.re_no_epoch.sub('', version))) if not os.path.exists(dsc_file): - utils.fubar("%s not found." % (dsc_file)); - install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))); - fingerprint = check_signature(dsc_file); - fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); + utils.fubar("%s not found." % (dsc_file)) + install_date = time.strftime("%Y-%m-%d", time.localtime(os.path.getmtime(dsc_file))) + fingerprint = check_signature(dsc_file) + fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint) if reject_message: - utils.fubar("%s: %s" % (dsc_file, reject_message)); + utils.fubar("%s: %s" % (dsc_file, reject_message)) maintainer = Scanner.Section["maintainer"] - maintainer = maintainer.replace("'", "\\'"); - maintainer_id = db_access.get_or_set_maintainer_id(maintainer); - location = get_location_path(directory.split('/')[0]); - location_id = db_access.get_location_id (location, component, archive); + maintainer = maintainer.replace("'", "\\'") + maintainer_id = db_access.get_or_set_maintainer_id(maintainer) + location = get_location_path(directory.split('/')[0]) + location_id = db_access.get_location_id (location, component, archive) if not directory.endswith("/"): - directory += '/'; - directory = poolify (directory, location); + directory += '/' + directory = poolify (directory, location) if directory != "" and not directory.endswith("/"): - directory += '/'; - no_epoch_version = utils.re_no_epoch.sub('', version); + directory += '/' + no_epoch_version = utils.re_no_epoch.sub('', version) # Add all files referenced by the .dsc to the files table - ids = []; + ids = [] for line in Scanner.Section["files"].split('\n'): - id = None; - (md5sum, size, filename) = line.strip().split(); + id = None + (md5sum, size, filename) = line.strip().split() # Don't duplicate .orig.tar.gz's if filename.endswith(".orig.tar.gz"): - cache_key = "%s~%s~%s" % (filename, size, md5sum); + cache_key = "%s~%s~%s" % (filename, size, md5sum) if orig_tar_gz_cache.has_key(cache_key): - id = orig_tar_gz_cache[cache_key]; + id = orig_tar_gz_cache[cache_key] else: - id = get_or_set_files_id (directory + filename, size, md5sum, location_id); - orig_tar_gz_cache[cache_key] = id; + id = get_or_set_files_id (directory + filename, size, md5sum, location_id) + orig_tar_gz_cache[cache_key] = id else: - id = get_or_set_files_id (directory + filename, size, md5sum, location_id); - ids.append(id); + id = get_or_set_files_id (directory + filename, size, md5sum, location_id) + ids.append(id) # If this is the .dsc itself; save the ID for later. if filename.endswith(".dsc"): - files_id = id; + files_id = id filename = directory + package + '_' + no_epoch_version + '.dsc' - cache_key = "%s~%s" % (package, version); + cache_key = "%s~%s" % (package, version) if not source_cache.has_key(cache_key): nasty_key = "%s~%s" % (package, version) - source_id_serial += 1; + source_id_serial += 1 if not source_cache_for_binaries.has_key(nasty_key): - source_cache_for_binaries[nasty_key] = source_id_serial; - tmp_source_id = source_id_serial; - source_cache[cache_key] = source_id_serial; + source_cache_for_binaries[nasty_key] = source_id_serial + tmp_source_id = source_id_serial + source_cache[cache_key] = source_id_serial source_query_cache.write("%d\t%s\t%s\t%d\t%d\t%s\t%s\n" % (source_id_serial, package, version, maintainer_id, files_id, install_date, fingerprint_id)) for id in ids: - dsc_files_id_serial += 1; - dsc_files_query_cache.write("%d\t%d\t%d\n" % (dsc_files_id_serial, tmp_source_id,id)); + dsc_files_id_serial += 1 + dsc_files_query_cache.write("%d\t%d\t%d\n" % (dsc_files_id_serial, tmp_source_id,id)) else: - tmp_source_id = source_cache[cache_key]; + tmp_source_id = source_cache[cache_key] - src_associations_id_serial += 1; + src_associations_id_serial += 1 src_associations_query_cache.write("%d\t%d\t%d\n" % (src_associations_id_serial, suite_id, tmp_source_id)) - file.close(); + file.close() ############################################################################### def process_packages (filename, suite, component, archive): - global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message; + global arch_all_cache, binary_cache, binaries_id_serial, binaries_query_cache, bin_associations_id_serial, bin_associations_query_cache, reject_message - count_total = 0; - count_bad = 0; - suite = suite.lower(); - suite_id = db_access.get_suite_id(suite); + count_total = 0 + count_bad = 0 + suite = suite.lower() + suite_id = db_access.get_suite_id(suite) try: - file = utils.open_file (filename); + file = utils.open_file (filename) except utils.cant_open_exc: - utils.warn("can't open '%s'" % (filename)); - return; - Scanner = apt_pkg.ParseTagFile(file); + utils.warn("can't open '%s'" % (filename)) + return + Scanner = apt_pkg.ParseTagFile(file) while Scanner.Step() != 0: package = Scanner.Section["package"] version = Scanner.Section["version"] maintainer = Scanner.Section["maintainer"] maintainer = maintainer.replace("'", "\\'") - maintainer_id = db_access.get_or_set_maintainer_id(maintainer); + maintainer_id = db_access.get_or_set_maintainer_id(maintainer) architecture = Scanner.Section["architecture"] - architecture_id = db_access.get_architecture_id (architecture); - fingerprint = "NOSIG"; - fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint); + architecture_id = db_access.get_architecture_id (architecture) + fingerprint = "NOSIG" + fingerprint_id = db_access.get_or_set_fingerprint_id(fingerprint) if not Scanner.Section.has_key("source"): source = package else: @@ -433,185 +433,185 @@ def process_packages (filename, suite, component, archive): if not source_version: source_version = version filename = Scanner.Section["filename"] - location = get_location_path(filename.split('/')[0]); + location = get_location_path(filename.split('/')[0]) location_id = db_access.get_location_id (location, component, archive) filename = poolify (filename, location) if architecture == "all": - filename = re_arch_from_filename.sub("binary-all", filename); - cache_key = "%s~%s" % (source, source_version); - source_id = source_cache_for_binaries.get(cache_key, None); - size = Scanner.Section["size"]; - md5sum = Scanner.Section["md5sum"]; - files_id = get_or_set_files_id (filename, size, md5sum, location_id); + filename = re_arch_from_filename.sub("binary-all", filename) + cache_key = "%s~%s" % (source, source_version) + source_id = source_cache_for_binaries.get(cache_key, None) + size = Scanner.Section["size"] + md5sum = Scanner.Section["md5sum"] + files_id = get_or_set_files_id (filename, size, md5sum, location_id) type = "deb"; # FIXME - cache_key = "%s~%s~%s~%d~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id); + cache_key = "%s~%s~%s~%d~%d~%d~%d" % (package, version, repr(source_id), architecture_id, location_id, files_id, suite_id) if not arch_all_cache.has_key(cache_key): - arch_all_cache[cache_key] = 1; - cache_key = "%s~%s~%s~%d" % (package, version, repr(source_id), architecture_id); + arch_all_cache[cache_key] = 1 + cache_key = "%s~%s~%s~%d" % (package, version, repr(source_id), architecture_id) if not binary_cache.has_key(cache_key): if not source_id: - source_id = "\N"; - count_bad += 1; + source_id = "\N" + count_bad += 1 else: - source_id = repr(source_id); - binaries_id_serial += 1; - binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type, fingerprint_id)); - binary_cache[cache_key] = binaries_id_serial; - tmp_binaries_id = binaries_id_serial; + source_id = repr(source_id) + binaries_id_serial += 1 + binaries_query_cache.write("%d\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\n" % (binaries_id_serial, package, version, maintainer_id, source_id, architecture_id, files_id, type, fingerprint_id)) + binary_cache[cache_key] = binaries_id_serial + tmp_binaries_id = binaries_id_serial else: - tmp_binaries_id = binary_cache[cache_key]; + tmp_binaries_id = binary_cache[cache_key] - bin_associations_id_serial += 1; - bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, suite_id, tmp_binaries_id)); - count_total += 1; + bin_associations_id_serial += 1 + bin_associations_query_cache.write("%d\t%d\t%d\n" % (bin_associations_id_serial, suite_id, tmp_binaries_id)) + count_total += 1 - file.close(); + file.close() if count_bad != 0: - print "%d binary packages processed; %d with no source match which is %.2f%%" % (count_total, count_bad, (float(count_bad)/count_total)*100); + print "%d binary packages processed; %d with no source match which is %.2f%%" % (count_total, count_bad, (float(count_bad)/count_total)*100) else: - print "%d binary packages processed; 0 with no source match which is 0%%" % (count_total); + print "%d binary packages processed; 0 with no source match which is 0%%" % (count_total) ############################################################################### def do_sources(sources, suite, component, server): - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename)); + temp_filename = utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (sources, temp_filename)) if (result != 0): - utils.fubar("Gunzip invocation failed!\n%s" % (output), result); - print 'Processing '+sources+'...'; - process_sources (temp_filename, suite, component, server); - os.unlink(temp_filename); + utils.fubar("Gunzip invocation failed!\n%s" % (output), result) + print 'Processing '+sources+'...' + process_sources (temp_filename, suite, component, server) + os.unlink(temp_filename) ############################################################################### def do_da_do_da (): - global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache; + global Cnf, projectB, query_cache, files_query_cache, source_query_cache, src_associations_query_cache, dsc_files_query_cache, bin_associations_query_cache, binaries_query_cache - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('a', "action", "Neve::Options::Action"), - ('h', "help", "Neve::Options::Help")]; + ('h', "help", "Neve::Options::Help")] for i in [ "action", "help" ]: if not Cnf.has_key("Neve::Options::%s" % (i)): - Cnf["Neve::Options::%s" % (i)] = ""; + Cnf["Neve::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Neve::Options") if Options["Help"]: - usage(); + usage() if not Options["Action"]: utils.warn("""no -a/--action given; not doing anything. Please read the documentation before running this script. -"""); - usage(1); +""") + usage(1) print "Re-Creating DB..." - (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1"); + (result, output) = commands.getstatusoutput("psql -f init_pool.sql template1") if (result != 0): - utils.fubar("psql invocation failed!\n", result); - print output; + utils.fubar("psql invocation failed!\n", result) + print output - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - db_access.init (Cnf, projectB); + db_access.init (Cnf, projectB) print "Adding static tables from conf file..." - projectB.query("BEGIN WORK"); - update_architectures(); - update_components(); - update_archives(); - update_locations(); - update_suites(); - update_override_type(); - update_priority(); - update_section(); - projectB.query("COMMIT WORK"); - - files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w"); - source_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"source","w"); - src_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"src_associations","w"); - dsc_files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"dsc_files","w"); - binaries_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"binaries","w"); - bin_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"bin_associations","w"); - - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") + update_architectures() + update_components() + update_archives() + update_locations() + update_suites() + update_override_type() + update_priority() + update_section() + projectB.query("COMMIT WORK") + + files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"files","w") + source_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"source","w") + src_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"src_associations","w") + dsc_files_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"dsc_files","w") + binaries_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"binaries","w") + bin_associations_query_cache = utils.open_file(Cnf["Neve::ExportDir"]+"bin_associations","w") + + projectB.query("BEGIN WORK") # Process Sources files to popoulate `source' and friends for location in Cnf.SubTree("Location").List(): - SubSec = Cnf.SubTree("Location::%s" % (location)); - server = SubSec["Archive"]; - type = Cnf.Find("Location::%s::Type" % (location)); + SubSec = Cnf.SubTree("Location::%s" % (location)) + server = SubSec["Archive"] + type = Cnf.Find("Location::%s::Type" % (location)) if type == "legacy-mixed": - sources = location + 'Sources.gz'; - suite = Cnf.Find("Location::%s::Suite" % (location)); - do_sources(sources, suite, "", server); + sources = location + 'Sources.gz' + suite = Cnf.Find("Location::%s::Suite" % (location)) + do_sources(sources, suite, "", server) elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): - sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz'; - do_sources(sources, suite, component, server); + sources = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/source/' + 'Sources.gz' + do_sources(sources, suite, component, server) else: - utils.fubar("Unknown location type ('%s')." % (type)); + utils.fubar("Unknown location type ('%s')." % (type)) # Process Packages files to populate `binaries' and friends for location in Cnf.SubTree("Location").List(): - SubSec = Cnf.SubTree("Location::%s" % (location)); - server = SubSec["Archive"]; - type = Cnf.Find("Location::%s::Type" % (location)); + SubSec = Cnf.SubTree("Location::%s" % (location)) + server = SubSec["Archive"] + type = Cnf.Find("Location::%s::Type" % (location)) if type == "legacy-mixed": - packages = location + 'Packages'; - suite = Cnf.Find("Location::%s::Suite" % (location)); - print 'Processing '+location+'...'; - process_packages (packages, suite, "", server); + packages = location + 'Packages' + suite = Cnf.Find("Location::%s::Suite" % (location)) + print 'Processing '+location+'...' + process_packages (packages, suite, "", server) elif type == "legacy" or type == "pool": for suite in Cnf.ValueList("Location::%s::Suites" % (location)): for component in Cnf.SubTree("Component").List(): architectures = filter(utils.real_arch, - Cnf.ValueList("Suite::%s::Architectures" % (suite))); + Cnf.ValueList("Suite::%s::Architectures" % (suite))) for architecture in architectures: packages = Cnf["Dir::Root"] + "dists/" + Cnf["Suite::%s::CodeName" % (suite)] + '/' + component + '/binary-' + architecture + '/Packages' - print 'Processing '+packages+'...'; - process_packages (packages, suite, component, server); - - files_query_cache.close(); - source_query_cache.close(); - src_associations_query_cache.close(); - dsc_files_query_cache.close(); - binaries_query_cache.close(); - bin_associations_query_cache.close(); - print "Writing data to `files' table..."; - projectB.query("COPY files FROM '%s'" % (Cnf["Neve::ExportDir"]+"files")); - print "Writing data to `source' table..."; - projectB.query("COPY source FROM '%s'" % (Cnf["Neve::ExportDir"]+"source")); - print "Writing data to `src_associations' table..."; - projectB.query("COPY src_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"src_associations")); - print "Writing data to `dsc_files' table..."; - projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Neve::ExportDir"]+"dsc_files")); - print "Writing data to `binaries' table..."; - projectB.query("COPY binaries FROM '%s'" % (Cnf["Neve::ExportDir"]+"binaries")); - print "Writing data to `bin_associations' table..."; - projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"bin_associations")); - print "Committing..."; - projectB.query("COMMIT WORK"); + print 'Processing '+packages+'...' + process_packages (packages, suite, component, server) + + files_query_cache.close() + source_query_cache.close() + src_associations_query_cache.close() + dsc_files_query_cache.close() + binaries_query_cache.close() + bin_associations_query_cache.close() + print "Writing data to `files' table..." + projectB.query("COPY files FROM '%s'" % (Cnf["Neve::ExportDir"]+"files")) + print "Writing data to `source' table..." + projectB.query("COPY source FROM '%s'" % (Cnf["Neve::ExportDir"]+"source")) + print "Writing data to `src_associations' table..." + projectB.query("COPY src_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"src_associations")) + print "Writing data to `dsc_files' table..." + projectB.query("COPY dsc_files FROM '%s'" % (Cnf["Neve::ExportDir"]+"dsc_files")) + print "Writing data to `binaries' table..." + projectB.query("COPY binaries FROM '%s'" % (Cnf["Neve::ExportDir"]+"binaries")) + print "Writing data to `bin_associations' table..." + projectB.query("COPY bin_associations FROM '%s'" % (Cnf["Neve::ExportDir"]+"bin_associations")) + print "Committing..." + projectB.query("COMMIT WORK") # Add the constraints and otherwise generally clean up the database. # See add_constraints.sql for more details... - print "Running add_constraints.sql..."; - (result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"])); + print "Running add_constraints.sql..." + (result, output) = commands.getstatusoutput("psql %s < add_constraints.sql" % (Cnf["DB::Name"])) print output if (result != 0): - utils.fubar("psql invocation failed!\n%s" % (output), result); + utils.fubar("psql invocation failed!\n%s" % (output), result) - return; + return ################################################################################ def main(): - utils.try_with_debug(do_da_do_da); + utils.try_with_debug(do_da_do_da) ################################################################################ if __name__ == '__main__': - main(); + main() diff --git a/dak/import_ldap_fingerprints.py b/dak/import_ldap_fingerprints.py index cb58049a..504f597b 100755 --- a/dak/import_ldap_fingerprints.py +++ b/dak/import_ldap_fingerprints.py @@ -45,17 +45,17 @@ ################################################################################ -import commands, ldap, pg, re, sys, time; -import apt_pkg; -import db_access, utils; +import commands, ldap, pg, re, sys, time +import apt_pkg +import db_access, utils ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None -re_gpg_fingerprint = re.compile(r"^\s+Key fingerprint = (.*)$", re.MULTILINE); -re_debian_address = re.compile(r"^.*<(.*)@debian\.org>$", re.MULTILINE); +re_gpg_fingerprint = re.compile(r"^\s+Key fingerprint = (.*)$", re.MULTILINE) +re_debian_address = re.compile(r"^.*<(.*)@debian\.org>$", re.MULTILINE) ################################################################################ @@ -69,148 +69,148 @@ Syncs fingerprint and uid tables with a debian.org LDAP DB ################################################################################ def get_ldap_value(entry, value): - ret = entry.get(value); + ret = entry.get(value) if not ret: - return ""; + return "" else: # FIXME: what about > 0 ? - return ret[0]; + return ret[0] def main(): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() - Arguments = [('h',"help","Emilie::Options::Help")]; + Arguments = [('h',"help","Emilie::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Emilie::Options::%s" % (i)): - Cnf["Emilie::Options::%s" % (i)] = ""; + Cnf["Emilie::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Emilie::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) - #before = time.time(); - #sys.stderr.write("[Getting info from the LDAP server..."); - LDAPDn = Cnf["Emilie::LDAPDn"]; - LDAPServer = Cnf["Emilie::LDAPServer"]; - l = ldap.open(LDAPServer); - l.simple_bind_s("",""); + #before = time.time() + #sys.stderr.write("[Getting info from the LDAP server...") + LDAPDn = Cnf["Emilie::LDAPDn"] + LDAPServer = Cnf["Emilie::LDAPServer"] + l = ldap.open(LDAPServer) + l.simple_bind_s("","") Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, "(&(keyfingerprint=*)(gidnumber=%s))" % (Cnf["Julia::ValidGID"]), - ["uid", "keyfingerprint"]); - #sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); + ["uid", "keyfingerprint"]) + #sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") # Sync LDAP with DB - db_fin_uid = {}; - ldap_fin_uid_id = {}; + db_fin_uid = {} + ldap_fin_uid_id = {} q = projectB.query(""" SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id - UNION SELECT f.fingerprint, f.id, null FROM fingerprint f where f.uid is null"""); + UNION SELECT f.fingerprint, f.id, null FROM fingerprint f where f.uid is null""") for i in q.getresult(): - (fingerprint, fingerprint_id, uid) = i; - db_fin_uid[fingerprint] = (uid, fingerprint_id); + (fingerprint, fingerprint_id, uid) = i + db_fin_uid[fingerprint] = (uid, fingerprint_id) for i in Attrs: - entry = i[1]; - fingerprints = entry["keyFingerPrint"]; - uid = entry["uid"][0]; - uid_id = db_access.get_or_set_uid_id(uid); + entry = i[1] + fingerprints = entry["keyFingerPrint"] + uid = entry["uid"][0] + uid_id = db_access.get_or_set_uid_id(uid) for fingerprint in fingerprints: - ldap_fin_uid_id[fingerprint] = (uid, uid_id); + ldap_fin_uid_id[fingerprint] = (uid, uid_id) if db_fin_uid.has_key(fingerprint): - (existing_uid, fingerprint_id) = db_fin_uid[fingerprint]; + (existing_uid, fingerprint_id) = db_fin_uid[fingerprint] if not existing_uid: - q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)); - print "Assigning %s to 0x%s." % (uid, fingerprint); + q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) + print "Assigning %s to 0x%s." % (uid, fingerprint) else: if existing_uid != uid: - utils.fubar("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid)); + utils.fubar("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid)) # Try to update people who sign with non-primary key - q = projectB.query("SELECT fingerprint, id FROM fingerprint WHERE uid is null"); + q = projectB.query("SELECT fingerprint, id FROM fingerprint WHERE uid is null") for i in q.getresult(): - (fingerprint, fingerprint_id) = i; + (fingerprint, fingerprint_id) = i cmd = "gpg --no-default-keyring --keyring=%s --keyring=%s --fingerprint %s" \ % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], - fingerprint); - (result, output) = commands.getstatusoutput(cmd); + fingerprint) + (result, output) = commands.getstatusoutput(cmd) if result == 0: - m = re_gpg_fingerprint.search(output); + m = re_gpg_fingerprint.search(output) if not m: print output - utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % (fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] "))); - primary_key = m.group(1); - primary_key = primary_key.replace(" ",""); + utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % (fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] "))) + primary_key = m.group(1) + primary_key = primary_key.replace(" ","") if not ldap_fin_uid_id.has_key(primary_key): - utils.fubar("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint)); - (uid, uid_id) = ldap_fin_uid_id[primary_key]; - q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)); - print "Assigning %s to 0x%s." % (uid, fingerprint); + utils.fubar("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint)) + (uid, uid_id) = ldap_fin_uid_id[primary_key] + q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) + print "Assigning %s to 0x%s." % (uid, fingerprint) else: - extra_keyrings = ""; + extra_keyrings = "" for keyring in Cnf.ValueList("Emilie::ExtraKeyrings"): - extra_keyrings += " --keyring=%s" % (keyring); + extra_keyrings += " --keyring=%s" % (keyring) cmd = "gpg --keyring=%s --keyring=%s %s --list-key %s" \ % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], - extra_keyrings, fingerprint); - (result, output) = commands.getstatusoutput(cmd); + extra_keyrings, fingerprint) + (result, output) = commands.getstatusoutput(cmd) if result != 0: - cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (Cnf["Emilie::KeyServer"], fingerprint); - (result, output) = commands.getstatusoutput(cmd); + cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (Cnf["Emilie::KeyServer"], fingerprint) + (result, output) = commands.getstatusoutput(cmd) if result != 0: - print "0x%s: NOT found on keyserver." % (fingerprint); + print "0x%s: NOT found on keyserver." % (fingerprint) print cmd print result print output - continue; + continue else: - cmd = "gpg --list-key %s" % (fingerprint); - (result, output) = commands.getstatusoutput(cmd); + cmd = "gpg --list-key %s" % (fingerprint) + (result, output) = commands.getstatusoutput(cmd) if result != 0: - print "0x%s: --list-key returned error after --recv-key didn't." % (fingerprint); + print "0x%s: --list-key returned error after --recv-key didn't." % (fingerprint) print cmd print result print output - continue; - m = re_debian_address.search(output); + continue + m = re_debian_address.search(output) if m: - guess_uid = m.group(1); + guess_uid = m.group(1) else: - guess_uid = "???"; - name = " ".join(output.split('\n')[0].split()[3:]); - print "0x%s -> %s -> %s" % (fingerprint, name, guess_uid); + guess_uid = "???" + name = " ".join(output.split('\n')[0].split()[3:]) + print "0x%s -> %s -> %s" % (fingerprint, name, guess_uid) # FIXME: make me optionally non-interactive # FIXME: default to the guessed ID - uid = None; + uid = None while not uid: - uid = utils.our_raw_input("Map to which UID ? "); + uid = utils.our_raw_input("Map to which UID ? ") Attrs = l.search_s(LDAPDn,ldap.SCOPE_ONELEVEL,"(uid=%s)" % (uid), ["cn","mn","sn"]) if not Attrs: print "That UID doesn't exist in LDAP!" - uid = None; + uid = None else: - entry = Attrs[0][1]; + entry = Attrs[0][1] name = " ".join([get_ldap_value(entry, "cn"), get_ldap_value(entry, "mn"), - get_ldap_value(entry, "sn")]); - prompt = "Map to %s - %s (y/N) ? " % (uid, name.replace(" "," ")); - yn = utils.our_raw_input(prompt).lower(); + get_ldap_value(entry, "sn")]) + prompt = "Map to %s - %s (y/N) ? " % (uid, name.replace(" "," ")) + yn = utils.our_raw_input(prompt).lower() if yn == "y": - uid_id = db_access.get_or_set_uid_id(uid); - projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)); - print "Assigning %s to 0x%s." % (uid, fingerprint); + uid_id = db_access.get_or_set_uid_id(uid) + projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) + print "Assigning %s to 0x%s." % (uid, fingerprint) else: - uid = None; - projectB.query("COMMIT WORK"); + uid = None + projectB.query("COMMIT WORK") ############################################################ diff --git a/dak/import_users_from_passwd.py b/dak/import_users_from_passwd.py index 6cf0e46a..e28afa98 100755 --- a/dak/import_users_from_passwd.py +++ b/dak/import_users_from_passwd.py @@ -30,14 +30,14 @@ ################################################################################ -import pg, pwd, sys; -import utils; -import apt_pkg; +import pg, pwd, sys +import utils +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ def usage (exit_code=0): @@ -53,66 +53,66 @@ Sync PostgreSQL's users with system users. ################################################################################ def main (): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() Arguments = [('n', "no-action", "Julia::Options::No-Action"), ('q', "quiet", "Julia::Options::Quiet"), ('v', "verbose", "Julia::Options::Verbose"), - ('h', "help", "Julia::Options::Help")]; + ('h', "help", "Julia::Options::Help")] for i in [ "no-action", "quiet", "verbose", "help" ]: if not Cnf.has_key("Julia::Options::%s" % (i)): - Cnf["Julia::Options::%s" % (i)] = ""; + Cnf["Julia::Options::%s" % (i)] = "" - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Julia::Options") if Options["Help"]: - usage(); + usage() elif arguments: - utils.warn("julia takes no non-option arguments."); - usage(1); + utils.warn("julia takes no non-option arguments.") + usage(1) - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - valid_gid = int(Cnf.get("Julia::ValidGID","")); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + valid_gid = int(Cnf.get("Julia::ValidGID","")) - passwd_unames = {}; + passwd_unames = {} for entry in pwd.getpwall(): - uname = entry[0]; - gid = entry[3]; + uname = entry[0] + gid = entry[3] if valid_gid and gid != valid_gid: if Options["Verbose"]: - print "Skipping %s (GID %s != Valid GID %s)." % (uname, gid, valid_gid); - continue; - passwd_unames[uname] = ""; + print "Skipping %s (GID %s != Valid GID %s)." % (uname, gid, valid_gid) + continue + passwd_unames[uname] = "" - postgres_unames = {}; - q = projectB.query("SELECT usename FROM pg_user"); - ql = q.getresult(); + postgres_unames = {} + q = projectB.query("SELECT usename FROM pg_user") + ql = q.getresult() for i in ql: - uname = i[0]; - postgres_unames[uname] = ""; + uname = i[0] + postgres_unames[uname] = "" - known_postgres_unames = {}; + known_postgres_unames = {} for i in Cnf.get("Julia::KnownPostgres","").split(","): - uname = i.strip(); - known_postgres_unames[uname] = ""; + uname = i.strip() + known_postgres_unames[uname] = "" keys = postgres_unames.keys() - keys.sort(); + keys.sort() for uname in keys: if not passwd_unames.has_key(uname)and not known_postgres_unames.has_key(uname): - print "W: %s is in Postgres but not the passwd file or list of known Postgres users." % (uname); + print "W: %s is in Postgres but not the passwd file or list of known Postgres users." % (uname) keys = passwd_unames.keys() - keys.sort(); + keys.sort() for uname in keys: if not postgres_unames.has_key(uname): if not Options["Quiet"]: - print "Creating %s user in Postgres." % (uname); + print "Creating %s user in Postgres." % (uname) if not Options["No-Action"]: - q = projectB.query('CREATE USER "%s"' % (uname)); + q = projectB.query('CREATE USER "%s"' % (uname)) ####################################################################################### diff --git a/dak/init_db.py b/dak/init_db.py index a1b75d34..92e6a6e7 100755 --- a/dak/init_db.py +++ b/dak/init_db.py @@ -20,14 +20,14 @@ ################################################################################ -import pg, sys; -import utils, db_access; -import apt_pkg; +import pg, sys +import utils, db_access +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ @@ -42,136 +42,136 @@ Initalizes some tables in the projectB database based on the config file. def get (c, i): if c.has_key(i): - return "'%s'" % (c[i]); + return "'%s'" % (c[i]) else: - return "NULL"; + return "NULL" def main (): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() - Arguments = [('h',"help","Alyson::Options::Help")]; + Arguments = [('h',"help","Alyson::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Alyson::Options::%s" % (i)): - Cnf["Alyson::Options::%s" % (i)] = ""; + Cnf["Alyson::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Alyson::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) # archive - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM archive"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM archive") for name in Cnf.SubTree("Archive").List(): - Archive = Cnf.SubTree("Archive::%s" % (name)); - origin_server = get(Archive, "OriginServer"); - description = get(Archive, "Description"); - projectB.query("INSERT INTO archive (name, origin_server, description) VALUES ('%s', %s, %s)" % (name, origin_server, description)); - projectB.query("COMMIT WORK"); + Archive = Cnf.SubTree("Archive::%s" % (name)) + origin_server = get(Archive, "OriginServer") + description = get(Archive, "Description") + projectB.query("INSERT INTO archive (name, origin_server, description) VALUES ('%s', %s, %s)" % (name, origin_server, description)) + projectB.query("COMMIT WORK") # architecture - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM architecture"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM architecture") for arch in Cnf.SubTree("Architectures").List(): - description = Cnf["Architectures::%s" % (arch)]; - projectB.query("INSERT INTO architecture (arch_string, description) VALUES ('%s', '%s')" % (arch, description)); - projectB.query("COMMIT WORK"); + description = Cnf["Architectures::%s" % (arch)] + projectB.query("INSERT INTO architecture (arch_string, description) VALUES ('%s', '%s')" % (arch, description)) + projectB.query("COMMIT WORK") # component - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM component"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM component") for name in Cnf.SubTree("Component").List(): - Component = Cnf.SubTree("Component::%s" % (name)); - description = get(Component, "Description"); + Component = Cnf.SubTree("Component::%s" % (name)) + description = get(Component, "Description") if Component.get("MeetsDFSG").lower() == "true": - meets_dfsg = "true"; + meets_dfsg = "true" else: - meets_dfsg = "false"; - projectB.query("INSERT INTO component (name, description, meets_dfsg) VALUES ('%s', %s, %s)" % (name, description, meets_dfsg)); - projectB.query("COMMIT WORK"); + meets_dfsg = "false" + projectB.query("INSERT INTO component (name, description, meets_dfsg) VALUES ('%s', %s, %s)" % (name, description, meets_dfsg)) + projectB.query("COMMIT WORK") # location - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM location"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM location") for location in Cnf.SubTree("Location").List(): - Location = Cnf.SubTree("Location::%s" % (location)); - archive_id = db_access.get_archive_id(Location["Archive"]); - type = Location.get("type"); + Location = Cnf.SubTree("Location::%s" % (location)) + archive_id = db_access.get_archive_id(Location["Archive"]) + type = Location.get("type") if type == "legacy-mixed": - projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, Location["type"])); + projectB.query("INSERT INTO location (path, archive, type) VALUES ('%s', %d, '%s')" % (location, archive_id, Location["type"])) elif type == "legacy" or type == "pool": for component in Cnf.SubTree("Component").List(): - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) projectB.query("INSERT INTO location (path, component, archive, type) VALUES ('%s', %d, %d, '%s')" % - (location, component_id, archive_id, type)); + (location, component_id, archive_id, type)) else: - utils.fubar("E: type '%s' not recognised in location %s." % (type, location)); - projectB.query("COMMIT WORK"); + utils.fubar("E: type '%s' not recognised in location %s." % (type, location)) + projectB.query("COMMIT WORK") # suite - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") projectB.query("DELETE FROM suite") for suite in Cnf.SubTree("Suite").List(): Suite = Cnf.SubTree("Suite::%s" %(suite)) - version = get(Suite, "Version"); - origin = get(Suite, "Origin"); - description = get(Suite, "Description"); + version = get(Suite, "Version") + origin = get(Suite, "Origin") + description = get(Suite, "Description") projectB.query("INSERT INTO suite (suite_name, version, origin, description) VALUES ('%s', %s, %s, %s)" - % (suite.lower(), version, origin, description)); + % (suite.lower(), version, origin, description)) for architecture in Cnf.ValueList("Suite::%s::Architectures" % (suite)): - architecture_id = db_access.get_architecture_id (architecture); + architecture_id = db_access.get_architecture_id (architecture) if architecture_id < 0: - utils.fubar("architecture '%s' not found in architecture table for suite %s." % (architecture, suite)); - projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)); - projectB.query("COMMIT WORK"); + utils.fubar("architecture '%s' not found in architecture table for suite %s." % (architecture, suite)) + projectB.query("INSERT INTO suite_architectures (suite, architecture) VALUES (currval('suite_id_seq'), %d)" % (architecture_id)) + projectB.query("COMMIT WORK") # override_type - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM override_type"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM override_type") for type in Cnf.ValueList("OverrideType"): - projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)); - projectB.query("COMMIT WORK"); + projectB.query("INSERT INTO override_type (type) VALUES ('%s')" % (type)) + projectB.query("COMMIT WORK") # priority - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM priority"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM priority") for priority in Cnf.SubTree("Priority").List(): - projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])); - projectB.query("COMMIT WORK"); + projectB.query("INSERT INTO priority (priority, level) VALUES ('%s', %s)" % (priority, Cnf["Priority::%s" % (priority)])) + projectB.query("COMMIT WORK") # section - projectB.query("BEGIN WORK"); - projectB.query("DELETE FROM section"); + projectB.query("BEGIN WORK") + projectB.query("DELETE FROM section") for component in Cnf.SubTree("Component").List(): if Cnf["Natalie::ComponentPosition"] == "prefix": - suffix = ""; + suffix = "" if component != "main": - prefix = component + '/'; + prefix = component + '/' else: - prefix = ""; + prefix = "" else: - prefix = ""; - component = component.replace("non-US/", ""); + prefix = "" + component = component.replace("non-US/", "") if component != "main": - suffix = '/' + component; + suffix = '/' + component else: - suffix = ""; + suffix = "" for section in Cnf.ValueList("Section"): - projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)); - projectB.query("COMMIT WORK"); + projectB.query("INSERT INTO section (section) VALUES ('%s%s%s')" % (prefix, section, suffix)) + projectB.query("COMMIT WORK") ################################################################################ diff --git a/dak/init_dirs.py b/dak/init_dirs.py index d462ca48..b38b4b86 100755 --- a/dak/init_dirs.py +++ b/dak/init_dirs.py @@ -20,14 +20,14 @@ ################################################################################ -import os, sys; -import utils; -import apt_pkg; +import os, sys +import utils +import apt_pkg ################################################################################ -Cnf = None; -AptCnf = None; +Cnf = None +AptCnf = None ################################################################################ @@ -43,80 +43,80 @@ Creates directories for an archive based on katie.conf configuration file. def do_dir(target, config_name): if os.path.exists(target): if not os.path.isdir(target): - utils.fubar("%s (%s) is not a directory." % (target, config_name)); + utils.fubar("%s (%s) is not a directory." % (target, config_name)) else: - print "Creating %s ..." % (target); - os.makedirs(target); + print "Creating %s ..." % (target) + os.makedirs(target) def process_file(config, config_name): if config.has_key(config_name): - target = os.path.dirname(config[config_name]); - do_dir(target, config_name); + target = os.path.dirname(config[config_name]) + do_dir(target, config_name) def process_tree(config, tree): for entry in config.SubTree(tree).List(): - entry = entry.lower(); + entry = entry.lower() if tree == "Dir": if entry in [ "poolroot", "queue" , "morguereject" ]: - continue; - config_name = "%s::%s" % (tree, entry); - target = config[config_name]; - do_dir(target, config_name); + continue + config_name = "%s::%s" % (tree, entry) + target = config[config_name] + do_dir(target, config_name) def process_morguesubdir(subdir): - config_name = "%s::MorgueSubDir" % (subdir); + config_name = "%s::MorgueSubDir" % (subdir) if Cnf.has_key(config_name): - target = os.path.join(Cnf["Dir::Morgue"], Cnf[config_name]); - do_dir(target, config_name); + target = os.path.join(Cnf["Dir::Morgue"], Cnf[config_name]) + do_dir(target, config_name) ###################################################################### def create_directories(): # Process directories from apt.conf - process_tree(Cnf, "Dir"); - process_tree(Cnf, "Dir::Queue"); + process_tree(Cnf, "Dir") + process_tree(Cnf, "Dir::Queue") for file in [ "Dinstall::LockFile", "Melanie::LogFile", "Neve::ExportDir" ]: - process_file(Cnf, file); + process_file(Cnf, file) for subdir in [ "Shania", "Rhona" ]: - process_morguesubdir(subdir); + process_morguesubdir(subdir) # Process directories from apt.conf - process_tree(AptCnf, "Dir"); + process_tree(AptCnf, "Dir") for tree in AptCnf.SubTree("Tree").List(): - config_name = "Tree::%s" % (tree); - tree_dir = os.path.join(Cnf["Dir::Root"], tree); - do_dir(tree_dir, tree); + config_name = "Tree::%s" % (tree) + tree_dir = os.path.join(Cnf["Dir::Root"], tree) + do_dir(tree_dir, tree) for file in [ "FileList", "SourceFileList" ]: - process_file(AptCnf, "%s::%s" % (config_name, file)); + process_file(AptCnf, "%s::%s" % (config_name, file)) for component in AptCnf["%s::Sections" % (config_name)].split(): for architecture in AptCnf["%s::Architectures" % (config_name)].split(): if architecture != "source": - architecture = "binary-"+architecture; - target = os.path.join(tree_dir,component,architecture); - do_dir(target, "%s, %s, %s" % (tree, component, architecture)); + architecture = "binary-"+architecture + target = os.path.join(tree_dir,component,architecture) + do_dir(target, "%s, %s, %s" % (tree, component, architecture)) ################################################################################ def main (): - global AptCnf, Cnf, projectB; + global AptCnf, Cnf, projectB Cnf = utils.get_conf() - Arguments = [('h',"help","Rose::Options::Help")]; + Arguments = [('h',"help","Rose::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Rose::Options::%s" % (i)): - Cnf["Rose::Options::%s" % (i)] = ""; + Cnf["Rose::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Rose::Options") if Options["Help"]: - usage(); + usage() - AptCnf = apt_pkg.newConfiguration(); - apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file()); + AptCnf = apt_pkg.newConfiguration() + apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file()) - create_directories(); + create_directories() ################################################################################ diff --git a/dak/lib/database.py b/dak/lib/database.py index cdc113f7..6976f2fb 100644 --- a/dak/lib/database.py +++ b/dak/lib/database.py @@ -20,50 +20,50 @@ ################################################################################ -import sys, time, types; +import sys, time, types ################################################################################ -Cnf = None; -projectB = None; -suite_id_cache = {}; -section_id_cache = {}; -priority_id_cache = {}; -override_type_id_cache = {}; -architecture_id_cache = {}; -archive_id_cache = {}; -component_id_cache = {}; -location_id_cache = {}; -maintainer_id_cache = {}; -source_id_cache = {}; -files_id_cache = {}; -maintainer_cache = {}; -fingerprint_id_cache = {}; -queue_id_cache = {}; -uid_id_cache = {}; +Cnf = None +projectB = None +suite_id_cache = {} +section_id_cache = {} +priority_id_cache = {} +override_type_id_cache = {} +architecture_id_cache = {} +archive_id_cache = {} +component_id_cache = {} +location_id_cache = {} +maintainer_id_cache = {} +source_id_cache = {} +files_id_cache = {} +maintainer_cache = {} +fingerprint_id_cache = {} +queue_id_cache = {} +uid_id_cache = {} ################################################################################ def init (config, sql): global Cnf, projectB - Cnf = config; - projectB = sql; + Cnf = config + projectB = sql def do_query(q): - sys.stderr.write("query: \"%s\" ... " % (q)); - before = time.time(); - r = projectB.query(q); - time_diff = time.time()-before; - sys.stderr.write("took %.3f seconds.\n" % (time_diff)); + sys.stderr.write("query: \"%s\" ... " % (q)) + before = time.time() + r = projectB.query(q) + time_diff = time.time()-before + sys.stderr.write("took %.3f seconds.\n" % (time_diff)) if type(r) is int: - sys.stderr.write("int result: %s\n" % (r)); + sys.stderr.write("int result: %s\n" % (r)) elif type(r) is types.NoneType: - sys.stderr.write("result: None\n"); + sys.stderr.write("result: None\n") else: - sys.stderr.write("pgresult: %s\n" % (r.getresult())); - return r; + sys.stderr.write("pgresult: %s\n" % (r.getresult())) + return r ################################################################################ @@ -74,11 +74,11 @@ def get_suite_id (suite): return suite_id_cache[suite] q = projectB.query("SELECT id FROM suite WHERE suite_name = '%s'" % (suite)) - ql = q.getresult(); + ql = q.getresult() if not ql: - return -1; + return -1 - suite_id = ql[0][0]; + suite_id = ql[0][0] suite_id_cache[suite] = suite_id return suite_id @@ -90,11 +90,11 @@ def get_section_id (section): return section_id_cache[section] q = projectB.query("SELECT id FROM section WHERE section = '%s'" % (section)) - ql = q.getresult(); + ql = q.getresult() if not ql: - return -1; + return -1 - section_id = ql[0][0]; + section_id = ql[0][0] section_id_cache[section] = section_id return section_id @@ -106,59 +106,59 @@ def get_priority_id (priority): return priority_id_cache[priority] q = projectB.query("SELECT id FROM priority WHERE priority = '%s'" % (priority)) - ql = q.getresult(); + ql = q.getresult() if not ql: - return -1; + return -1 - priority_id = ql[0][0]; + priority_id = ql[0][0] priority_id_cache[priority] = priority_id return priority_id def get_override_type_id (type): - global override_type_id_cache; + global override_type_id_cache if override_type_id_cache.has_key(type): - return override_type_id_cache[type]; + return override_type_id_cache[type] - q = projectB.query("SELECT id FROM override_type WHERE type = '%s'" % (type)); - ql = q.getresult(); + q = projectB.query("SELECT id FROM override_type WHERE type = '%s'" % (type)) + ql = q.getresult() if not ql: - return -1; + return -1 - override_type_id = ql[0][0]; - override_type_id_cache[type] = override_type_id; + override_type_id = ql[0][0] + override_type_id_cache[type] = override_type_id - return override_type_id; + return override_type_id def get_architecture_id (architecture): - global architecture_id_cache; + global architecture_id_cache if architecture_id_cache.has_key(architecture): - return architecture_id_cache[architecture]; + return architecture_id_cache[architecture] q = projectB.query("SELECT id FROM architecture WHERE arch_string = '%s'" % (architecture)) - ql = q.getresult(); + ql = q.getresult() if not ql: - return -1; + return -1 - architecture_id = ql[0][0]; - architecture_id_cache[architecture] = architecture_id; + architecture_id = ql[0][0] + architecture_id_cache[architecture] = architecture_id - return architecture_id; + return architecture_id def get_archive_id (archive): global archive_id_cache - archive = archive.lower(); + archive = archive.lower() if archive_id_cache.has_key(archive): return archive_id_cache[archive] - q = projectB.query("SELECT id FROM archive WHERE lower(name) = '%s'" % (archive)); - ql = q.getresult(); + q = projectB.query("SELECT id FROM archive WHERE lower(name) = '%s'" % (archive)) + ql = q.getresult() if not ql: - return -1; + return -1 archive_id = ql[0][0] archive_id_cache[archive] = archive_id @@ -168,17 +168,17 @@ def get_archive_id (archive): def get_component_id (component): global component_id_cache - component = component.lower(); + component = component.lower() if component_id_cache.has_key(component): return component_id_cache[component] q = projectB.query("SELECT id FROM component WHERE lower(name) = '%s'" % (component)) - ql = q.getresult(); + ql = q.getresult() if not ql: - return -1; + return -1 - component_id = ql[0][0]; + component_id = ql[0][0] component_id_cache[component] = component_id return component_id @@ -197,9 +197,9 @@ def get_location_id (location, component, archive): q = projectB.query("SELECT id FROM location WHERE path = '%s' AND component = %d AND archive = %d" % (location, component_id, archive_id)) else: q = projectB.query("SELECT id FROM location WHERE path = '%s' AND archive = %d" % (location, archive_id)) - ql = q.getresult(); + ql = q.getresult() if not ql: - return -1; + return -1 location_id = ql[0][0] location_id_cache[cache_key] = location_id @@ -243,59 +243,59 @@ def get_or_set_maintainer_id (maintainer): ################################################################################ def get_or_set_uid_id (uid): - global uid_id_cache; + global uid_id_cache if uid_id_cache.has_key(uid): - return uid_id_cache[uid]; + return uid_id_cache[uid] q = projectB.query("SELECT id FROM uid WHERE uid = '%s'" % (uid)) if not q.getresult(): - projectB.query("INSERT INTO uid (uid) VALUES ('%s')" % (uid)); - q = projectB.query("SELECT id FROM uid WHERE uid = '%s'" % (uid)); - uid_id = q.getresult()[0][0]; - uid_id_cache[uid] = uid_id; + projectB.query("INSERT INTO uid (uid) VALUES ('%s')" % (uid)) + q = projectB.query("SELECT id FROM uid WHERE uid = '%s'" % (uid)) + uid_id = q.getresult()[0][0] + uid_id_cache[uid] = uid_id - return uid_id; + return uid_id ################################################################################ def get_or_set_fingerprint_id (fingerprint): - global fingerprint_id_cache; + global fingerprint_id_cache if fingerprint_id_cache.has_key(fingerprint): return fingerprint_id_cache[fingerprint] - q = projectB.query("SELECT id FROM fingerprint WHERE fingerprint = '%s'" % (fingerprint)); + q = projectB.query("SELECT id FROM fingerprint WHERE fingerprint = '%s'" % (fingerprint)) if not q.getresult(): - projectB.query("INSERT INTO fingerprint (fingerprint) VALUES ('%s')" % (fingerprint)); - q = projectB.query("SELECT id FROM fingerprint WHERE fingerprint = '%s'" % (fingerprint)); - fingerprint_id = q.getresult()[0][0]; - fingerprint_id_cache[fingerprint] = fingerprint_id; + projectB.query("INSERT INTO fingerprint (fingerprint) VALUES ('%s')" % (fingerprint)) + q = projectB.query("SELECT id FROM fingerprint WHERE fingerprint = '%s'" % (fingerprint)) + fingerprint_id = q.getresult()[0][0] + fingerprint_id_cache[fingerprint] = fingerprint_id - return fingerprint_id; + return fingerprint_id ################################################################################ def get_files_id (filename, size, md5sum, location_id): global files_id_cache - cache_key = "%s~%d" % (filename, location_id); + cache_key = "%s~%d" % (filename, location_id) if files_id_cache.has_key(cache_key): return files_id_cache[cache_key] - size = int(size); - q = projectB.query("SELECT id, size, md5sum FROM files WHERE filename = '%s' AND location = %d" % (filename, location_id)); - ql = q.getresult(); + size = int(size) + q = projectB.query("SELECT id, size, md5sum FROM files WHERE filename = '%s' AND location = %d" % (filename, location_id)) + ql = q.getresult() if ql: if len(ql) != 1: - return -1; - ql = ql[0]; - orig_size = int(ql[1]); - orig_md5sum = ql[2]; + return -1 + ql = ql[0] + orig_size = int(ql[1]) + orig_md5sum = ql[2] if orig_size != size or orig_md5sum != md5sum: - return -2; - files_id_cache[cache_key] = ql[0]; + return -2 + files_id_cache[cache_key] = ql[0] return files_id_cache[cache_key] else: return None @@ -322,29 +322,29 @@ def get_or_set_queue_id (queue): def set_files_id (filename, size, md5sum, location_id): global files_id_cache - projectB.query("INSERT INTO files (filename, size, md5sum, location) VALUES ('%s', %d, '%s', %d)" % (filename, long(size), md5sum, location_id)); + projectB.query("INSERT INTO files (filename, size, md5sum, location) VALUES ('%s', %d, '%s', %d)" % (filename, long(size), md5sum, location_id)) - return get_files_id (filename, size, md5sum, location_id); + return get_files_id (filename, size, md5sum, location_id) - ### currval has issues with postgresql 7.1.3 when the table is big; + ### currval has issues with postgresql 7.1.3 when the table is big ### it was taking ~3 seconds to return on auric which is very Not ### Cool(tm). ## - ##q = projectB.query("SELECT id FROM files WHERE id = currval('files_id_seq')"); - ##ql = q.getresult()[0]; - ##cache_key = "%s~%d" % (filename, location_id); + ##q = projectB.query("SELECT id FROM files WHERE id = currval('files_id_seq')") + ##ql = q.getresult()[0] + ##cache_key = "%s~%d" % (filename, location_id) ##files_id_cache[cache_key] = ql[0] - ##return files_id_cache[cache_key]; + ##return files_id_cache[cache_key] ################################################################################ def get_maintainer (maintainer_id): - global maintainer_cache; + global maintainer_cache if not maintainer_cache.has_key(maintainer_id): - q = projectB.query("SELECT name FROM maintainer WHERE id = %s" % (maintainer_id)); - maintainer_cache[maintainer_id] = q.getresult()[0][0]; + q = projectB.query("SELECT name FROM maintainer WHERE id = %s" % (maintainer_id)) + maintainer_cache[maintainer_id] = q.getresult()[0][0] - return maintainer_cache[maintainer_id]; + return maintainer_cache[maintainer_id] ################################################################################ diff --git a/dak/lib/logging.py b/dak/lib/logging.py index 5fe0cf63..bd81582a 100644 --- a/dak/lib/logging.py +++ b/dak/lib/logging.py @@ -20,53 +20,53 @@ ################################################################################ -import os, pwd, time, sys; -import utils; +import os, pwd, time, sys +import utils ################################################################################ class Logger: "Logger object" - Cnf = None; - logfile = None; - program = None; + Cnf = None + logfile = None + program = None def __init__ (self, Cnf, program, debug=0): "Initialize a new Logger object" - self.Cnf = Cnf; - self.program = program; + self.Cnf = Cnf + self.program = program # Create the log directory if it doesn't exist - logdir = Cnf["Dir::Log"]; + logdir = Cnf["Dir::Log"] if not os.path.exists(logdir): - umask = os.umask(00000); - os.makedirs(logdir, 02775); + umask = os.umask(00000) + os.makedirs(logdir, 02775) # Open the logfile - logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m")); + logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m")) logfile = None if debug: logfile = sys.stderr else: - logfile = utils.open_file(logfilename, 'a'); - self.logfile = logfile; + logfile = utils.open_file(logfilename, 'a') + self.logfile = logfile # Log the start of the program - user = pwd.getpwuid(os.getuid())[0]; - self.log(["program start", user]); + user = pwd.getpwuid(os.getuid())[0] + self.log(["program start", user]) def log (self, details): "Log an event" # Prepend the timestamp and program name - details.insert(0, self.program); - timestamp = time.strftime("%Y%m%d%H%M%S"); - details.insert(0, timestamp); + details.insert(0, self.program) + timestamp = time.strftime("%Y%m%d%H%M%S") + details.insert(0, timestamp) # Force the contents of the list to be string.join-able - details = map(str, details); + details = map(str, details) # Write out the log in TSV - self.logfile.write("|".join(details)+'\n'); + self.logfile.write("|".join(details)+'\n') # Flush the output to enable tail-ing - self.logfile.flush(); + self.logfile.flush() def close (self): "Close a Logger object" - self.log(["program end"]); - self.logfile.flush(); - self.logfile.close(); + self.log(["program end"]) + self.logfile.flush() + self.logfile.close() diff --git a/dak/lib/queue.py b/dak/lib/queue.py index 7f428318..d565c4de 100644 --- a/dak/lib/queue.py +++ b/dak/lib/queue.py @@ -20,116 +20,116 @@ ############################################################################### -import cPickle, errno, os, pg, re, stat, string, sys, time; -import utils, db_access; -import apt_inst, apt_pkg; +import cPickle, errno, os, pg, re, stat, string, sys, time +import utils, db_access +import apt_inst, apt_pkg -from types import *; +from types import * ############################################################################### -re_isanum = re.compile (r"^\d+$"); -re_default_answer = re.compile(r"\[(.*)\]"); -re_fdnic = re.compile(r"\n\n"); -re_bin_only_nmu = re.compile(r"\+b\d+$"); +re_isanum = re.compile (r"^\d+$") +re_default_answer = re.compile(r"\[(.*)\]") +re_fdnic = re.compile(r"\n\n") +re_bin_only_nmu = re.compile(r"\+b\d+$") ############################################################################### # Convenience wrapper to carry around all the package information in class Pkg: def __init__(self, **kwds): - self.__dict__.update(kwds); + self.__dict__.update(kwds) def update(self, **kwds): - self.__dict__.update(kwds); + self.__dict__.update(kwds) ############################################################################### class nmu_p: # Read in the group maintainer override file def __init__ (self, Cnf): - self.group_maint = {}; - self.Cnf = Cnf; + self.group_maint = {} + self.Cnf = Cnf if Cnf.get("Dinstall::GroupOverrideFilename"): - filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"]; - file = utils.open_file(filename); + filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"] + file = utils.open_file(filename) for line in file.readlines(): - line = utils.re_comments.sub('', line).lower().strip(); + line = utils.re_comments.sub('', line).lower().strip() if line != "": - self.group_maint[line] = 1; - file.close(); + self.group_maint[line] = 1 + file.close() def is_an_nmu (self, pkg): - Cnf = self.Cnf; - changes = pkg.changes; - dsc = pkg.dsc; + Cnf = self.Cnf + changes = pkg.changes + dsc = pkg.dsc i = utils.fix_maintainer (dsc.get("maintainer", - Cnf["Dinstall::MyEmailAddress"]).lower()); - (dsc_rfc822, dsc_rfc2047, dsc_name, dsc_email) = i; + Cnf["Dinstall::MyEmailAddress"]).lower()) + (dsc_rfc822, dsc_rfc2047, dsc_name, dsc_email) = i # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry if dsc_name == changes["maintainername"].lower() and \ (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name): - return 0; + return 0 if dsc.has_key("uploaders"): - uploaders = dsc["uploaders"].lower().split(","); - uploadernames = {}; + uploaders = dsc["uploaders"].lower().split(",") + uploadernames = {} for i in uploaders: - (rfc822, rfc2047, name, email) = utils.fix_maintainer (i.strip()); - uploadernames[name] = ""; + (rfc822, rfc2047, name, email) = utils.fix_maintainer (i.strip()) + uploadernames[name] = "" if uploadernames.has_key(changes["changedbyname"].lower()): - return 0; + return 0 # Some group maintained packages (e.g. Debian QA) are never NMU's if self.group_maint.has_key(changes["maintaineremail"].lower()): - return 0; + return 0 - return 1; + return 1 ############################################################################### class Katie: def __init__(self, Cnf): - self.Cnf = Cnf; + self.Cnf = Cnf # Read in the group-maint override file - self.nmu = nmu_p(Cnf); - self.accept_count = 0; - self.accept_bytes = 0L; + self.nmu = nmu_p(Cnf) + self.accept_count = 0 + self.accept_bytes = 0L self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {}, - legacy_source_untouchable = {}); + legacy_source_untouchable = {}) # Initialize the substitution template mapping global - Subst = self.Subst = {}; - Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]; - Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]; - Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]; - Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]; + Subst = self.Subst = {} + Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"] + Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"] + Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"] + Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] - self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, self.projectB); + self.projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, self.projectB) ########################################################################### def init_vars (self): for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]: - exec "self.pkg.%s.clear();" % (i); - self.pkg.orig_tar_id = None; - self.pkg.orig_tar_location = ""; - self.pkg.orig_tar_gz = None; + exec "self.pkg.%s.clear();" % (i) + self.pkg.orig_tar_id = None + self.pkg.orig_tar_location = "" + self.pkg.orig_tar_gz = None ########################################################################### def update_vars (self): - dump_filename = self.pkg.changes_file[:-8]+".katie"; - dump_file = utils.open_file(dump_filename); - p = cPickle.Unpickler(dump_file); + dump_filename = self.pkg.changes_file[:-8]+".katie" + dump_file = utils.open_file(dump_filename) + p = cPickle.Unpickler(dump_file) for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]: - exec "self.pkg.%s.update(p.load());" % (i); + exec "self.pkg.%s.update(p.load());" % (i) for i in [ "orig_tar_id", "orig_tar_location" ]: - exec "self.pkg.%s = p.load();" % (i); - dump_file.close(); + exec "self.pkg.%s = p.load();" % (i) + dump_file.close() ########################################################################### @@ -139,32 +139,32 @@ class Katie: def dump_vars(self, dest_dir): for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]: - exec "%s = self.pkg.%s;" % (i,i); - dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie"); - dump_file = utils.open_file(dump_filename, 'w'); + exec "%s = self.pkg.%s;" % (i,i) + dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie") + dump_file = utils.open_file(dump_filename, 'w') try: - os.chmod(dump_filename, 0660); + os.chmod(dump_filename, 0660) except OSError, e: if errno.errorcode[e.errno] == 'EPERM': - perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]); + perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]) if perms & stat.S_IROTH: - utils.fubar("%s is world readable and chmod failed." % (dump_filename)); + utils.fubar("%s is world readable and chmod failed." % (dump_filename)) else: - raise; + raise - p = cPickle.Pickler(dump_file, 1); + p = cPickle.Pickler(dump_file, 1) for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]: - exec "%s = {}" % i; + exec "%s = {}" % i ## files for file in files.keys(): - d_files[file] = {}; + d_files[file] = {} for i in [ "package", "version", "architecture", "type", "size", "md5sum", "component", "location id", "source package", "source version", "maintainer", "dbtype", "files id", "new", "section", "priority", "othercomponents", "pool name", "original component" ]: if files[file].has_key(i): - d_files[file][i] = files[file][i]; + d_files[file][i] = files[file][i] ## changes # Mandatory changes fields for i in [ "distribution", "source", "architecture", "version", @@ -172,87 +172,87 @@ class Katie: "changedby2047", "changedbyname", "maintainer822", "maintainer2047", "maintainername", "maintaineremail", "closes", "changes" ]: - d_changes[i] = changes[i]; + d_changes[i] = changes[i] # Optional changes fields for i in [ "changed-by", "filecontents", "format", "lisa note", "distribution-version" ]: if changes.has_key(i): - d_changes[i] = changes[i]; + d_changes[i] = changes[i] ## dsc for i in [ "source", "version", "maintainer", "fingerprint", "uploaders", "bts changelog" ]: if dsc.has_key(i): - d_dsc[i] = dsc[i]; + d_dsc[i] = dsc[i] ## dsc_files for file in dsc_files.keys(): - d_dsc_files[file] = {}; + d_dsc_files[file] = {} # Mandatory dsc_files fields for i in [ "size", "md5sum" ]: - d_dsc_files[file][i] = dsc_files[file][i]; + d_dsc_files[file][i] = dsc_files[file][i] # Optional dsc_files fields for i in [ "files id" ]: if dsc_files[file].has_key(i): - d_dsc_files[file][i] = dsc_files[file][i]; + d_dsc_files[file][i] = dsc_files[file][i] for i in [ d_changes, d_dsc, d_files, d_dsc_files, legacy_source_untouchable, orig_tar_id, orig_tar_location ]: - p.dump(i); - dump_file.close(); + p.dump(i) + dump_file.close() ########################################################################### # Set up the per-package template substitution mappings def update_subst (self, reject_message = ""): - Subst = self.Subst; - changes = self.pkg.changes; + Subst = self.Subst + changes = self.pkg.changes # If jennifer crashed out in the right place, architecture may still be a string. if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType): - changes["architecture"] = { "Unknown" : "" }; + changes["architecture"] = { "Unknown" : "" } # and maintainer2047 may not exist. if not changes.has_key("maintainer2047"): - changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"]; + changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"] - Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys()); - Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file); - Subst["__FILE_CONTENTS__"] = changes.get("filecontents", ""); + Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys()) + Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file) + Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "") # For source uploads the Changed-By field wins; otherwise Maintainer wins. if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]): - Subst["__MAINTAINER_FROM__"] = changes["changedby2047"]; + Subst["__MAINTAINER_FROM__"] = changes["changedby2047"] Subst["__MAINTAINER_TO__"] = "%s, %s" % (changes["changedby2047"], - changes["maintainer2047"]); - Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown"); + changes["maintainer2047"]) + Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown") else: - Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"]; - Subst["__MAINTAINER_TO__"] = changes["maintainer2047"]; - Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown"); + Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"] + Subst["__MAINTAINER_TO__"] = changes["maintainer2047"] + Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown") if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"): Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"]) # Apply any global override of the Maintainer field if self.Cnf.get("Dinstall::OverrideMaintainer"): - Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"]; - Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"]; + Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"] + Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"] - Subst["__REJECT_MESSAGE__"] = reject_message; - Subst["__SOURCE__"] = changes.get("source", "Unknown"); - Subst["__VERSION__"] = changes.get("version", "Unknown"); + Subst["__REJECT_MESSAGE__"] = reject_message + Subst["__SOURCE__"] = changes.get("source", "Unknown") + Subst["__VERSION__"] = changes.get("version", "Unknown") ########################################################################### def build_summaries(self): - changes = self.pkg.changes; - files = self.pkg.files; + changes = self.pkg.changes + files = self.pkg.files - byhand = summary = new = ""; + byhand = summary = new = "" # changes["distribution"] may not exist in corner cases # (e.g. unreadable changes files) if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType): - changes["distribution"] = {}; + changes["distribution"] = {} - file_keys = files.keys(); - file_keys.sort(); + file_keys = files.keys() + file_keys.sort() for file in file_keys: if files[file].has_key("byhand"): byhand = 1 @@ -264,152 +264,152 @@ class Katie: summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"]) if files[file]["type"] == "deb": deb_fh = utils.open_file(file) - summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n'; + summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n' deb_fh.close() else: files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"]) destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file summary += file + "\n to " + destination + "\n" - short_summary = summary; + short_summary = summary # This is for direport's benefit... - f = re_fdnic.sub("\n .\n", changes.get("changes","")); + f = re_fdnic.sub("\n .\n", changes.get("changes","")) if byhand or new: - summary += "Changes: " + f; + summary += "Changes: " + f summary += self.announce(short_summary, 0) - return (summary, short_summary); + return (summary, short_summary) ########################################################################### def close_bugs (self, summary, action): - changes = self.pkg.changes; - Subst = self.Subst; - Cnf = self.Cnf; + changes = self.pkg.changes + Subst = self.Subst + Cnf = self.Cnf - bugs = changes["closes"].keys(); + bugs = changes["closes"].keys() if not bugs: - return summary; + return summary - bugs.sort(); + bugs.sort() if not self.nmu.is_an_nmu(self.pkg): if changes["distribution"].has_key("experimental"): # tag bugs as fixed-in-experimental for uploads to experimental - summary += "Setting bugs to severity fixed: "; - control_message = ""; + summary += "Setting bugs to severity fixed: " + control_message = "" for bug in bugs: - summary += "%s " % (bug); - control_message += "tag %s + fixed-in-experimental\n" % (bug); + summary += "%s " % (bug) + control_message += "tag %s + fixed-in-experimental\n" % (bug) if action and control_message != "": - Subst["__CONTROL_MESSAGE__"] = control_message; - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-experimental-fixed"); - utils.send_mail (mail_message); + Subst["__CONTROL_MESSAGE__"] = control_message + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-experimental-fixed") + utils.send_mail (mail_message) if action: - self.Logger.log(["setting bugs to fixed"]+bugs); + self.Logger.log(["setting bugs to fixed"]+bugs) else: - summary += "Closing bugs: "; + summary += "Closing bugs: " for bug in bugs: - summary += "%s " % (bug); + summary += "%s " % (bug) if action: - Subst["__BUG_NUMBER__"] = bug; + Subst["__BUG_NUMBER__"] = bug if changes["distribution"].has_key("stable"): Subst["__STABLE_WARNING__"] = """ Note that this package is not part of the released stable Debian distribution. It may have dependencies on other unreleased software, or other instabilities. Please take care if you wish to install it. The update will eventually make its way into the next released Debian -distribution."""; +distribution.""" else: - Subst["__STABLE_WARNING__"] = ""; - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close"); - utils.send_mail (mail_message); + Subst["__STABLE_WARNING__"] = "" + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close") + utils.send_mail (mail_message) if action: - self.Logger.log(["closing bugs"]+bugs); + self.Logger.log(["closing bugs"]+bugs) else: # NMU - summary += "Setting bugs to severity fixed: "; - control_message = ""; + summary += "Setting bugs to severity fixed: " + control_message = "" for bug in bugs: - summary += "%s " % (bug); - control_message += "tag %s + fixed\n" % (bug); + summary += "%s " % (bug) + control_message += "tag %s + fixed\n" % (bug) if action and control_message != "": - Subst["__CONTROL_MESSAGE__"] = control_message; - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed"); - utils.send_mail (mail_message); + Subst["__CONTROL_MESSAGE__"] = control_message + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed") + utils.send_mail (mail_message) if action: - self.Logger.log(["setting bugs to fixed"]+bugs); - summary += "\n"; - return summary; + self.Logger.log(["setting bugs to fixed"]+bugs) + summary += "\n" + return summary ########################################################################### def announce (self, short_summary, action): - Subst = self.Subst; - Cnf = self.Cnf; - changes = self.pkg.changes; + Subst = self.Subst + Cnf = self.Cnf + changes = self.pkg.changes # Only do announcements for source uploads with a recent dpkg-dev installed if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"): - return ""; + return "" - lists_done = {}; - summary = ""; - Subst["__SHORT_SUMMARY__"] = short_summary; + lists_done = {} + summary = "" + Subst["__SHORT_SUMMARY__"] = short_summary for dist in changes["distribution"].keys(): - list = Cnf.Find("Suite::%s::Announce" % (dist)); + list = Cnf.Find("Suite::%s::Announce" % (dist)) if list == "" or lists_done.has_key(list): - continue; - lists_done[list] = 1; - summary += "Announcing to %s\n" % (list); + continue + lists_done[list] = 1 + summary += "Announcing to %s\n" % (list) if action: - Subst["__ANNOUNCE_LIST_ADDRESS__"] = list; + Subst["__ANNOUNCE_LIST_ADDRESS__"] = list if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"): - Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]); - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce"); - utils.send_mail (mail_message); + Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]) + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce") + utils.send_mail (mail_message) if Cnf.FindB("Dinstall::CloseBugs"): - summary = self.close_bugs(summary, action); + summary = self.close_bugs(summary, action) - return summary; + return summary ########################################################################### def accept (self, summary, short_summary): - Cnf = self.Cnf; - Subst = self.Subst; - files = self.pkg.files; - changes = self.pkg.changes; - changes_file = self.pkg.changes_file; - dsc = self.pkg.dsc; + Cnf = self.Cnf + Subst = self.Subst + files = self.pkg.files + changes = self.pkg.changes + changes_file = self.pkg.changes_file + dsc = self.pkg.dsc print "Accepting." - self.Logger.log(["Accepting changes",changes_file]); + self.Logger.log(["Accepting changes",changes_file]) - self.dump_vars(Cnf["Dir::Queue::Accepted"]); + self.dump_vars(Cnf["Dir::Queue::Accepted"]) # Move all the files into the accepted directory - utils.move(changes_file, Cnf["Dir::Queue::Accepted"]); - file_keys = files.keys(); + utils.move(changes_file, Cnf["Dir::Queue::Accepted"]) + file_keys = files.keys() for file in file_keys: - utils.move(file, Cnf["Dir::Queue::Accepted"]); + utils.move(file, Cnf["Dir::Queue::Accepted"]) self.accept_bytes += float(files[file]["size"]) - self.accept_count += 1; + self.accept_count += 1 # Send accept mail, announce to lists, close bugs and check for # override disparities if not Cnf["Dinstall::Options::No-Mail"]: - Subst["__SUITE__"] = ""; - Subst["__SUMMARY__"] = summary; - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted"); + Subst["__SUITE__"] = "" + Subst["__SUMMARY__"] = summary + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted") utils.send_mail(mail_message) self.announce(short_summary, 1) @@ -425,29 +425,29 @@ distribution."""; dsc.has_key("bts changelog"): temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], - dotprefix=1, perms=0644); - version_history = utils.open_file(temp_filename, 'w'); - version_history.write(dsc["bts changelog"]); - version_history.close(); + dotprefix=1, perms=0644) + version_history = utils.open_file(temp_filename, 'w') + version_history.write(dsc["bts changelog"]) + version_history.close() filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"], - changes_file[:-8]+".versions"); - os.rename(temp_filename, filename); + changes_file[:-8]+".versions") + os.rename(temp_filename, filename) # Write out the binary -> source mapping. temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], - dotprefix=1, perms=0644); - debinfo = utils.open_file(temp_filename, 'w'); + dotprefix=1, perms=0644) + debinfo = utils.open_file(temp_filename, 'w') for file in file_keys: - f = files[file]; + f = files[file] if f["type"] == "deb": line = " ".join([f["package"], f["version"], f["architecture"], f["source package"], - f["source version"]]); - debinfo.write(line+"\n"); - debinfo.close(); + f["source version"]]) + debinfo.write(line+"\n") + debinfo.close() filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"], - changes_file[:-8]+".debinfo"); - os.rename(temp_filename, filename); + changes_file[:-8]+".debinfo") + os.rename(temp_filename, filename) self.queue_build("accepted", Cnf["Dir::Queue::Accepted"]) @@ -465,57 +465,57 @@ distribution."""; ## Special support to enable clean auto-building of queued packages queue_id = db_access.get_or_set_queue_id(queue) - self.projectB.query("BEGIN WORK"); + self.projectB.query("BEGIN WORK") for suite in changes["distribution"].keys(): if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"): - continue; - suite_id = db_access.get_suite_id(suite); - dest_dir = Cnf["Dir::QueueBuild"]; + continue + suite_id = db_access.get_suite_id(suite) + dest_dir = Cnf["Dir::QueueBuild"] if Cnf.FindB("Dinstall::SecurityQueueBuild"): - dest_dir = os.path.join(dest_dir, suite); + dest_dir = os.path.join(dest_dir, suite) for file in file_keys: - src = os.path.join(path, file); - dest = os.path.join(dest_dir, file); + src = os.path.join(path, file) + dest = os.path.join(dest_dir, file) if Cnf.FindB("Dinstall::SecurityQueueBuild"): # Copy it since the original won't be readable by www-data - utils.copy(src, dest); + utils.copy(src, dest) else: # Create a symlink to it - os.symlink(src, dest); + os.symlink(src, dest) # Add it to the list of packages for later processing by apt-ftparchive - self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest)); + self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest)) # If the .orig.tar.gz is in the pool, create a symlink to # it (if one doesn't already exist) if self.pkg.orig_tar_id: # Determine the .orig.tar.gz file name for dsc_file in self.pkg.dsc_files.keys(): if dsc_file.endswith(".orig.tar.gz"): - filename = dsc_file; - dest = os.path.join(dest_dir, filename); + filename = dsc_file + dest = os.path.join(dest_dir, filename) # If it doesn't exist, create a symlink if not os.path.exists(dest): # Find the .orig.tar.gz in the pool - q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id)); - ql = q.getresult(); + q = self.projectB.query("SELECT l.path, f.filename from location l, files f WHERE f.id = %s and f.location = l.id" % (self.pkg.orig_tar_id)) + ql = q.getresult() if not ql: - utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id)); - src = os.path.join(ql[0][0], ql[0][1]); - os.symlink(src, dest); + utils.fubar("[INTERNAL ERROR] Couldn't find id %s in files table." % (self.pkg.orig_tar_id)) + src = os.path.join(ql[0][0], ql[0][1]) + os.symlink(src, dest) # Add it to the list of packages for later processing by apt-ftparchive - self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest)); + self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest)) # if it does, update things to ensure it's not removed prematurely else: - self.projectB.query("UPDATE queue_build SET in_queue = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id)); + self.projectB.query("UPDATE queue_build SET in_queue = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id)) - self.projectB.query("COMMIT WORK"); + self.projectB.query("COMMIT WORK") ########################################################################### def check_override (self): - Subst = self.Subst; - changes = self.pkg.changes; - files = self.pkg.files; - Cnf = self.Cnf; + Subst = self.Subst + changes = self.pkg.changes + files = self.pkg.files + Cnf = self.Cnf # Abandon the check if: # a) it's a non-sourceful upload @@ -524,31 +524,31 @@ distribution."""; if not changes["architecture"].has_key("source") or \ not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \ Cnf["Dinstall::Options::No-Mail"]: - return; + return - summary = ""; - file_keys = files.keys(); - file_keys.sort(); + summary = "" + file_keys = files.keys() + file_keys.sort() for file in file_keys: if not files[file].has_key("new") and files[file]["type"] == "deb": - section = files[file]["section"]; - override_section = files[file]["override section"]; + section = files[file]["section"] + override_section = files[file]["override section"] if section.lower() != override_section.lower() and section != "-": # Ignore this; it's a common mistake and not worth whining about if section.lower() == "non-us/main" and override_section.lower() == "non-us": - continue; - summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section); - priority = files[file]["priority"]; - override_priority = files[file]["override priority"]; + continue + summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section) + priority = files[file]["priority"] + override_priority = files[file]["override priority"] if priority != override_priority and priority != "-": - summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority); + summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority) if summary == "": - return; + return - Subst["__SUMMARY__"] = summary; - mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity"); - utils.send_mail(mail_message); + Subst["__SUMMARY__"] = summary + mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity") + utils.send_mail(mail_message) ########################################################################### @@ -563,33 +563,33 @@ distribution."""; for file in files: # Skip any files which don't exist or which we don't have permission to copy. if os.access(file,os.R_OK) == 0: - continue; - dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file); + continue + dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file) try: - dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) except OSError, e: # File exists? Let's try and move it to the morgue if errno.errorcode[e.errno] == 'EEXIST': - morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file); + morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file) try: - morgue_file = utils.find_next_free(morgue_file); + morgue_file = utils.find_next_free(morgue_file) except utils.tried_too_hard_exc: # Something's either gone badly Pete Tong, or # someone is trying to exploit us. - utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file)); - return; - utils.move(dest_file, morgue_file, perms=0660); + utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file)) + return + utils.move(dest_file, morgue_file, perms=0660) try: - dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) except OSError, e: # Likewise - utils.warn("**WARNING** failed to claim %s in the reject directory." % (file)); - return; + utils.warn("**WARNING** failed to claim %s in the reject directory." % (file)) + return else: - raise; + raise # If we got here, we own the destination file, so we can # safely overwrite it. - utils.move(file, dest_file, 1, perms=0660); + utils.move(file, dest_file, 1, perms=0660) os.close(dest_fd) ########################################################################### @@ -598,74 +598,74 @@ distribution."""; # If we weren't given a manual rejection message, spawn an # editor so the user can add one in... if manual and not reject_message: - temp_filename = utils.temp_filename(); + temp_filename = utils.temp_filename() editor = os.environ.get("EDITOR","vi") - answer = 'E'; + answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - temp_fh = utils.open_file(temp_filename); - reject_message = "".join(temp_fh.readlines()); - temp_fh.close(); - print "Reject message:"; - print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1); + temp_fh = utils.open_file(temp_filename) + reject_message = "".join(temp_fh.readlines()) + temp_fh.close() + print "Reject message:" + print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1) prompt = "[R]eject, Edit, Abandon, Quit ?" - answer = "XXX"; + answer = "XXX" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = re_default_answer.search(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - os.unlink(temp_filename); + answer = m.group(1) + answer = answer[:1].upper() + os.unlink(temp_filename) if answer == 'A': - return 1; + return 1 elif answer == 'Q': - sys.exit(0); + sys.exit(0) print "Rejecting.\n" - Cnf = self.Cnf; - Subst = self.Subst; - pkg = self.pkg; + Cnf = self.Cnf + Subst = self.Subst + pkg = self.pkg - reason_filename = pkg.changes_file[:-8] + ".reason"; - reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename; + reason_filename = pkg.changes_file[:-8] + ".reason" + reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename # Move all the files into the reject directory - reject_files = pkg.files.keys() + [pkg.changes_file]; - self.force_reject(reject_files); + reject_files = pkg.files.keys() + [pkg.changes_file] + self.force_reject(reject_files) # If we fail here someone is probably trying to exploit the race # so let's just raise an exception ... if os.path.exists(reason_filename): - os.unlink(reason_filename); - reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + os.unlink(reason_filename) + reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) if not manual: - Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]; - Subst["__MANUAL_REJECT_MESSAGE__"] = ""; - Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)"; - os.write(reason_fd, reject_message); - reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected"); + Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] + Subst["__MANUAL_REJECT_MESSAGE__"] = "" + Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)" + os.write(reason_fd, reject_message) + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected") else: # Build up the rejection email - user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]); + user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]) - Subst["__REJECTOR_ADDRESS__"] = user_email_address; - Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message; - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; - reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected"); + Subst["__REJECTOR_ADDRESS__"] = user_email_address + Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message + Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected") # Write the rejection email out as the .reason file - os.write(reason_fd, reject_mail_message); + os.write(reason_fd, reject_mail_message) os.close(reason_fd) # Send the rejection mail if appropriate if not Cnf["Dinstall::Options::No-Mail"]: - utils.send_mail(reject_mail_message); + utils.send_mail(reject_mail_message) - self.Logger.log(["rejected", pkg.changes_file]); - return 0; + self.Logger.log(["rejected", pkg.changes_file]) + return 0 ################################################################################ @@ -695,11 +695,11 @@ distribution."""; if x[1] in s and x[0] not in s: s.append(x[0]) - que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR ")); + que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR ")) q = self.projectB.query(que) # Reduce the query results to a list of version numbers - ql = map(lambda x: x[0], q.getresult()); + ql = map(lambda x: x[0], q.getresult()) # Try (1) if source_version in ql: @@ -718,45 +718,45 @@ distribution."""; ################################################################################ def in_override_p (self, package, component, suite, binary_type, file): - files = self.pkg.files; + files = self.pkg.files if binary_type == "": # must be source - type = "dsc"; + type = "dsc" else: - type = binary_type; + type = binary_type # Override suite name; used for example with proposed-updates if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "": - suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)]; + suite = self.Cnf["Suite::%s::OverrideSuite" % (suite)] # Avoid on unknown distributions - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id == -1: - return None; - component_id = db_access.get_component_id(component); - type_id = db_access.get_override_type_id(type); + return None + component_id = db_access.get_component_id(component) + type_id = db_access.get_override_type_id(type) # FIXME: nasty non-US speficic hack if component.lower().startswith("non-us/"): - component = component[7:]; + component = component[7:] q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" - % (package, suite_id, component_id, type_id)); - result = q.getresult(); + % (package, suite_id, component_id, type_id)) + result = q.getresult() # If checking for a source package fall back on the binary override type if type == "dsc" and not result: - deb_type_id = db_access.get_override_type_id("deb"); - udeb_type_id = db_access.get_override_type_id("udeb"); + deb_type_id = db_access.get_override_type_id("deb") + udeb_type_id = db_access.get_override_type_id("udeb") q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id" - % (package, suite_id, component_id, deb_type_id, udeb_type_id)); - result = q.getresult(); + % (package, suite_id, component_id, deb_type_id, udeb_type_id)) + result = q.getresult() # Remember the section and priority so we can check them later if appropriate if result: - files[file]["override section"] = result[0][0]; - files[file]["override priority"] = result[0][1]; + files[file]["override section"] = result[0][0] + files[file]["override priority"] = result[0][1] - return result; + return result ################################################################################ @@ -765,8 +765,8 @@ distribution."""; # Unlike other rejects we add new lines first to avoid trailing # new lines when this message is passed back up to a caller. if self.reject_message: - self.reject_message += "\n"; - self.reject_message += prefix + str; + self.reject_message += "\n" + self.reject_message += prefix + str ################################################################################ @@ -788,17 +788,17 @@ distribution."""; # Check versions for each target suite for target_suite in self.pkg.changes["distribution"].keys(): - must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite))); - must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite))); + must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite))) + must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite))) # Enforce "must be newer than target suite" even if conffile omits it if target_suite not in must_be_newer_than: - must_be_newer_than.append(target_suite); + must_be_newer_than.append(target_suite) for entry in query_result: - existent_version = entry[0]; - suite = entry[1]; + existent_version = entry[0] + suite = entry[1] if suite in must_be_newer_than and \ apt_pkg.VersionCompare(new_version, existent_version) < 1: - self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)); + self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)) if suite in must_be_older_than and \ apt_pkg.VersionCompare(new_version, existent_version) > -1: ch = self.pkg.changes @@ -847,8 +847,8 @@ distribution."""; ################################################################################ def check_binary_against_db(self, file): - self.reject_message = ""; - files = self.pkg.files; + self.reject_message = "" + files = self.pkg.files # Ensure version is sane q = self.projectB.query(""" @@ -857,8 +857,8 @@ SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su, WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id""" % (files[file]["package"], - files[file]["architecture"])); - self.cross_suite_version_check(q.getresult(), file, files[file]["version"]); + files[file]["architecture"])) + self.cross_suite_version_check(q.getresult(), file, files[file]["version"]) # Check for any existing copies of the file q = self.projectB.query(""" @@ -869,23 +869,23 @@ SELECT b.id FROM binaries b, architecture a files[file]["version"], files[file]["architecture"])) if q.getresult(): - self.reject("%s: can not overwrite existing copy already in the archive." % (file)); + self.reject("%s: can not overwrite existing copy already in the archive." % (file)) - return self.reject_message; + return self.reject_message ################################################################################ def check_source_against_db(self, file): - self.reject_message = ""; - dsc = self.pkg.dsc; + self.reject_message = "" + dsc = self.pkg.dsc # Ensure version is sane q = self.projectB.query(""" SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su - WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source"))); - self.cross_suite_version_check(q.getresult(), file, dsc.get("version")); + WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source"))) + self.cross_suite_version_check(q.getresult(), file, dsc.get("version")) - return self.reject_message; + return self.reject_message ################################################################################ @@ -898,28 +898,28 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su # **WARNING** def check_dsc_against_db(self, file): - self.reject_message = ""; - files = self.pkg.files; - dsc_files = self.pkg.dsc_files; - legacy_source_untouchable = self.pkg.legacy_source_untouchable; - self.pkg.orig_tar_gz = None; + self.reject_message = "" + files = self.pkg.files + dsc_files = self.pkg.dsc_files + legacy_source_untouchable = self.pkg.legacy_source_untouchable + self.pkg.orig_tar_gz = None # Try and find all files mentioned in the .dsc. This has # to work harder to cope with the multiple possible # locations of an .orig.tar.gz. for dsc_file in dsc_files.keys(): - found = None; + found = None if files.has_key(dsc_file): - actual_md5 = files[dsc_file]["md5sum"]; - actual_size = int(files[dsc_file]["size"]); + actual_md5 = files[dsc_file]["md5sum"] + actual_size = int(files[dsc_file]["size"]) found = "%s in incoming" % (dsc_file) # Check the file does not already exist in the archive - q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file)); - ql = q.getresult(); + q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file)) + ql = q.getresult() # Strip out anything that isn't '%s' or '/%s$' for i in ql: if i[3] != dsc_file and i[3][-(len(dsc_file)+1):] != '/'+dsc_file: - ql.remove(i); + ql.remove(i) # "[katie] has not broken them. [katie] has fixed a # brokenness. Your crappy hack exploited a bug in @@ -932,27 +932,27 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su if ql: # Ignore exact matches for .orig.tar.gz - match = 0; + match = 0 if dsc_file.endswith(".orig.tar.gz"): for i in ql: if files.has_key(dsc_file) and \ int(files[dsc_file]["size"]) == int(i[0]) and \ files[dsc_file]["md5sum"] == i[1]: - self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: "); - del files[dsc_file]; - self.pkg.orig_tar_gz = i[2] + i[3]; - match = 1; + self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ") + del files[dsc_file] + self.pkg.orig_tar_gz = i[2] + i[3] + match = 1 if not match: - self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file)); + self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file)) elif dsc_file.endswith(".orig.tar.gz"): # Check in the pool - q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file)); - ql = q.getresult(); + q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file)) + ql = q.getresult() # Strip out anything that isn't '%s' or '/%s$' for i in ql: if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file: - ql.remove(i); + ql.remove(i) if ql: # Unfortunately, we may get more than one match here if, @@ -963,67 +963,67 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su if len(ql) > 1: for i in ql: - old_file = i[0] + i[1]; + old_file = i[0] + i[1] old_file_fh = utils.open_file(old_file) - actual_md5 = apt_pkg.md5sum(old_file_fh); + actual_md5 = apt_pkg.md5sum(old_file_fh) old_file_fh.close() - actual_size = os.stat(old_file)[stat.ST_SIZE]; + actual_size = os.stat(old_file)[stat.ST_SIZE] if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]): - x = i; + x = i else: - legacy_source_untouchable[i[3]] = ""; + legacy_source_untouchable[i[3]] = "" - old_file = x[0] + x[1]; + old_file = x[0] + x[1] old_file_fh = utils.open_file(old_file) - actual_md5 = apt_pkg.md5sum(old_file_fh); + actual_md5 = apt_pkg.md5sum(old_file_fh) old_file_fh.close() - actual_size = os.stat(old_file)[stat.ST_SIZE]; - found = old_file; - suite_type = x[2]; + actual_size = os.stat(old_file)[stat.ST_SIZE] + found = old_file + suite_type = x[2] dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install() # See install() in katie... - self.pkg.orig_tar_id = x[3]; - self.pkg.orig_tar_gz = old_file; + self.pkg.orig_tar_id = x[3] + self.pkg.orig_tar_gz = old_file if suite_type == "legacy" or suite_type == "legacy-mixed": - self.pkg.orig_tar_location = "legacy"; + self.pkg.orig_tar_location = "legacy" else: - self.pkg.orig_tar_location = x[4]; + self.pkg.orig_tar_location = x[4] else: # Not there? Check the queue directories... - in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file); + in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file) # See process_it() in jennifer for explanation of this if os.path.exists(in_unchecked): - return (self.reject_message, in_unchecked); + return (self.reject_message, in_unchecked) else: for dir in [ "Accepted", "New", "Byhand" ]: - in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file); + in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file) if os.path.exists(in_otherdir): in_otherdir_fh = utils.open_file(in_otherdir) - actual_md5 = apt_pkg.md5sum(in_otherdir_fh); + actual_md5 = apt_pkg.md5sum(in_otherdir_fh) in_otherdir_fh.close() - actual_size = os.stat(in_otherdir)[stat.ST_SIZE]; - found = in_otherdir; - self.pkg.orig_tar_gz = in_otherdir; + actual_size = os.stat(in_otherdir)[stat.ST_SIZE] + found = in_otherdir + self.pkg.orig_tar_gz = in_otherdir if not found: - self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file)); - self.pkg.orig_tar_gz = -1; - continue; + self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file)) + self.pkg.orig_tar_gz = -1 + continue else: - self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file)); - continue; + self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file)) + continue if actual_md5 != dsc_files[dsc_file]["md5sum"]: - self.reject("md5sum for %s doesn't match %s." % (found, file)); + self.reject("md5sum for %s doesn't match %s." % (found, file)) if actual_size != int(dsc_files[dsc_file]["size"]): - self.reject("size for %s doesn't match %s." % (found, file)); + self.reject("size for %s doesn't match %s." % (found, file)) - return (self.reject_message, None); + return (self.reject_message, None) def do_query(self, q): - sys.stderr.write("query: \"%s\" ... " % (q)); - before = time.time(); - r = self.projectB.query(q); - time_diff = time.time()-before; - sys.stderr.write("took %.3f seconds.\n" % (time_diff)); - return r; + sys.stderr.write("query: \"%s\" ... " % (q)) + before = time.time() + r = self.projectB.query(q) + time_diff = time.time()-before + sys.stderr.write("took %.3f seconds.\n" % (time_diff)) + return r diff --git a/dak/lib/utils.py b/dak/lib/utils.py index 580ceda8..0c1a5ff8 100644 --- a/dak/lib/utils.py +++ b/dak/lib/utils.py @@ -23,9 +23,9 @@ ################################################################################ import codecs, commands, email.Header, os, pwd, re, select, socket, shutil, \ - string, sys, tempfile, traceback; -import apt_pkg; -import db_access; + string, sys, tempfile, traceback +import apt_pkg +import db_access ################################################################################ @@ -34,34 +34,34 @@ re_no_epoch = re.compile(r"^\d+\:") re_no_revision = re.compile(r"-[^-]+$") re_arch_from_filename = re.compile(r"/binary-[^/]+/") re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)") -re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$"); -re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$"); +re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$") +re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$") -re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)"); -re_multi_line_field = re.compile(r"^\s(.*)"); -re_taint_free = re.compile(r"^[-+~/\.\w]+$"); +re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)") +re_multi_line_field = re.compile(r"^\s(.*)") +re_taint_free = re.compile(r"^[-+~/\.\w]+$") -re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>"); +re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>") -changes_parse_error_exc = "Can't parse line in .changes file"; -invalid_dsc_format_exc = "Invalid .dsc file"; -nk_format_exc = "Unknown Format: in .changes file"; -no_files_exc = "No Files: field in .dsc or .changes file."; -cant_open_exc = "Can't open file"; -unknown_hostname_exc = "Unknown hostname"; +changes_parse_error_exc = "Can't parse line in .changes file" +invalid_dsc_format_exc = "Invalid .dsc file" +nk_format_exc = "Unknown Format: in .changes file" +no_files_exc = "No Files: field in .dsc or .changes file." +cant_open_exc = "Can't open file" +unknown_hostname_exc = "Unknown hostname" cant_overwrite_exc = "Permission denied; can't overwrite existent file." -file_exists_exc = "Destination file exists"; -sendmail_failed_exc = "Sendmail invocation failed"; -tried_too_hard_exc = "Tried too hard to find a free filename."; +file_exists_exc = "Destination file exists" +sendmail_failed_exc = "Sendmail invocation failed" +tried_too_hard_exc = "Tried too hard to find a free filename." -default_config = "/etc/katie/katie.conf"; -default_apt_config = "/etc/katie/apt.conf"; +default_config = "/etc/katie/katie.conf" +default_apt_config = "/etc/katie/apt.conf" ################################################################################ class Error(Exception): """Base class for exceptions in this module.""" - pass; + pass class ParseMaintError(Error): """Exception raised for errors in parsing a maintainer field. @@ -71,68 +71,68 @@ class ParseMaintError(Error): """ def __init__(self, message): - self.args = message,; - self.message = message; + self.args = message, + self.message = message ################################################################################ def open_file(filename, mode='r'): try: - f = open(filename, mode); + f = open(filename, mode) except IOError: - raise cant_open_exc, filename; + raise cant_open_exc, filename return f ################################################################################ def our_raw_input(prompt=""): if prompt: - sys.stdout.write(prompt); - sys.stdout.flush(); + sys.stdout.write(prompt) + sys.stdout.flush() try: - ret = raw_input(); - return ret; + ret = raw_input() + return ret except EOFError: - sys.stderr.write("\nUser interrupt (^D).\n"); - raise SystemExit; + sys.stderr.write("\nUser interrupt (^D).\n") + raise SystemExit ################################################################################ def str_isnum (s): for c in s: if c not in string.digits: - return 0; - return 1; + return 0 + return 1 ################################################################################ def extract_component_from_section(section): - component = ""; + component = "" if section.find('/') != -1: - component = section.split('/')[0]; + component = section.split('/')[0] if component.lower() == "non-us" and section.find('/') != -1: - s = component + '/' + section.split('/')[1]; + s = component + '/' + section.split('/')[1] if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs - component = s; + component = s if section.lower() == "non-us": - component = "non-US/main"; + component = "non-US/main" # non-US prefix is case insensitive if component.lower()[:6] == "non-us": - component = "non-US"+component[6:]; + component = "non-US"+component[6:] # Expand default component if component == "": if Cnf.has_key("Component::%s" % section): - component = section; + component = section else: - component = "main"; + component = "main" elif component == "non-US": - component = "non-US/main"; + component = "non-US/main" - return (section, component); + return (section, component) ################################################################################ @@ -157,127 +157,127 @@ The rules for (signing_rules == 1)-mode are: "-----BEGIN PGP SIGNATURE-----". """ - error = ""; - changes = {}; + error = "" + changes = {} - changes_in = open_file(filename); - lines = changes_in.readlines(); + changes_in = open_file(filename) + lines = changes_in.readlines() if not lines: - raise changes_parse_error_exc, "[Empty changes file]"; + raise changes_parse_error_exc, "[Empty changes file]" # Reindex by line number so we can easily verify the format of # .dsc files... - index = 0; - indexed_lines = {}; + index = 0 + indexed_lines = {} for line in lines: - index += 1; - indexed_lines[index] = line[:-1]; + index += 1 + indexed_lines[index] = line[:-1] - inside_signature = 0; + inside_signature = 0 - num_of_lines = len(indexed_lines.keys()); - index = 0; - first = -1; + num_of_lines = len(indexed_lines.keys()) + index = 0 + first = -1 while index < num_of_lines: - index += 1; - line = indexed_lines[index]; + index += 1 + line = indexed_lines[index] if line == "": if signing_rules == 1: - index += 1; + index += 1 if index > num_of_lines: - raise invalid_dsc_format_exc, index; - line = indexed_lines[index]; + raise invalid_dsc_format_exc, index + line = indexed_lines[index] if not line.startswith("-----BEGIN PGP SIGNATURE"): - raise invalid_dsc_format_exc, index; - inside_signature = 0; - break; + raise invalid_dsc_format_exc, index + inside_signature = 0 + break else: - continue; + continue if line.startswith("-----BEGIN PGP SIGNATURE"): - break; + break if line.startswith("-----BEGIN PGP SIGNED MESSAGE"): - inside_signature = 1; + inside_signature = 1 if signing_rules == 1: while index < num_of_lines and line != "": - index += 1; - line = indexed_lines[index]; - continue; + index += 1 + line = indexed_lines[index] + continue # If we're not inside the signed data, don't process anything if signing_rules >= 0 and not inside_signature: - continue; - slf = re_single_line_field.match(line); + continue + slf = re_single_line_field.match(line) if slf: - field = slf.groups()[0].lower(); - changes[field] = slf.groups()[1]; - first = 1; - continue; + field = slf.groups()[0].lower() + changes[field] = slf.groups()[1] + first = 1 + continue if line == " .": - changes[field] += '\n'; - continue; - mlf = re_multi_line_field.match(line); + changes[field] += '\n' + continue + mlf = re_multi_line_field.match(line) if mlf: if first == -1: - raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line); + raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line) if first == 1 and changes[field] != "": - changes[field] += '\n'; - first = 0; - changes[field] += mlf.groups()[0] + '\n'; - continue; - error += line; + changes[field] += '\n' + first = 0 + changes[field] += mlf.groups()[0] + '\n' + continue + error += line if signing_rules == 1 and inside_signature: - raise invalid_dsc_format_exc, index; + raise invalid_dsc_format_exc, index - changes_in.close(); - changes["filecontents"] = "".join(lines); + changes_in.close() + changes["filecontents"] = "".join(lines) if error: - raise changes_parse_error_exc, error; + raise changes_parse_error_exc, error - return changes; + return changes ################################################################################ # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl def build_file_list(changes, is_a_dsc=0): - files = {}; + files = {} # Make sure we have a Files: field to parse... if not changes.has_key("files"): - raise no_files_exc; + raise no_files_exc # Make sure we recognise the format of the Files: field - format = changes.get("format", ""); + format = changes.get("format", "") if format != "": - format = float(format); + format = float(format) if not is_a_dsc and (format < 1.5 or format > 2.0): - raise nk_format_exc, format; + raise nk_format_exc, format # Parse each entry/line: for i in changes["files"].split('\n'): if not i: - break; - s = i.split(); - section = priority = ""; + break + s = i.split() + section = priority = "" try: if is_a_dsc: - (md5, size, name) = s; + (md5, size, name) = s else: - (md5, size, section, priority, name) = s; + (md5, size, section, priority, name) = s except ValueError: - raise changes_parse_error_exc, i; + raise changes_parse_error_exc, i if section == "": - section = "-"; + section = "-" if priority == "": - priority = "-"; + priority = "-" - (section, component) = extract_component_from_section(section); + (section, component) = extract_component_from_section(section) files[name] = Dict(md5sum=md5, size=size, section=section, - priority=priority, component=component); + priority=priority, component=component) return files @@ -287,27 +287,27 @@ def force_to_utf8(s): """Forces a string to UTF-8. If the string isn't already UTF-8, it's assumed to be ISO-8859-1.""" try: - unicode(s, 'utf-8'); - return s; + unicode(s, 'utf-8') + return s except UnicodeError: - latin1_s = unicode(s,'iso8859-1'); - return latin1_s.encode('utf-8'); + latin1_s = unicode(s,'iso8859-1') + return latin1_s.encode('utf-8') def rfc2047_encode(s): """Encodes a (header) string per RFC2047 if necessary. If the string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.""" try: codecs.lookup('ascii')[1](s) - return s; + return s except UnicodeError: - pass; + pass try: codecs.lookup('utf-8')[1](s) - h = email.Header.Header(s, 'utf-8', 998); - return str(h); + h = email.Header.Header(s, 'utf-8', 998) + return str(h) except UnicodeError: - h = email.Header.Header(s, 'iso-8859-1', 998); - return str(h); + h = email.Header.Header(s, 'iso-8859-1', 998) + return str(h) ################################################################################ @@ -327,38 +327,38 @@ contains '.' or ',' (as allowed by Debian policy), (1) and (2) are switched to 'email (name)' format.""" maintainer = maintainer.strip() if not maintainer: - return ('', '', '', ''); + return ('', '', '', '') if maintainer.find("<") == -1: - email = maintainer; - name = ""; + email = maintainer + name = "" elif (maintainer[0] == "<" and maintainer[-1:] == ">"): - email = maintainer[1:-1]; - name = ""; + email = maintainer[1:-1] + name = "" else: - m = re_parse_maintainer.match(maintainer); + m = re_parse_maintainer.match(maintainer) if not m: raise ParseMaintError, "Doesn't parse as a valid Maintainer field." - name = m.group(1); - email = m.group(2); + name = m.group(1) + email = m.group(2) # Get an RFC2047 compliant version of the name - rfc2047_name = rfc2047_encode(name); + rfc2047_name = rfc2047_encode(name) # Force the name to be UTF-8 - name = force_to_utf8(name); + name = force_to_utf8(name) if name.find(',') != -1 or name.find('.') != -1: - rfc822_maint = "%s (%s)" % (email, name); - rfc2047_maint = "%s (%s)" % (email, rfc2047_name); + rfc822_maint = "%s (%s)" % (email, name) + rfc2047_maint = "%s (%s)" % (email, rfc2047_name) else: - rfc822_maint = "%s <%s>" % (name, email); - rfc2047_maint = "%s <%s>" % (rfc2047_name, email); + rfc822_maint = "%s <%s>" % (name, email) + rfc2047_maint = "%s <%s>" % (rfc2047_name, email) if email.find("@") == -1 and email.find("buildd_") != 0: raise ParseMaintError, "No @ found in email address part." - return (rfc822_maint, rfc2047_maint, name, email); + return (rfc822_maint, rfc2047_maint, name, email) ################################################################################ @@ -366,27 +366,27 @@ switched to 'email (name)' format.""" def send_mail (message, filename=""): # If we've been passed a string dump it into a temporary file if message: - filename = tempfile.mktemp(); - fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700); - os.write (fd, message); - os.close (fd); + filename = tempfile.mktemp() + fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700) + os.write (fd, message) + os.close (fd) # Invoke sendmail - (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename)); + (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename)) if (result != 0): - raise sendmail_failed_exc, output; + raise sendmail_failed_exc, output # Clean up any temporary files if message: - os.unlink (filename); + os.unlink (filename) ################################################################################ def poolify (source, component): if component: - component += '/'; + component += '/' # FIXME: this is nasty - component = component.lower().replace("non-us/", "non-US/"); + component = component.lower().replace("non-us/", "non-US/") if source[:3] == "lib": return component + source[:4] + '/' + source + '/' else: @@ -396,39 +396,39 @@ def poolify (source, component): def move (src, dest, overwrite = 0, perms = 0664): if os.path.exists(dest) and os.path.isdir(dest): - dest_dir = dest; + dest_dir = dest else: - dest_dir = os.path.dirname(dest); + dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): - umask = os.umask(00000); - os.makedirs(dest_dir, 02775); - os.umask(umask); - #print "Moving %s to %s..." % (src, dest); + umask = os.umask(00000) + os.makedirs(dest_dir, 02775) + os.umask(umask) + #print "Moving %s to %s..." % (src, dest) if os.path.exists(dest) and os.path.isdir(dest): - dest += '/' + os.path.basename(src); + dest += '/' + os.path.basename(src) # Don't overwrite unless forced to if os.path.exists(dest): if not overwrite: - fubar("Can't move %s to %s - file already exists." % (src, dest)); + fubar("Can't move %s to %s - file already exists." % (src, dest)) else: if not os.access(dest, os.W_OK): - fubar("Can't move %s to %s - can't write to existing file." % (src, dest)); - shutil.copy2(src, dest); - os.chmod(dest, perms); - os.unlink(src); + fubar("Can't move %s to %s - can't write to existing file." % (src, dest)) + shutil.copy2(src, dest) + os.chmod(dest, perms) + os.unlink(src) def copy (src, dest, overwrite = 0, perms = 0664): if os.path.exists(dest) and os.path.isdir(dest): - dest_dir = dest; + dest_dir = dest else: - dest_dir = os.path.dirname(dest); + dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): - umask = os.umask(00000); - os.makedirs(dest_dir, 02775); - os.umask(umask); - #print "Copying %s to %s..." % (src, dest); + umask = os.umask(00000) + os.makedirs(dest_dir, 02775) + os.umask(umask) + #print "Copying %s to %s..." % (src, dest) if os.path.exists(dest) and os.path.isdir(dest): - dest += '/' + os.path.basename(src); + dest += '/' + os.path.basename(src) # Don't overwrite unless forced to if os.path.exists(dest): if not overwrite: @@ -436,32 +436,32 @@ def copy (src, dest, overwrite = 0, perms = 0664): else: if not os.access(dest, os.W_OK): raise cant_overwrite_exc - shutil.copy2(src, dest); - os.chmod(dest, perms); + shutil.copy2(src, dest) + os.chmod(dest, perms) ################################################################################ def where_am_i (): - res = socket.gethostbyaddr(socket.gethostname()); - database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname"); + res = socket.gethostbyaddr(socket.gethostname()) + database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname") if database_hostname: - return database_hostname; + return database_hostname else: - return res[0]; + return res[0] def which_conf_file (): - res = socket.gethostbyaddr(socket.gethostname()); + res = socket.gethostbyaddr(socket.gethostname()) if Cnf.get("Config::" + res[0] + "::KatieConfig"): return Cnf["Config::" + res[0] + "::KatieConfig"] else: - return default_config; + return default_config def which_apt_conf_file (): - res = socket.gethostbyaddr(socket.gethostname()); + res = socket.gethostbyaddr(socket.gethostname()) if Cnf.get("Config::" + res[0] + "::AptConfig"): return Cnf["Config::" + res[0] + "::AptConfig"] else: - return default_apt_config; + return default_apt_config ################################################################################ @@ -469,134 +469,134 @@ def which_apt_conf_file (): # (woefully incomplete) def regex_safe (s): - s = s.replace('+', '\\\\+'); - s = s.replace('.', '\\\\.'); + s = s.replace('+', '\\\\+') + s = s.replace('.', '\\\\.') return s ################################################################################ # Perform a substition of template def TemplateSubst(map, filename): - file = open_file(filename); - template = file.read(); + file = open_file(filename) + template = file.read() for x in map.keys(): - template = template.replace(x,map[x]); - file.close(); - return template; + template = template.replace(x,map[x]) + file.close() + return template ################################################################################ def fubar(msg, exit_code=1): - sys.stderr.write("E: %s\n" % (msg)); - sys.exit(exit_code); + sys.stderr.write("E: %s\n" % (msg)) + sys.exit(exit_code) def warn(msg): - sys.stderr.write("W: %s\n" % (msg)); + sys.stderr.write("W: %s\n" % (msg)) ################################################################################ # Returns the user name with a laughable attempt at rfc822 conformancy # (read: removing stray periods). def whoami (): - return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', ''); + return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '') ################################################################################ def size_type (c): - t = " B"; + t = " B" if c > 10240: - c = c / 1024; - t = " KB"; + c = c / 1024 + t = " KB" if c > 10240: - c = c / 1024; - t = " MB"; + c = c / 1024 + t = " MB" return ("%d%s" % (c, t)) ################################################################################ def cc_fix_changes (changes): - o = changes.get("architecture", ""); + o = changes.get("architecture", "") if o: - del changes["architecture"]; - changes["architecture"] = {}; + del changes["architecture"] + changes["architecture"] = {} for j in o.split(): - changes["architecture"][j] = 1; + changes["architecture"][j] = 1 # Sort by source name, source version, 'have source', and then by filename def changes_compare (a, b): try: - a_changes = parse_changes(a); + a_changes = parse_changes(a) except: - return -1; + return -1 try: - b_changes = parse_changes(b); + b_changes = parse_changes(b) except: - return 1; + return 1 - cc_fix_changes (a_changes); - cc_fix_changes (b_changes); + cc_fix_changes (a_changes) + cc_fix_changes (b_changes) # Sort by source name - a_source = a_changes.get("source"); - b_source = b_changes.get("source"); - q = cmp (a_source, b_source); + a_source = a_changes.get("source") + b_source = b_changes.get("source") + q = cmp (a_source, b_source) if q: - return q; + return q # Sort by source version - a_version = a_changes.get("version", "0"); - b_version = b_changes.get("version", "0"); - q = apt_pkg.VersionCompare(a_version, b_version); + a_version = a_changes.get("version", "0") + b_version = b_changes.get("version", "0") + q = apt_pkg.VersionCompare(a_version, b_version) if q: - return q; + return q # Sort by 'have source' - a_has_source = a_changes["architecture"].get("source"); - b_has_source = b_changes["architecture"].get("source"); + a_has_source = a_changes["architecture"].get("source") + b_has_source = b_changes["architecture"].get("source") if a_has_source and not b_has_source: - return -1; + return -1 elif b_has_source and not a_has_source: - return 1; + return 1 # Fall back to sort by filename - return cmp(a, b); + return cmp(a, b) ################################################################################ def find_next_free (dest, too_many=100): - extra = 0; - orig_dest = dest; + extra = 0 + orig_dest = dest while os.path.exists(dest) and extra < too_many: - dest = orig_dest + '.' + repr(extra); - extra += 1; + dest = orig_dest + '.' + repr(extra) + extra += 1 if extra >= too_many: - raise tried_too_hard_exc; - return dest; + raise tried_too_hard_exc + return dest ################################################################################ def result_join (original, sep = '\t'): - list = []; + list = [] for i in xrange(len(original)): if original[i] == None: - list.append(""); + list.append("") else: - list.append(original[i]); - return sep.join(list); + list.append(original[i]) + return sep.join(list) ################################################################################ def prefix_multi_line_string(str, prefix, include_blank_lines=0): - out = ""; + out = "" for line in str.split('\n'): - line = line.strip(); + line = line.strip() if line or include_blank_lines: - out += "%s%s\n" % (prefix, line); + out += "%s%s\n" % (prefix, line) # Strip trailing new line if out: - out = out[:-1]; - return out; + out = out[:-1] + return out ################################################################################ @@ -613,61 +613,61 @@ argument: o If 'require_changes' == 0, a warning is given and 'None' is returned. o If 'require_changes' == 1, a fatal error is raised. """ - error = None; + error = None orig_filename = filename if filename.endswith(".katie"): - filename = filename[:-6]+".changes"; + filename = filename[:-6]+".changes" if not filename.endswith(".changes"): - error = "invalid file type; not a changes file"; + error = "invalid file type; not a changes file" else: if not os.access(filename,os.R_OK): if os.path.exists(filename): - error = "permission denied"; + error = "permission denied" else: - error = "file not found"; + error = "file not found" if error: if require_changes == 1: - fubar("%s: %s." % (orig_filename, error)); + fubar("%s: %s." % (orig_filename, error)) elif require_changes == 0: - warn("Skipping %s - %s" % (orig_filename, error)); - return None; + warn("Skipping %s - %s" % (orig_filename, error)) + return None else: # We only care about the .katie file - return filename; + return filename else: - return filename; + return filename ################################################################################ def real_arch(arch): - return (arch != "source" and arch != "all"); + return (arch != "source" and arch != "all") ################################################################################ def join_with_commas_and(list): - if len(list) == 0: return "nothing"; - if len(list) == 1: return list[0]; - return ", ".join(list[:-1]) + " and " + list[-1]; + if len(list) == 0: return "nothing" + if len(list) == 1: return list[0] + return ", ".join(list[:-1]) + " and " + list[-1] ################################################################################ def pp_deps (deps): - pp_deps = []; + pp_deps = [] for atom in deps: - (pkg, version, constraint) = atom; + (pkg, version, constraint) = atom if constraint: - pp_dep = "%s (%s %s)" % (pkg, constraint, version); + pp_dep = "%s (%s %s)" % (pkg, constraint, version) else: - pp_dep = pkg; - pp_deps.append(pp_dep); - return " |".join(pp_deps); + pp_dep = pkg + pp_deps.append(pp_dep) + return " |".join(pp_deps) ################################################################################ def get_conf(): - return Cnf; + return Cnf ################################################################################ @@ -675,59 +675,59 @@ def get_conf(): def parse_args(Options): # Process suite if Options["Suite"]: - suite_ids_list = []; + suite_ids_list = [] for suite in split_args(Options["Suite"]): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id == -1: - warn("suite '%s' not recognised." % (suite)); + warn("suite '%s' not recognised." % (suite)) else: - suite_ids_list.append(suite_id); + suite_ids_list.append(suite_id) if suite_ids_list: - con_suites = "AND su.id IN (%s)" % ", ".join(map(str, suite_ids_list)); + con_suites = "AND su.id IN (%s)" % ", ".join(map(str, suite_ids_list)) else: - fubar("No valid suite given."); + fubar("No valid suite given.") else: - con_suites = ""; + con_suites = "" # Process component if Options["Component"]: - component_ids_list = []; + component_ids_list = [] for component in split_args(Options["Component"]): - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) if component_id == -1: - warn("component '%s' not recognised." % (component)); + warn("component '%s' not recognised." % (component)) else: - component_ids_list.append(component_id); + component_ids_list.append(component_id) if component_ids_list: - con_components = "AND c.id IN (%s)" % ", ".join(map(str, component_ids_list)); + con_components = "AND c.id IN (%s)" % ", ".join(map(str, component_ids_list)) else: - fubar("No valid component given."); + fubar("No valid component given.") else: - con_components = ""; + con_components = "" # Process architecture - con_architectures = ""; + con_architectures = "" if Options["Architecture"]: - arch_ids_list = []; - check_source = 0; + arch_ids_list = [] + check_source = 0 for architecture in split_args(Options["Architecture"]): if architecture == "source": - check_source = 1; + check_source = 1 else: - architecture_id = db_access.get_architecture_id(architecture); + architecture_id = db_access.get_architecture_id(architecture) if architecture_id == -1: - warn("architecture '%s' not recognised." % (architecture)); + warn("architecture '%s' not recognised." % (architecture)) else: - arch_ids_list.append(architecture_id); + arch_ids_list.append(architecture_id) if arch_ids_list: - con_architectures = "AND a.id IN (%s)" % ", ".join(map(str, arch_ids_list)); + con_architectures = "AND a.id IN (%s)" % ", ".join(map(str, arch_ids_list)) else: if not check_source: - fubar("No valid architecture given."); + fubar("No valid architecture given.") else: - check_source = 1; + check_source = 1 - return (con_suites, con_architectures, con_components, check_source); + return (con_suites, con_architectures, con_components, check_source) ################################################################################ @@ -735,36 +735,36 @@ def parse_args(Options): # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52215) def print_exc(): - tb = sys.exc_info()[2]; + tb = sys.exc_info()[2] while tb.tb_next: - tb = tb.tb_next; - stack = []; - frame = tb.tb_frame; + tb = tb.tb_next + stack = [] + frame = tb.tb_frame while frame: - stack.append(frame); - frame = frame.f_back; - stack.reverse(); - traceback.print_exc(); + stack.append(frame) + frame = frame.f_back + stack.reverse() + traceback.print_exc() for frame in stack: print "\nFrame %s in %s at line %s" % (frame.f_code.co_name, frame.f_code.co_filename, - frame.f_lineno); + frame.f_lineno) for key, value in frame.f_locals.items(): - print "\t%20s = " % key,; + print "\t%20s = " % key, try: - print value; + print value except: - print ""; + print "" ################################################################################ def try_with_debug(function): try: - function(); + function() except SystemExit: - raise; + raise except: - print_exc(); + print_exc() ################################################################################ @@ -773,13 +773,13 @@ def try_with_debug(function): def arch_compare_sw (a, b): if a == "source" and b == "source": - return 0; + return 0 elif a == "source": - return -1; + return -1 elif b == "source": - return 1; + return 1 - return cmp (a, b); + return cmp (a, b) ################################################################################ @@ -791,11 +791,11 @@ def arch_compare_sw (a, b): def split_args (s, dwim=1): if s.find(",") == -1: - return s.split(); + return s.split() else: if s[-1:] == "," and dwim: - fubar("split_args: found trailing comma, spurious space maybe?"); - return s.split(","); + fubar("split_args: found trailing comma, spurious space maybe?") + return s.split(",") ################################################################################ @@ -806,64 +806,64 @@ def Dict(**dict): return dict # Our very own version of commands.getouputstatus(), hacked to support # gpgv's status fd. def gpgv_get_status_output(cmd, status_read, status_write): - cmd = ['/bin/sh', '-c', cmd]; - p2cread, p2cwrite = os.pipe(); - c2pread, c2pwrite = os.pipe(); - errout, errin = os.pipe(); - pid = os.fork(); + cmd = ['/bin/sh', '-c', cmd] + p2cread, p2cwrite = os.pipe() + c2pread, c2pwrite = os.pipe() + errout, errin = os.pipe() + pid = os.fork() if pid == 0: # Child - os.close(0); - os.close(1); - os.dup(p2cread); - os.dup(c2pwrite); - os.close(2); - os.dup(errin); + os.close(0) + os.close(1) + os.dup(p2cread) + os.dup(c2pwrite) + os.close(2) + os.dup(errin) for i in range(3, 256): if i != status_write: try: - os.close(i); + os.close(i) except: - pass; + pass try: - os.execvp(cmd[0], cmd); + os.execvp(cmd[0], cmd) finally: - os._exit(1); + os._exit(1) # Parent os.close(p2cread) - os.dup2(c2pread, c2pwrite); - os.dup2(errout, errin); + os.dup2(c2pread, c2pwrite) + os.dup2(errout, errin) - output = status = ""; + output = status = "" while 1: - i, o, e = select.select([c2pwrite, errin, status_read], [], []); - more_data = []; + i, o, e = select.select([c2pwrite, errin, status_read], [], []) + more_data = [] for fd in i: - r = os.read(fd, 8196); + r = os.read(fd, 8196) if len(r) > 0: - more_data.append(fd); + more_data.append(fd) if fd == c2pwrite or fd == errin: - output += r; + output += r elif fd == status_read: - status += r; + status += r else: - fubar("Unexpected file descriptor [%s] returned from select\n" % (fd)); + fubar("Unexpected file descriptor [%s] returned from select\n" % (fd)) if not more_data: pid, exit_status = os.waitpid(pid, 0) try: - os.close(status_write); - os.close(status_read); - os.close(c2pread); - os.close(c2pwrite); - os.close(p2cwrite); - os.close(errin); - os.close(errout); + os.close(status_write) + os.close(status_read) + os.close(c2pread) + os.close(c2pwrite) + os.close(p2cwrite) + os.close(errin) + os.close(errout) except: - pass; - break; + pass + break - return output, status, exit_status; + return output, status, exit_status ############################################################ @@ -883,168 +883,168 @@ a *list* of keyrings to use. # Ensure the filename contains no shell meta-characters or other badness if not re_taint_free.match(sig_filename): - reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename)); - return None; + reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename)) + return None if data_filename and not re_taint_free.match(data_filename): - reject("!!WARNING!! tainted data filename: '%s'." % (data_filename)); - return None; + reject("!!WARNING!! tainted data filename: '%s'." % (data_filename)) + return None if not keyrings: keyrings = (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"]) # Build the command line status_read, status_write = os.pipe(); - cmd = "gpgv --status-fd %s" % (status_write); + cmd = "gpgv --status-fd %s" % (status_write) for keyring in keyrings: - cmd += " --keyring %s" % (keyring); - cmd += " %s %s" % (sig_filename, data_filename); + cmd += " --keyring %s" % (keyring) + cmd += " %s %s" % (sig_filename, data_filename) # Invoke gpgv on the file - (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write); + (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write) # Process the status-fd output - keywords = {}; - bad = internal_error = ""; + keywords = {} + bad = internal_error = "" for line in status.split('\n'): - line = line.strip(); + line = line.strip() if line == "": - continue; - split = line.split(); + continue + split = line.split() if len(split) < 2: - internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line); - continue; - (gnupg, keyword) = split[:2]; + internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line) + continue + (gnupg, keyword) = split[:2] if gnupg != "[GNUPG:]": - internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg); - continue; - args = split[2:]; + internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg) + continue + args = split[2:] if keywords.has_key(keyword) and (keyword != "NODATA" and keyword != "SIGEXPIRED"): - internal_error += "found duplicate status token ('%s').\n" % (keyword); - continue; + internal_error += "found duplicate status token ('%s').\n" % (keyword) + continue else: - keywords[keyword] = args; + keywords[keyword] = args # If we failed to parse the status-fd output, let's just whine and bail now if internal_error: - reject("internal error while performing signature check on %s." % (sig_filename)); - reject(internal_error, ""); - reject("Please report the above errors to the Archive maintainers by replying to this mail.", ""); - return None; + reject("internal error while performing signature check on %s." % (sig_filename)) + reject(internal_error, "") + reject("Please report the above errors to the Archive maintainers by replying to this mail.", "") + return None # Now check for obviously bad things in the processed output if keywords.has_key("SIGEXPIRED"): - reject("The key used to sign %s has expired." % (sig_filename)); - bad = 1; + reject("The key used to sign %s has expired." % (sig_filename)) + bad = 1 if keywords.has_key("KEYREVOKED"): - reject("The key used to sign %s has been revoked." % (sig_filename)); - bad = 1; + reject("The key used to sign %s has been revoked." % (sig_filename)) + bad = 1 if keywords.has_key("BADSIG"): - reject("bad signature on %s." % (sig_filename)); - bad = 1; + reject("bad signature on %s." % (sig_filename)) + bad = 1 if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"): - reject("failed to check signature on %s." % (sig_filename)); - bad = 1; + reject("failed to check signature on %s." % (sig_filename)) + bad = 1 if keywords.has_key("NO_PUBKEY"): - args = keywords["NO_PUBKEY"]; + args = keywords["NO_PUBKEY"] if len(args) >= 1: - key = args[0]; - reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename)); - bad = 1; + key = args[0] + reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename)) + bad = 1 if keywords.has_key("BADARMOR"): - reject("ASCII armour of signature was corrupt in %s." % (sig_filename)); - bad = 1; + reject("ASCII armour of signature was corrupt in %s." % (sig_filename)) + bad = 1 if keywords.has_key("NODATA"): - reject("no signature found in %s." % (sig_filename)); - bad = 1; + reject("no signature found in %s." % (sig_filename)) + bad = 1 if bad: - return None; + return None # Next check gpgv exited with a zero return code if exit_status: - reject("gpgv failed while checking %s." % (sig_filename)); + reject("gpgv failed while checking %s." % (sig_filename)) if status.strip(): - reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), ""); + reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "") else: - reject(prefix_multi_line_string(output, " [GPG output:] "), ""); - return None; + reject(prefix_multi_line_string(output, " [GPG output:] "), "") + return None # Sanity check the good stuff we expect if not keywords.has_key("VALIDSIG"): - reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename)) + bad = 1 else: - args = keywords["VALIDSIG"]; + args = keywords["VALIDSIG"] if len(args) < 1: - reject("internal error while checking signature on %s." % (sig_filename)); - bad = 1; + reject("internal error while checking signature on %s." % (sig_filename)) + bad = 1 else: - fingerprint = args[0]; + fingerprint = args[0] if not keywords.has_key("GOODSIG"): - reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename)) + bad = 1 if not keywords.has_key("SIG_ID"): - reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)); - bad = 1; + reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename)) + bad = 1 # Finally ensure there's not something we don't recognise known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="", SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="", - NODATA=""); + NODATA="") for keyword in keywords.keys(): if not known_keywords.has_key(keyword): - reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename)); - bad = 1; + reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename)) + bad = 1 if bad: - return None; + return None else: - return fingerprint; + return fingerprint ################################################################################ # Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603 def wrap(paragraph, max_length, prefix=""): - line = ""; - s = ""; - have_started = 0; - words = paragraph.split(); + line = "" + s = "" + have_started = 0 + words = paragraph.split() for word in words: - word_size = len(word); + word_size = len(word) if word_size > max_length: if have_started: - s += line + '\n' + prefix; - s += word + '\n' + prefix; + s += line + '\n' + prefix + s += word + '\n' + prefix else: if have_started: - new_length = len(line) + word_size + 1; + new_length = len(line) + word_size + 1 if new_length > max_length: - s += line + '\n' + prefix; - line = word; + s += line + '\n' + prefix + line = word else: - line += ' ' + word; + line += ' ' + word else: - line = word; - have_started = 1; + line = word + have_started = 1 if have_started: - s += line; + s += line - return s; + return s ################################################################################ # Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'. # Returns fixed 'src' def clean_symlink (src, dest, root): - src = src.replace(root, '', 1); - dest = dest.replace(root, '', 1); - dest = os.path.dirname(dest); - new_src = '../' * len(dest.split('/')); - return new_src + src; + src = src.replace(root, '', 1) + dest = dest.replace(root, '', 1) + dest = os.path.dirname(dest) + new_src = '../' * len(dest.split('/')) + return new_src + src ################################################################################ @@ -1054,29 +1054,29 @@ If 'directory' is non-null, it will be the directory the file is pre-created in. If 'dotprefix' is non-null, the filename will be prefixed with a '.'.""" if directory: - old_tempdir = tempfile.tempdir; - tempfile.tempdir = directory; + old_tempdir = tempfile.tempdir + tempfile.tempdir = directory - filename = tempfile.mktemp(); + filename = tempfile.mktemp() if dotprefix: - filename = "%s/.%s" % (os.path.dirname(filename), os.path.basename(filename)); - fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, perms); - os.close(fd); + filename = "%s/.%s" % (os.path.dirname(filename), os.path.basename(filename)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, perms) + os.close(fd) if directory: - tempfile.tempdir = old_tempdir; + tempfile.tempdir = old_tempdir - return filename; + return filename ################################################################################ -apt_pkg.init(); +apt_pkg.init() -Cnf = apt_pkg.newConfiguration(); -apt_pkg.ReadConfigFileISC(Cnf,default_config); +Cnf = apt_pkg.newConfiguration() +apt_pkg.ReadConfigFileISC(Cnf,default_config) if which_conf_file() != default_config: - apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()); + apt_pkg.ReadConfigFileISC(Cnf,which_conf_file()) ################################################################################ diff --git a/dak/ls.py b/dak/ls.py index 5a17892b..9f7b5566 100755 --- a/dak/ls.py +++ b/dak/ls.py @@ -27,14 +27,14 @@ ################################################################################ -import os, pg, sys; -import utils, db_access; -import apt_pkg; +import os, pg, sys +import utils, db_access +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ @@ -59,7 +59,7 @@ ARCH, COMPONENT and SUITE can be comma (or space) separated lists, e.g. ################################################################################ def main (): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() @@ -72,64 +72,64 @@ def main (): ('r', "regex", "Madison::Options::Regex"), ('s', "suite", "Madison::Options::Suite", "HasArg"), ('S', "source-and-binary", "Madison::Options::Source-And-Binary"), - ('h', "help", "Madison::Options::Help")]; + ('h', "help", "Madison::Options::Help")] for i in [ "architecture", "binarytype", "component", "format", "greaterorequal", "greaterthan", "regex", "suite", "source-and-binary", "help" ]: if not Cnf.has_key("Madison::Options::%s" % (i)): - Cnf["Madison::Options::%s" % (i)] = ""; + Cnf["Madison::Options::%s" % (i)] = "" - packages = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + packages = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Madison::Options") if Options["Help"]: - usage(); + usage() if not packages: - utils.fubar("need at least one package name as an argument."); + utils.fubar("need at least one package name as an argument.") - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) # If cron.daily is running; warn the user that our output might seem strange if os.path.exists(os.path.join(Cnf["Dir::Root"], "Archive_Maintenance_In_Progress")): - utils.warn("Archive maintenance is in progress; database inconsistencies are possible."); + utils.warn("Archive maintenance is in progress; database inconsistencies are possible.") # Handle buildd maintenance helper options if Options["GreaterOrEqual"] or Options["GreaterThan"]: if Options["GreaterOrEqual"] and Options["GreaterThan"]: - utils.fubar("-g/--greaterorequal and -G/--greaterthan are mutually exclusive."); + utils.fubar("-g/--greaterorequal and -G/--greaterthan are mutually exclusive.") if not Options["Suite"]: - Options["Suite"] = "unstable"; + Options["Suite"] = "unstable" # Parse -a/--architecture, -c/--component and -s/--suite (con_suites, con_architectures, con_components, check_source) = \ - utils.parse_args(Options); + utils.parse_args(Options) if Options["BinaryType"]: if Options["BinaryType"] != "udeb" and Options["BinaryType"] != "deb": - utils.fubar("Invalid binary type. 'udeb' and 'deb' recognised."); - con_bintype = "AND b.type = '%s'" % (Options["BinaryType"]); + utils.fubar("Invalid binary type. 'udeb' and 'deb' recognised.") + con_bintype = "AND b.type = '%s'" % (Options["BinaryType"]) # REMOVE ME TRAMP if Options["BinaryType"] == "udeb": - check_source = 0; + check_source = 0 else: - con_bintype = ""; + con_bintype = "" if Options["Regex"]: - comparison_operator = "~"; + comparison_operator = "~" else: - comparison_operator = "="; + comparison_operator = "=" if Options["Source-And-Binary"]: - new_packages = []; + new_packages = [] for package in packages: - q = projectB.query("SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s '%s' %s" % (comparison_operator, package, con_suites)); - new_packages.extend(map(lambda x: x[0], q.getresult())); + q = projectB.query("SELECT DISTINCT b.package FROM binaries b, bin_associations ba, suite su, source s WHERE b.source = s.id AND su.id = ba.suite AND b.id = ba.bin AND s.source %s '%s' %s" % (comparison_operator, package, con_suites)) + new_packages.extend(map(lambda x: x[0], q.getresult())) if package not in new_packages: - new_packages.append(package); - packages = new_packages; + new_packages.append(package) + packages = new_packages - results = 0; + results = 0 for package in packages: q = projectB.query(""" SELECT b.package, b.version, a.arch_string, su.suite_name, c.name, m.name @@ -138,8 +138,8 @@ SELECT b.package, b.version, a.arch_string, su.suite_name, c.name, m.name WHERE b.package %s '%s' AND a.id = b.architecture AND su.id = ba.suite AND b.id = ba.bin AND b.file = f.id AND f.location = l.id AND l.component = c.id AND b.maintainer = m.id %s %s %s -""" % (comparison_operator, package, con_suites, con_architectures, con_bintype)); - ql = q.getresult(); +""" % (comparison_operator, package, con_suites, con_architectures, con_bintype)) + ql = q.getresult() if check_source: q = projectB.query(""" SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name @@ -148,51 +148,51 @@ SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name WHERE s.source %s '%s' AND su.id = sa.suite AND s.id = sa.source AND s.file = f.id AND f.location = l.id AND l.component = c.id AND s.maintainer = m.id %s -""" % (comparison_operator, package, con_suites)); - ql.extend(q.getresult()); - d = {}; - highver = {}; +""" % (comparison_operator, package, con_suites)) + ql.extend(q.getresult()) + d = {} + highver = {} for i in ql: - results += 1; - (pkg, version, architecture, suite, component, maintainer) = i; + results += 1 + (pkg, version, architecture, suite, component, maintainer) = i if component != "main": - suite = "%s/%s" % (suite, component); + suite = "%s/%s" % (suite, component) if not d.has_key(pkg): - d[pkg] = {}; - highver.setdefault(pkg,""); + d[pkg] = {} + highver.setdefault(pkg,"") if not d[pkg].has_key(version): - d[pkg][version] = {}; + d[pkg][version] = {} if apt_pkg.VersionCompare(version, highver[pkg]) > 0: - highver[pkg] = version; + highver[pkg] = version if not d[pkg][version].has_key(suite): - d[pkg][version][suite] = []; - d[pkg][version][suite].append(architecture); + d[pkg][version][suite] = [] + d[pkg][version][suite].append(architecture) - packages = d.keys(); - packages.sort(); + packages = d.keys() + packages.sort() for pkg in packages: - versions = d[pkg].keys(); - versions.sort(apt_pkg.VersionCompare); + versions = d[pkg].keys() + versions.sort(apt_pkg.VersionCompare) for version in versions: - suites = d[pkg][version].keys(); - suites.sort(); + suites = d[pkg][version].keys() + suites.sort() for suite in suites: - arches = d[pkg][version][suite]; - arches.sort(utils.arch_compare_sw); + arches = d[pkg][version][suite] + arches.sort(utils.arch_compare_sw) if Options["Format"] == "": #normal - sys.stdout.write("%10s | %10s | %13s | " % (pkg, version, suite)); - sys.stdout.write(", ".join(arches)); - sys.stdout.write('\n'); + sys.stdout.write("%10s | %10s | %13s | " % (pkg, version, suite)) + sys.stdout.write(", ".join(arches)) + sys.stdout.write('\n') elif Options["Format"] == "heidi": for arch in arches: - sys.stdout.write("%s %s %s\n" % (pkg, version, arch)); + sys.stdout.write("%s %s %s\n" % (pkg, version, arch)) if Options["GreaterOrEqual"]: print "\n%s (>= %s)" % (pkg, highver[pkg]) if Options["GreaterThan"]: print "\n%s (>> %s)" % (pkg, highver[pkg]) if not results: - sys.exit(1); + sys.exit(1) ####################################################################################### diff --git a/dak/make_maintainers.py b/dak/make_maintainers.py index b9eae147..c1cc11d7 100755 --- a/dak/make_maintainers.py +++ b/dak/make_maintainers.py @@ -26,9 +26,9 @@ ################################################################################ -import pg, sys; -import db_access, utils; -import apt_pkg; +import pg, sys +import db_access, utils +import apt_pkg ################################################################################ @@ -51,7 +51,7 @@ Generate an index of packages <=> Maintainers. ################################################################################ def fix_maintainer (maintainer): - global fixed_maintainer_cache; + global fixed_maintainer_cache if not fixed_maintainer_cache.has_key(maintainer): fixed_maintainer_cache[maintainer] = utils.fix_maintainer(maintainer)[0] @@ -59,13 +59,13 @@ def fix_maintainer (maintainer): return fixed_maintainer_cache[maintainer] def get_maintainer (maintainer): - return fix_maintainer(db_access.get_maintainer(maintainer)); + return fix_maintainer(db_access.get_maintainer(maintainer)) def get_maintainer_from_source (source_id): global maintainer_from_source_cache if not maintainer_from_source_cache.has_key(source_id): - q = projectB.query("SELECT m.name FROM maintainer m, source s WHERE s.id = %s and s.maintainer = m.id" % (source_id)); + q = projectB.query("SELECT m.name FROM maintainer m, source s WHERE s.id = %s and s.maintainer = m.id" % (source_id)) maintainer = q.getresult()[0][0] maintainer_from_source_cache[source_id] = fix_maintainer(maintainer) @@ -74,86 +74,86 @@ def get_maintainer_from_source (source_id): ################################################################################ def main(): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() - Arguments = [('h',"help","Charisma::Options::Help")]; + Arguments = [('h',"help","Charisma::Options::Help")] if not Cnf.has_key("Charisma::Options::Help"): - Cnf["Charisma::Options::Help"] = ""; + Cnf["Charisma::Options::Help"] = "" - extra_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Options = Cnf.SubTree("Charisma::Options"); + extra_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + Options = Cnf.SubTree("Charisma::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) for suite in Cnf.SubTree("Suite").List(): - suite = suite.lower(); - suite_priority = int(Cnf["Suite::%s::Priority" % (suite)]); + suite = suite.lower() + suite_priority = int(Cnf["Suite::%s::Priority" % (suite)]) # Source packages q = projectB.query("SELECT s.source, s.version, m.name FROM src_associations sa, source s, suite su, maintainer m WHERE su.suite_name = '%s' AND sa.suite = su.id AND sa.source = s.id AND m.id = s.maintainer" % (suite)) - sources = q.getresult(); + sources = q.getresult() for source in sources: - package = source[0]; - version = source[1]; - maintainer = fix_maintainer(source[2]); + package = source[0] + version = source[1] + maintainer = fix_maintainer(source[2]) if packages.has_key(package): if packages[package]["priority"] <= suite_priority: if apt_pkg.VersionCompare(packages[package]["version"], version) < 0: - packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version }; + packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version } else: - packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version }; + packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version } # Binary packages - q = projectB.query("SELECT b.package, b.source, b.maintainer, b.version FROM bin_associations ba, binaries b, suite s WHERE s.suite_name = '%s' AND ba.suite = s.id AND ba.bin = b.id" % (suite)); - binaries = q.getresult(); + q = projectB.query("SELECT b.package, b.source, b.maintainer, b.version FROM bin_associations ba, binaries b, suite s WHERE s.suite_name = '%s' AND ba.suite = s.id AND ba.bin = b.id" % (suite)) + binaries = q.getresult() for binary in binaries: - package = binary[0]; - source_id = binary[1]; - version = binary[3]; + package = binary[0] + source_id = binary[1] + version = binary[3] # Use the source maintainer first; falling back on the binary maintainer as a last resort only if source_id: - maintainer = get_maintainer_from_source(source_id); + maintainer = get_maintainer_from_source(source_id) else: - maintainer = get_maintainer(binary[2]); + maintainer = get_maintainer(binary[2]) if packages.has_key(package): if packages[package]["priority"] <= suite_priority: if apt_pkg.VersionCompare(packages[package]["version"], version) < 0: - packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version }; + packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version } else: - packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version }; + packages[package] = { "maintainer": maintainer, "priority": suite_priority, "version": version } # Process any additional Maintainer files (e.g. from non-US or pseudo packages) for filename in extra_files: - file = utils.open_file(filename); + file = utils.open_file(filename) for line in file.readlines(): - line = utils.re_comments.sub('', line).strip(); + line = utils.re_comments.sub('', line).strip() if line == "": - continue; - split = line.split(); - lhs = split[0]; - maintainer = fix_maintainer(" ".join(split[1:])); + continue + split = line.split() + lhs = split[0] + maintainer = fix_maintainer(" ".join(split[1:])) if lhs.find('~') != -1: - (package, version) = lhs.split('~'); + (package, version) = lhs.split('~') else: - package = lhs; - version = '*'; + package = lhs + version = '*' # A version of '*' overwhelms all real version numbers if not packages.has_key(package) or version == '*' \ or apt_pkg.VersionCompare(packages[package]["version"], version) < 0: - packages[package] = { "maintainer": maintainer, "version": version }; - file.close(); + packages[package] = { "maintainer": maintainer, "version": version } + file.close() package_keys = packages.keys() package_keys.sort() for package in package_keys: - lhs = "~".join([package, packages[package]["version"]]); - print "%-30s %s" % (lhs, packages[package]["maintainer"]); + lhs = "~".join([package, packages[package]["version"]]) + print "%-30s %s" % (lhs, packages[package]["maintainer"]) ################################################################################ diff --git a/dak/make_overrides.py b/dak/make_overrides.py index cae1bb6d..2517b67a 100755 --- a/dak/make_overrides.py +++ b/dak/make_overrides.py @@ -26,14 +26,14 @@ ################################################################################ -import pg, sys; -import utils, db_access; -import apt_pkg; +import pg, sys +import utils, db_access +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None override = {} ################################################################################ @@ -48,79 +48,79 @@ Outputs the override tables to text files. ################################################################################ def do_list(output_file, suite, component, otype): - global override; + global override - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id == -1: - utils.fubar("Suite '%s' not recognised." % (suite)); + utils.fubar("Suite '%s' not recognised." % (suite)) - component_id = db_access.get_component_id(component); + component_id = db_access.get_component_id(component) if component_id == -1: - utils.fubar("Component '%s' not recognised." % (component)); + utils.fubar("Component '%s' not recognised." % (component)) - otype_id = db_access.get_override_type_id(otype); + otype_id = db_access.get_override_type_id(otype) if otype_id == -1: - utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)); + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)) - override.setdefault(suite, {}); - override[suite].setdefault(component, {}); - override[suite][component].setdefault(otype, {}); + override.setdefault(suite, {}) + override[suite].setdefault(component, {}) + override[suite][component].setdefault(otype, {}) if otype == "dsc": - q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, otype_id)); + q = projectB.query("SELECT o.package, s.section, o.maintainer FROM override o, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.section = s.id ORDER BY s.section, o.package" % (suite_id, component_id, otype_id)) for i in q.getresult(): - override[suite][component][otype][i[0]] = i; - output_file.write(utils.result_join(i)+'\n'); + override[suite][component][otype][i[0]] = i + output_file.write(utils.result_join(i)+'\n') else: - q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, otype_id)); + q = projectB.query("SELECT o.package, p.priority, s.section, o.maintainer, p.level FROM override o, priority p, section s WHERE o.suite = %s AND o.component = %s AND o.type = %s AND o.priority = p.id AND o.section = s.id ORDER BY s.section, p.level, o.package" % (suite_id, component_id, otype_id)) for i in q.getresult(): i = i[:-1]; # Strip the priority level - override[suite][component][otype][i[0]] = i; - output_file.write(utils.result_join(i)+'\n'); + override[suite][component][otype][i[0]] = i + output_file.write(utils.result_join(i)+'\n') ################################################################################ def main (): - global Cnf, projectB, override; + global Cnf, projectB, override Cnf = utils.get_conf() - Arguments = [('h',"help","Denise::Options::Help")]; + Arguments = [('h',"help","Denise::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Denise::Options::%s" % (i)): - Cnf["Denise::Options::%s" % (i)] = ""; - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + Cnf["Denise::Options::%s" % (i)] = "" + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Denise::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) for suite in Cnf.SubTree("Cindy::OverrideSuites").List(): if Cnf.has_key("Suite::%s::Untouchable" % suite) and Cnf["Suite::%s::Untouchable" % suite] != 0: continue suite = suite.lower() - sys.stderr.write("Processing %s...\n" % (suite)); - override_suite = Cnf["Suite::%s::OverrideCodeName" % (suite)]; + sys.stderr.write("Processing %s...\n" % (suite)) + override_suite = Cnf["Suite::%s::OverrideCodeName" % (suite)] for component in Cnf.SubTree("Component").List(): if component == "mixed": continue; # Ick for otype in Cnf.ValueList("OverrideType"): if otype == "deb": - suffix = ""; + suffix = "" elif otype == "udeb": if component != "main": continue; # Ick2 - suffix = ".debian-installer"; + suffix = ".debian-installer" elif otype == "dsc": - suffix = ".src"; - filename = "%s/override.%s.%s%s" % (Cnf["Dir::Override"], override_suite, component.replace("non-US/", ""), suffix); - output_file = utils.open_file(filename, 'w'); - do_list(output_file, suite, component, otype); - output_file.close(); + suffix = ".src" + filename = "%s/override.%s.%s%s" % (Cnf["Dir::Override"], override_suite, component.replace("non-US/", ""), suffix) + output_file = utils.open_file(filename, 'w') + do_list(output_file, suite, component, otype) + output_file.close() ################################################################################ if __name__ == '__main__': - main(); + main() diff --git a/dak/make_suite_file_list.py b/dak/make_suite_file_list.py index 99de03b2..927de7ff 100755 --- a/dak/make_suite_file_list.py +++ b/dak/make_suite_file_list.py @@ -34,16 +34,16 @@ ################################################################################ -import copy, os, pg, string, sys; -import apt_pkg; -import claire, db_access, logging, utils; +import copy, os, pg, string, sys +import apt_pkg +import claire, db_access, logging, utils ################################################################################ -projectB = None; -Cnf = None; -Logger = None; -Options = None; +projectB = None +Cnf = None +Logger = None +Options = None ################################################################################ @@ -62,162 +62,162 @@ Write out file lists suitable for use with apt-ftparchive. -s, --suite=SUITE only write file lists for this suite ARCH, COMPONENT and SUITE can be space separated lists, e.g. - --architecture=\"m68k i386\""""; - sys.exit(exit_code); + --architecture=\"m68k i386\"""" + sys.exit(exit_code) ################################################################################ def version_cmp(a, b): - return -apt_pkg.VersionCompare(a[0], b[0]); + return -apt_pkg.VersionCompare(a[0], b[0]) ##################################################### def delete_packages(delete_versions, pkg, dominant_arch, suite, dominant_version, delete_table, delete_col, packages): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) for version in delete_versions: - delete_unique_id = version[1]; + delete_unique_id = version[1] if not packages.has_key(delete_unique_id): - continue; - delete_version = version[0]; - delete_id = packages[delete_unique_id]["id"]; - delete_arch = packages[delete_unique_id]["arch"]; + continue + delete_version = version[0] + delete_id = packages[delete_unique_id]["id"] + delete_arch = packages[delete_unique_id]["arch"] if not Cnf.Find("Suite::%s::Untouchable" % (suite)): if Options["No-Delete"]: - print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch); + print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch) else: - Logger.log(["dominated", pkg, delete_arch, delete_version, dominant_version, dominant_arch]); - projectB.query("DELETE FROM %s WHERE suite = %s AND %s = %s" % (delete_table, suite_id, delete_col, delete_id)); - del packages[delete_unique_id]; + Logger.log(["dominated", pkg, delete_arch, delete_version, dominant_version, dominant_arch]) + projectB.query("DELETE FROM %s WHERE suite = %s AND %s = %s" % (delete_table, suite_id, delete_col, delete_id)) + del packages[delete_unique_id] else: if Options["No-Delete"]: - print "Would delete %s_%s_%s in favour of %s_%s, but %s is untouchable" % (pkg, delete_arch, delete_version, dominant_version, dominant_arch, suite); + print "Would delete %s_%s_%s in favour of %s_%s, but %s is untouchable" % (pkg, delete_arch, delete_version, dominant_version, dominant_arch, suite) else: - Logger.log(["dominated but untouchable", pkg, delete_arch, delete_version, dominant_version, dominant_arch]); + Logger.log(["dominated but untouchable", pkg, delete_arch, delete_version, dominant_version, dominant_arch]) ##################################################### # Per-suite&pkg: resolve arch-all, vs. arch-any, assumes only one arch-all def resolve_arch_all_vs_any(versions, packages): - arch_all_version = None; - arch_any_versions = copy.copy(versions); + arch_all_version = None + arch_any_versions = copy.copy(versions) for i in arch_any_versions: - unique_id = i[1]; - arch = packages[unique_id]["arch"]; + unique_id = i[1] + arch = packages[unique_id]["arch"] if arch == "all": - arch_all_versions = [i]; - arch_all_version = i[0]; - arch_any_versions.remove(i); + arch_all_versions = [i] + arch_all_version = i[0] + arch_any_versions.remove(i) # Sort arch: any versions into descending order - arch_any_versions.sort(version_cmp); - highest_arch_any_version = arch_any_versions[0][0]; + arch_any_versions.sort(version_cmp) + highest_arch_any_version = arch_any_versions[0][0] - pkg = packages[unique_id]["pkg"]; - suite = packages[unique_id]["suite"]; - delete_table = "bin_associations"; - delete_col = "bin"; + pkg = packages[unique_id]["pkg"] + suite = packages[unique_id]["suite"] + delete_table = "bin_associations" + delete_col = "bin" if apt_pkg.VersionCompare(highest_arch_any_version, arch_all_version) < 1: # arch: all dominates delete_packages(arch_any_versions, pkg, "all", suite, - arch_all_version, delete_table, delete_col, packages); + arch_all_version, delete_table, delete_col, packages) else: # arch: any dominates delete_packages(arch_all_versions, pkg, "any", suite, highest_arch_any_version, delete_table, delete_col, - packages); + packages) ##################################################### # Per-suite&pkg&arch: resolve duplicate versions def remove_duplicate_versions(versions, packages): # Sort versions into descending order - versions.sort(version_cmp); - dominant_versions = versions[0]; - dominated_versions = versions[1:]; - (dominant_version, dominant_unqiue_id) = dominant_versions; - pkg = packages[dominant_unqiue_id]["pkg"]; - arch = packages[dominant_unqiue_id]["arch"]; - suite = packages[dominant_unqiue_id]["suite"]; + versions.sort(version_cmp) + dominant_versions = versions[0] + dominated_versions = versions[1:] + (dominant_version, dominant_unqiue_id) = dominant_versions + pkg = packages[dominant_unqiue_id]["pkg"] + arch = packages[dominant_unqiue_id]["arch"] + suite = packages[dominant_unqiue_id]["suite"] if arch == "source": - delete_table = "src_associations"; - delete_col = "source"; + delete_table = "src_associations" + delete_col = "source" else: # !source - delete_table = "bin_associations"; - delete_col = "bin"; + delete_table = "bin_associations" + delete_col = "bin" # Remove all but the highest delete_packages(dominated_versions, pkg, arch, suite, - dominant_version, delete_table, delete_col, packages); - return [dominant_versions]; + dominant_version, delete_table, delete_col, packages) + return [dominant_versions] ################################################################################ def cleanup(packages): # Build up the index used by the clean up functions - d = {}; + d = {} for unique_id in packages.keys(): - suite = packages[unique_id]["suite"]; - pkg = packages[unique_id]["pkg"]; - arch = packages[unique_id]["arch"]; - version = packages[unique_id]["version"]; - d.setdefault(suite, {}); - d[suite].setdefault(pkg, {}); - d[suite][pkg].setdefault(arch, []); - d[suite][pkg][arch].append([version, unique_id]); + suite = packages[unique_id]["suite"] + pkg = packages[unique_id]["pkg"] + arch = packages[unique_id]["arch"] + version = packages[unique_id]["version"] + d.setdefault(suite, {}) + d[suite].setdefault(pkg, {}) + d[suite][pkg].setdefault(arch, []) + d[suite][pkg][arch].append([version, unique_id]) # Clean up old versions for suite in d.keys(): for pkg in d[suite].keys(): for arch in d[suite][pkg].keys(): - versions = d[suite][pkg][arch]; + versions = d[suite][pkg][arch] if len(versions) > 1: - d[suite][pkg][arch] = remove_duplicate_versions(versions, packages); + d[suite][pkg][arch] = remove_duplicate_versions(versions, packages) # Arch: all -> any and vice versa for suite in d.keys(): for pkg in d[suite].keys(): - arches = d[suite][pkg]; + arches = d[suite][pkg] # If we don't have any arch: all; we've nothing to do if not arches.has_key("all"): - continue; + continue # Check to see if we have arch: all and arch: !all (ignoring source) - num_arches = len(arches.keys()); + num_arches = len(arches.keys()) if arches.has_key("source"): - num_arches -= 1; + num_arches -= 1 # If we do, remove the duplicates if num_arches > 1: - versions = []; + versions = [] for arch in arches.keys(): if arch != "source": - versions.extend(d[suite][pkg][arch]); - resolve_arch_all_vs_any(versions, packages); + versions.extend(d[suite][pkg][arch]) + resolve_arch_all_vs_any(versions, packages) ################################################################################ def write_legacy_mixed_filelist(suite, list, packages, dislocated_files): # Work out the filename - filename = os.path.join(Cnf["Dir::Lists"], "%s_-_all.list" % (suite)); - output = utils.open_file(filename, "w"); + filename = os.path.join(Cnf["Dir::Lists"], "%s_-_all.list" % (suite)) + output = utils.open_file(filename, "w") # Generate the final list of files - files = {}; + files = {} for id in list: - path = packages[id]["path"]; - filename = packages[id]["filename"]; - file_id = packages[id]["file_id"]; + path = packages[id]["path"] + filename = packages[id]["filename"] + file_id = packages[id]["file_id"] if suite == "stable" and dislocated_files.has_key(file_id): - filename = dislocated_files[file_id]; + filename = dislocated_files[file_id] else: - filename = path + filename; + filename = path + filename if files.has_key(filename): - utils.warn("%s (in %s) is duplicated." % (filename, suite)); + utils.warn("%s (in %s) is duplicated." % (filename, suite)) else: - files[filename] = ""; + files[filename] = "" # Sort the files since apt-ftparchive doesn't - keys = files.keys(); - keys.sort(); + keys = files.keys() + keys.sort() # Write the list of files out for file in keys: output.write(file+'\n') - output.close(); + output.close() ############################################################ @@ -225,106 +225,106 @@ def write_filelist(suite, component, arch, type, list, packages, dislocated_file # Work out the filename if arch != "source": if type == "udeb": - arch = "debian-installer_binary-%s" % (arch); + arch = "debian-installer_binary-%s" % (arch) elif type == "deb": - arch = "binary-%s" % (arch); - filename = os.path.join(Cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch)); - output = utils.open_file(filename, "w"); + arch = "binary-%s" % (arch) + filename = os.path.join(Cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch)) + output = utils.open_file(filename, "w") # Generate the final list of files - files = {}; + files = {} for id in list: - path = packages[id]["path"]; - filename = packages[id]["filename"]; - file_id = packages[id]["file_id"]; - pkg = packages[id]["pkg"]; + path = packages[id]["path"] + filename = packages[id]["filename"] + file_id = packages[id]["file_id"] + pkg = packages[id]["pkg"] if suite == "stable" and dislocated_files.has_key(file_id): - filename = dislocated_files[file_id]; + filename = dislocated_files[file_id] else: - filename = path + filename; + filename = path + filename if files.has_key(pkg): - utils.warn("%s (in %s/%s, %s) is duplicated." % (pkg, suite, component, filename)); + utils.warn("%s (in %s/%s, %s) is duplicated." % (pkg, suite, component, filename)) else: - files[pkg] = filename; + files[pkg] = filename # Sort the files since apt-ftparchive doesn't - pkgs = files.keys(); - pkgs.sort(); + pkgs = files.keys() + pkgs.sort() # Write the list of files out for pkg in pkgs: output.write(files[pkg]+'\n') - output.close(); + output.close() ################################################################################ def write_filelists(packages, dislocated_files): # Build up the index to iterate over - d = {}; + d = {} for unique_id in packages.keys(): - suite = packages[unique_id]["suite"]; - component = packages[unique_id]["component"]; - arch = packages[unique_id]["arch"]; - type = packages[unique_id]["type"]; - d.setdefault(suite, {}); - d[suite].setdefault(component, {}); - d[suite][component].setdefault(arch, {}); - d[suite][component][arch].setdefault(type, []); - d[suite][component][arch][type].append(unique_id); + suite = packages[unique_id]["suite"] + component = packages[unique_id]["component"] + arch = packages[unique_id]["arch"] + type = packages[unique_id]["type"] + d.setdefault(suite, {}) + d[suite].setdefault(component, {}) + d[suite][component].setdefault(arch, {}) + d[suite][component][arch].setdefault(type, []) + d[suite][component][arch][type].append(unique_id) # Flesh out the index if not Options["Suite"]: - suites = Cnf.SubTree("Suite").List(); + suites = Cnf.SubTree("Suite").List() else: - suites = utils.split_args(Options["Suite"]); + suites = utils.split_args(Options["Suite"]) for suite in map(string.lower, suites): - d.setdefault(suite, {}); + d.setdefault(suite, {}) if not Options["Component"]: - components = Cnf.ValueList("Suite::%s::Components" % (suite)); + components = Cnf.ValueList("Suite::%s::Components" % (suite)) else: - components = utils.split_args(Options["Component"]); - udeb_components = Cnf.ValueList("Suite::%s::UdebComponents" % (suite)); - udeb_components = udeb_components; + components = utils.split_args(Options["Component"]) + udeb_components = Cnf.ValueList("Suite::%s::UdebComponents" % (suite)) + udeb_components = udeb_components for component in components: - d[suite].setdefault(component, {}); + d[suite].setdefault(component, {}) if component in udeb_components: - binary_types = [ "deb", "udeb" ]; + binary_types = [ "deb", "udeb" ] else: - binary_types = [ "deb" ]; + binary_types = [ "deb" ] if not Options["Architecture"]: - architectures = Cnf.ValueList("Suite::%s::Architectures" % (suite)); + architectures = Cnf.ValueList("Suite::%s::Architectures" % (suite)) else: - architectures = utils.split_args(Options["Architectures"]); + architectures = utils.split_args(Options["Architectures"]) for arch in map(string.lower, architectures): - d[suite][component].setdefault(arch, {}); + d[suite][component].setdefault(arch, {}) if arch == "source": - types = [ "dsc" ]; + types = [ "dsc" ] else: - types = binary_types; + types = binary_types for type in types: - d[suite][component][arch].setdefault(type, []); + d[suite][component][arch].setdefault(type, []) # Then walk it for suite in d.keys(): if Cnf.has_key("Suite::%s::Components" % (suite)): for component in d[suite].keys(): for arch in d[suite][component].keys(): if arch == "all": - continue; + continue for type in d[suite][component][arch].keys(): - list = d[suite][component][arch][type]; + list = d[suite][component][arch][type] # If it's a binary, we need to add in the arch: all debs too if arch != "source": - archall_suite = Cnf.get("Jenna::ArchAllMap::%s" % (suite)); + archall_suite = Cnf.get("Jenna::ArchAllMap::%s" % (suite)) if archall_suite: - list.extend(d[archall_suite][component]["all"][type]); + list.extend(d[archall_suite][component]["all"][type]) elif d[suite][component].has_key("all") and \ d[suite][component]["all"].has_key(type): - list.extend(d[suite][component]["all"][type]); + list.extend(d[suite][component]["all"][type]) write_filelist(suite, component, arch, type, list, - packages, dislocated_files); + packages, dislocated_files) else: # legacy-mixed suite - list = []; + list = [] for component in d[suite].keys(): for arch in d[suite][component].keys(): for type in d[suite][component][arch].keys(): - list.extend(d[suite][component][arch][type]); - write_legacy_mixed_filelist(suite, list, packages, dislocated_files); + list.extend(d[suite][component][arch][type]) + write_legacy_mixed_filelist(suite, list, packages, dislocated_files) ################################################################################ @@ -332,20 +332,20 @@ def write_filelists(packages, dislocated_files): def stable_dislocation_p(): # If the support is not explicitly enabled, assume it's disabled if not Cnf.FindB("Dinstall::StableDislocationSupport"): - return 0; + return 0 # If we don't have a stable suite, obviously a no-op if not Cnf.has_key("Suite::Stable"): - return 0; + return 0 # If the suite(s) weren't explicitly listed, all suites are done if not Options["Suite"]: - return 1; + return 1 # Otherwise, look in what suites the user specified - suites = utils.split_args(Options["Suite"]); + suites = utils.split_args(Options["Suite"]) if "stable" in suites: - return 1; + return 1 else: - return 0; + return 0 ################################################################################ @@ -353,21 +353,21 @@ def do_da_do_da(): # If we're only doing a subset of suites, ensure we do enough to # be able to do arch: all mapping. if Options["Suite"]: - suites = utils.split_args(Options["Suite"]); + suites = utils.split_args(Options["Suite"]) for suite in suites: - archall_suite = Cnf.get("Jenna::ArchAllMap::%s" % (suite)); + archall_suite = Cnf.get("Jenna::ArchAllMap::%s" % (suite)) if archall_suite and archall_suite not in suites: - utils.warn("Adding %s as %s maps Arch: all from it." % (archall_suite, suite)); - suites.append(archall_suite); - Options["Suite"] = ",".join(suites); + utils.warn("Adding %s as %s maps Arch: all from it." % (archall_suite, suite)) + suites.append(archall_suite) + Options["Suite"] = ",".join(suites) (con_suites, con_architectures, con_components, check_source) = \ - utils.parse_args(Options); + utils.parse_args(Options) if stable_dislocation_p(): - dislocated_files = claire.find_dislocated_stable(Cnf, projectB); + dislocated_files = claire.find_dislocated_stable(Cnf, projectB) else: - dislocated_files = {}; + dislocated_files = {} query = """ SELECT b.id, b.package, a.arch_string, b.version, l.path, f.filename, c.name, @@ -376,7 +376,7 @@ SELECT b.id, b.package, a.arch_string, b.version, l.path, f.filename, c.name, component c, suite su WHERE b.id = ba.bin AND b.file = f.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND ba.suite = su.id - %s %s %s""" % (con_suites, con_architectures, con_components); + %s %s %s""" % (con_suites, con_architectures, con_components) if check_source: query += """ UNION @@ -384,49 +384,49 @@ SELECT s.id, s.source, 'source', s.version, l.path, f.filename, c.name, f.id, su.suite_name, 'dsc' FROM source s, src_associations sa, files f, location l, component c, suite su WHERE s.id = sa.source AND s.file = f.id AND f.location = l.id - AND l.component = c.id AND sa.suite = su.id %s %s""" % (con_suites, con_components); - q = projectB.query(query); - ql = q.getresult(); + AND l.component = c.id AND sa.suite = su.id %s %s""" % (con_suites, con_components) + q = projectB.query(query) + ql = q.getresult() # Build up the main index of packages - packages = {}; - unique_id = 0; + packages = {} + unique_id = 0 for i in ql: - (id, pkg, arch, version, path, filename, component, file_id, suite, type) = i; + (id, pkg, arch, version, path, filename, component, file_id, suite, type) = i # 'id' comes from either 'binaries' or 'source', so it's not unique - unique_id += 1; + unique_id += 1 packages[unique_id] = Dict(id=id, pkg=pkg, arch=arch, version=version, path=path, filename=filename, component=component, file_id=file_id, - suite=suite, type = type); - cleanup(packages); - write_filelists(packages, dislocated_files); + suite=suite, type = type) + cleanup(packages) + write_filelists(packages, dislocated_files) ################################################################################ def main(): - global Cnf, projectB, Options, Logger; + global Cnf, projectB, Options, Logger - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('a', "architecture", "Jenna::Options::Architecture", "HasArg"), ('c', "component", "Jenna::Options::Component", "HasArg"), ('h', "help", "Jenna::Options::Help"), ('n', "no-delete", "Jenna::Options::No-Delete"), - ('s', "suite", "Jenna::Options::Suite", "HasArg")]; + ('s', "suite", "Jenna::Options::Suite", "HasArg")] for i in ["architecture", "component", "help", "no-delete", "suite" ]: if not Cnf.has_key("Jenna::Options::%s" % (i)): - Cnf["Jenna::Options::%s" % (i)] = ""; - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Options = Cnf.SubTree("Jenna::Options"); + Cnf["Jenna::Options::%s" % (i)] = "" + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + Options = Cnf.SubTree("Jenna::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); - Logger = logging.Logger(Cnf, "jenna"); - do_da_do_da(); - Logger.close(); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) + Logger = logging.Logger(Cnf, "jenna") + do_da_do_da() + Logger.close() ######################################################################################### if __name__ == '__main__': - main(); + main() diff --git a/dak/mirror_split.py b/dak/mirror_split.py index 5d9d0035..b2632b9e 100755 --- a/dak/mirror_split.py +++ b/dak/mirror_split.py @@ -24,20 +24,20 @@ ## Make something damned stupid up and attribute it to me, that's okay ############################################################################### -import pg, pwd, sys; -import utils, db_access; -import apt_pkg, logging; +import pg, pwd, sys +import utils, db_access +import apt_pkg, logging -from stat import S_ISDIR, S_ISLNK, S_ISREG; -import os; -import cPickle; +from stat import S_ISDIR, S_ISLNK, S_ISREG +import os +import cPickle ## Master path is the main repository -#MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master"; +#MASTER_PATH = "/org/ftp.debian.org/scratch/dsilvers/master" -MASTER_PATH = "***Configure Billie::FTPPath Please***"; -TREE_ROOT = "***Configure Billie::TreeRootPath Please***"; -TREE_DB_ROOT = "***Configure Billie::TreeDatabasePath Please***"; +MASTER_PATH = "***Configure Billie::FTPPath Please***" +TREE_ROOT = "***Configure Billie::TreeRootPath Please***" +TREE_DB_ROOT = "***Configure Billie::TreeDatabasePath Please***" trees = [] ############################################################################### @@ -47,54 +47,54 @@ trees = [] class BillieTarget: def __init__(self, name, archs, source): - self.name = name; - self.root = "%s/%s" % (TREE_ROOT,name); - self.archs = archs.split(","); - self.source = source; - self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name); - self.db = BillieDB(); + self.name = name + self.root = "%s/%s" % (TREE_ROOT,name) + self.archs = archs.split(",") + self.source = source + self.dbpath = "%s/%s.db" % (TREE_DB_ROOT,name) + self.db = BillieDB() if os.path.exists( self.dbpath ): - self.db.load_from_file( self.dbpath ); + self.db.load_from_file( self.dbpath ) ## Save the db back to disk def save_db(self): - self.db.save_to_file( self.dbpath ); + self.db.save_to_file( self.dbpath ) ## Returns true if it's a poolish match def poolish_match(self, path): for a in self.archs: if path.endswith( "_%s.deb" % (a) ): - return 1; + return 1 if path.endswith( "_%s.udeb" % (a) ): - return 1; + return 1 if self.source: if (path.endswith( ".tar.gz" ) or path.endswith( ".diff.gz" ) or path.endswith( ".dsc" )): - return 1; - return 0; + return 1 + return 0 ## Returns false if it's a badmatch distswise def distish_match(self,path): for a in self.archs: if path.endswith("/Contents-%s.gz" % (a)): - return 1; + return 1 if path.find("/binary-%s/" % (a)) != -1: - return 1; + return 1 if path.find("/installer-%s/" % (a)) != -1: - return 1; + return 1 if path.find("/source/") != -1: if self.source: - return 1; + return 1 else: - return 0; + return 0 if path.find("/Contents-") != -1: - return 0; + return 0 if path.find("/binary-") != -1: - return 0; + return 0 if path.find("/installer-") != -1: - return 0; - return 1; + return 0 + return 1 ############################################################################## # The applicable function is basically a predicate. Given a path and a @@ -106,26 +106,26 @@ class BillieTarget: ################## verbatim = [ - ]; + ] verbprefix = [ "/tools/", "/README", "/doc/" - ]; + ] def applicable(path, target): if path.startswith("/pool/"): - return target.poolish_match(path); + return target.poolish_match(path) if (path.startswith("/dists/") or path.startswith("/project/experimental/")): - return target.distish_match(path); + return target.distish_match(path) if path in verbatim: - return 1; + return 1 for prefix in verbprefix: if path.startswith(prefix): - return 1; - return 0; + return 1 + return 0 ############################################################################## @@ -138,9 +138,9 @@ def applicable(path, target): class BillieDir: def __init__(self): - self.dirs = {}; - self.files = {}; - self.links = {}; + self.dirs = {} + self.files = {} + self.links = {} ############################################################################## # A BillieDB is a container for a BillieDir... @@ -149,44 +149,44 @@ class BillieDir: class BillieDB: ## Initialise a BillieDB as containing nothing def __init__(self): - self.root = BillieDir(); + self.root = BillieDir() def _internal_recurse(self, path): - bdir = BillieDir(); - dl = os.listdir( path ); - dl.sort(); - dirs = []; + bdir = BillieDir() + dl = os.listdir( path ) + dl.sort() + dirs = [] for ln in dl: - lnl = os.lstat( "%s/%s" % (path, ln) ); + lnl = os.lstat( "%s/%s" % (path, ln) ) if S_ISDIR(lnl[0]): - dirs.append(ln); + dirs.append(ln) elif S_ISLNK(lnl[0]): - bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) ); + bdir.links[ln] = os.readlink( "%s/%s" % (path, ln) ) elif S_ISREG(lnl[0]): - bdir.files[ln] = lnl[1]; + bdir.files[ln] = lnl[1] else: util.fubar( "Confused by %s/%s -- not a dir, link or file" % - ( path, ln ) ); + ( path, ln ) ) for d in dirs: - bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) ); + bdir.dirs[d] = self._internal_recurse( "%s/%s" % (path,d) ) - return bdir; + return bdir ## Recurse through a given path, setting the sequence accordingly def init_from_dir(self, dirp): - self.root = self._internal_recurse( dirp ); + self.root = self._internal_recurse( dirp ) ## Load this BillieDB from file def load_from_file(self, fname): - f = open(fname, "r"); - self.root = cPickle.load(f); - f.close(); + f = open(fname, "r") + self.root = cPickle.load(f) + f.close() ## Save this BillieDB to a file def save_to_file(self, fname): - f = open(fname, "w"); - cPickle.dump( self.root, f, 1 ); - f.close(); + f = open(fname, "w") + cPickle.dump( self.root, f, 1 ) + f.close() ############################################################################## @@ -194,29 +194,29 @@ class BillieDB: ################## def _pth(a,b): - return "%s/%s" % (a,b); + return "%s/%s" % (a,b) def do_mkdir(targ,path): if not os.path.exists( _pth(targ.root, path) ): - os.makedirs( _pth(targ.root, path) ); + os.makedirs( _pth(targ.root, path) ) def do_mkdir_f(targ,path): - do_mkdir(targ, os.path.dirname(path)); + do_mkdir(targ, os.path.dirname(path)) def do_link(targ,path): - do_mkdir_f(targ,path); + do_mkdir_f(targ,path) os.link( _pth(MASTER_PATH, path), - _pth(targ.root, path)); + _pth(targ.root, path)) def do_symlink(targ,path,link): - do_mkdir_f(targ,path); - os.symlink( link, _pth(targ.root, path) ); + do_mkdir_f(targ,path) + os.symlink( link, _pth(targ.root, path) ) def do_unlink(targ,path): - os.unlink( _pth(targ.root, path) ); + os.unlink( _pth(targ.root, path) ) def do_unlink_dir(targ,path): - os.system( "rm -Rf '%s'" % _pth(targ.root, path) ); + os.system( "rm -Rf '%s'" % _pth(targ.root, path) ) ############################################################################## # Reconciling a target with the sourcedb @@ -229,13 +229,13 @@ def _internal_reconcile( path, srcdir, targdir, targ ): for k in targdir.links.keys(): if applicable( _pth(path, k), targ ): if not srcdir.links.has_key(k): - rm.append(k); + rm.append(k) else: - rm.append(k); + rm.append(k) for k in rm: #print "-L-", _pth(path,k) do_unlink(targ, _pth(path,k)) - del targdir.links[k]; + del targdir.links[k] # Remove any files in targdir which aren't in srcdir # Or which aren't applicable @@ -243,37 +243,37 @@ def _internal_reconcile( path, srcdir, targdir, targ ): for k in targdir.files.keys(): if applicable( _pth(path, k), targ ): if not srcdir.files.has_key(k): - rm.append(k); + rm.append(k) else: - rm.append(k); + rm.append(k) for k in rm: #print "-F-", _pth(path,k) do_unlink(targ, _pth(path,k)) - del targdir.files[k]; + del targdir.files[k] # Remove any dirs in targdir which aren't in srcdir rm = [] for k in targdir.dirs.keys(): if not srcdir.dirs.has_key(k): - rm.append(k); + rm.append(k) for k in rm: #print "-D-", _pth(path,k) do_unlink_dir(targ, _pth(path,k)) - del targdir.dirs[k]; + del targdir.dirs[k] # Add/update files for k in srcdir.files.keys(): if applicable( _pth(path,k), targ ): if not targdir.files.has_key(k): #print "+F+", _pth(path,k) - do_link( targ, _pth(path,k) ); - targdir.files[k] = srcdir.files[k]; + do_link( targ, _pth(path,k) ) + targdir.files[k] = srcdir.files[k] else: if targdir.files[k] != srcdir.files[k]: - #print "*F*", _pth(path,k); - do_unlink( targ, _pth(path,k) ); - do_link( targ, _pth(path,k) ); - targdir.files[k] = srcdir.files[k]; + #print "*F*", _pth(path,k) + do_unlink( targ, _pth(path,k) ) + do_link( targ, _pth(path,k) ) + targdir.files[k] = srcdir.files[k] # Add/update links for k in srcdir.links.keys(): @@ -281,25 +281,25 @@ def _internal_reconcile( path, srcdir, targdir, targ ): if not targdir.links.has_key(k): targdir.links[k] = srcdir.links[k]; #print "+L+",_pth(path,k), "->", srcdir.links[k] - do_symlink( targ, _pth(path,k), targdir.links[k] ); + do_symlink( targ, _pth(path,k), targdir.links[k] ) else: if targdir.links[k] != srcdir.links[k]: - do_unlink( targ, _pth(path,k) ); - targdir.links[k] = srcdir.links[k]; + do_unlink( targ, _pth(path,k) ) + targdir.links[k] = srcdir.links[k] #print "*L*", _pth(path,k), "to ->", srcdir.links[k] - do_symlink( targ, _pth(path,k), targdir.links[k] ); + do_symlink( targ, _pth(path,k), targdir.links[k] ) # Do dirs for k in srcdir.dirs.keys(): if not targdir.dirs.has_key(k): - targdir.dirs[k] = BillieDir(); + targdir.dirs[k] = BillieDir() #print "+D+", _pth(path,k) _internal_reconcile( _pth(path,k), srcdir.dirs[k], - targdir.dirs[k], targ ); + targdir.dirs[k], targ ) def reconcile_target_db( src, targ ): - _internal_reconcile( "", src.root, targ.db.root, targ ); + _internal_reconcile( "", src.root, targ.db.root, targ ) ############################################################################### @@ -309,9 +309,9 @@ def load_config(): global TREE_DB_ROOT global trees - MASTER_PATH = Cnf["Billie::FTPPath"]; - TREE_ROOT = Cnf["Billie::TreeRootPath"]; - TREE_DB_ROOT = Cnf["Billie::TreeDatabasePath"]; + MASTER_PATH = Cnf["Billie::FTPPath"] + TREE_ROOT = Cnf["Billie::TreeRootPath"] + TREE_DB_ROOT = Cnf["Billie::TreeDatabasePath"] for a in Cnf.ValueList("Billie::BasicTrees"): trees.append( BillieTarget( a, "%s,all" % a, 1 ) ) @@ -323,7 +323,7 @@ def load_config(): source = 1 archs.remove("source") archs = ",".join(archs) - trees.append( BillieTarget( n, archs, source ) ); + trees.append( BillieTarget( n, archs, source ) ) def do_list (): print "Master path",MASTER_PATH @@ -353,22 +353,22 @@ def main (): Arguments = [('h',"help","Billie::Options::Help"), ('l',"list","Billie::Options::List"), - ]; + ] - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - Cnf["Billie::Options::cake"] = ""; + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) + Cnf["Billie::Options::cake"] = "" Options = Cnf.SubTree("Billie::Options") print "Loading configuration..." - load_config(); + load_config() print "Loaded." if Options.has_key("Help"): - do_help(); - return; + do_help() + return if Options.has_key("List"): - do_list(); - return; + do_list() + return src = BillieDB() @@ -378,9 +378,9 @@ def main (): for tree in trees: print "Reconciling tree:",tree.name - reconcile_target_db( src, tree ); + reconcile_target_db( src, tree ) print "Saving updated DB...", - tree.save_db(); + tree.save_db() print "Done" ############################################################################## diff --git a/dak/override.py b/dak/override.py index 1627474c..f52ece94 100755 --- a/dak/override.py +++ b/dak/override.py @@ -26,23 +26,23 @@ ## That Alisha Rules The World ################################################################################ -import pg, sys; -import utils, db_access; -import apt_pkg, logging; +import pg, sys +import utils, db_access +import apt_pkg, logging ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ # Shamelessly stolen from melanie. Should probably end up in utils.py def game_over(): - answer = utils.our_raw_input("Continue (y/N)? ").lower(); + answer = utils.our_raw_input("Continue (y/N)? ").lower() if answer != "y": print "Aborted." - sys.exit(1); + sys.exit(1) def usage (exit_code=0): @@ -57,7 +57,7 @@ Make microchanges or microqueries of the overrides sys.exit(exit_code) def main (): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() @@ -65,44 +65,44 @@ def main (): ('d',"done","Alicia::Options::Done", "HasArg"), ('n',"no-action","Alicia::Options::No-Action"), ('s',"suite","Alicia::Options::Suite", "HasArg"), - ]; + ] for i in ["help", "no-action"]: if not Cnf.has_key("Alicia::Options::%s" % (i)): - Cnf["Alicia::Options::%s" % (i)] = ""; + Cnf["Alicia::Options::%s" % (i)] = "" if not Cnf.has_key("Alicia::Options::Suite"): - Cnf["Alicia::Options::Suite"] = "unstable"; + Cnf["Alicia::Options::Suite"] = "unstable" - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Alicia::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) if not arguments: - utils.fubar("package name is a required argument."); + utils.fubar("package name is a required argument.") - package = arguments.pop(0); + package = arguments.pop(0) suite = Options["Suite"] if arguments and len(arguments) > 2: - utils.fubar("Too many arguments"); + utils.fubar("Too many arguments") if arguments and len(arguments) == 1: # Determine if the argument is a priority or a section... - arg = arguments.pop(); + arg = arguments.pop() q = projectB.query(""" SELECT ( SELECT COUNT(*) FROM section WHERE section=%s ) AS secs, ( SELECT COUNT(*) FROM priority WHERE priority=%s ) AS prios - """ % ( pg._quote(arg,"str"), pg._quote(arg,"str"))); - r = q.getresult(); + """ % ( pg._quote(arg,"str"), pg._quote(arg,"str"))) + r = q.getresult() if r[0][0] == 1: - arguments = (arg,"."); + arguments = (arg,".") elif r[0][1] == 1: - arguments = (".",arg); + arguments = (".",arg) else: - utils.fubar("%s is not a valid section or priority" % (arg)); + utils.fubar("%s is not a valid section or priority" % (arg)) # Retrieve current section/priority... @@ -114,75 +114,75 @@ def main (): AND override.package = %s AND override.suite = suite.id AND suite.suite_name = %s - """ % (pg._quote(package,"str"), pg._quote(suite,"str"))); + """ % (pg._quote(package,"str"), pg._quote(suite,"str"))) if q.ntuples() == 0: - utils.fubar("Unable to find package %s" % (package)); + utils.fubar("Unable to find package %s" % (package)) if q.ntuples() > 1: - utils.fubar("%s is ambiguous. Matches %d packages" % (package,q.ntuples())); + utils.fubar("%s is ambiguous. Matches %d packages" % (package,q.ntuples())) - r = q.getresult(); - oldsection = r[0][1]; - oldpriority = r[0][0]; + r = q.getresult() + oldsection = r[0][1] + oldpriority = r[0][0] if not arguments: print "%s is in section '%s' at priority '%s'" % ( - package,oldsection,oldpriority); - sys.exit(0); + package,oldsection,oldpriority) + sys.exit(0) # At this point, we have a new section and priority... check they're valid... - newsection, newpriority = arguments; + newsection, newpriority = arguments if newsection == ".": - newsection = oldsection; + newsection = oldsection if newpriority == ".": - newpriority = oldpriority; + newpriority = oldpriority q = projectB.query("SELECT id FROM section WHERE section=%s" % ( - pg._quote(newsection,"str"))); + pg._quote(newsection,"str"))) if q.ntuples() == 0: - utils.fubar("Supplied section %s is invalid" % (newsection)); - newsecid = q.getresult()[0][0]; + utils.fubar("Supplied section %s is invalid" % (newsection)) + newsecid = q.getresult()[0][0] q = projectB.query("SELECT id FROM priority WHERE priority=%s" % ( - pg._quote(newpriority,"str"))); + pg._quote(newpriority,"str"))) if q.ntuples() == 0: - utils.fubar("Supplied priority %s is invalid" % (newpriority)); - newprioid = q.getresult()[0][0]; + utils.fubar("Supplied priority %s is invalid" % (newpriority)) + newprioid = q.getresult()[0][0] if newpriority == oldpriority and newsection == oldsection: print "I: Doing nothing" - sys.exit(0); + sys.exit(0) # If we're in no-action mode if Options["No-Action"]: if newpriority != oldpriority: - print "I: Would change priority from %s to %s" % (oldpriority,newpriority); + print "I: Would change priority from %s to %s" % (oldpriority,newpriority) if newsection != oldsection: - print "I: Would change section from %s to %s" % (oldsection,newsection); + print "I: Would change section from %s to %s" % (oldsection,newsection) if Options.has_key("Done"): - print "I: Would also close bug(s): %s" % (Options["Done"]); + print "I: Would also close bug(s): %s" % (Options["Done"]) - sys.exit(0); + sys.exit(0) if newpriority != oldpriority: - print "I: Will change priority from %s to %s" % (oldpriority,newpriority); + print "I: Will change priority from %s to %s" % (oldpriority,newpriority) if newsection != oldsection: - print "I: Will change section from %s to %s" % (oldsection,newsection); + print "I: Will change section from %s to %s" % (oldsection,newsection) if not Options.has_key("Done"): - pass; - #utils.warn("No bugs to close have been specified. Noone will know you have done this."); + pass + #utils.warn("No bugs to close have been specified. Noone will know you have done this.") else: - print "I: Will close bug(s): %s" % (Options["Done"]); + print "I: Will close bug(s): %s" % (Options["Done"]) - game_over(); + game_over() - Logger = logging.Logger(Cnf, "alicia"); + Logger = logging.Logger(Cnf, "alicia") - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") # We're in "do it" mode, we have something to do... do it if newpriority != oldpriority: q = projectB.query(""" @@ -192,8 +192,8 @@ def main (): AND suite = (SELECT id FROM suite WHERE suite_name=%s)""" % ( newprioid, pg._quote(package,"str"), - pg._quote(suite,"str") )); - Logger.log(["changed priority",package,oldpriority,newpriority]); + pg._quote(suite,"str") )) + Logger.log(["changed priority",package,oldpriority,newpriority]) if newsection != oldsection: q = projectB.query(""" @@ -203,46 +203,46 @@ def main (): AND suite = (SELECT id FROM suite WHERE suite_name=%s)""" % ( newsecid, pg._quote(package,"str"), - pg._quote(suite,"str") )); - Logger.log(["changed priority",package,oldsection,newsection]); - projectB.query("COMMIT WORK"); + pg._quote(suite,"str") )) + Logger.log(["changed priority",package,oldsection,newsection]) + projectB.query("COMMIT WORK") if Options.has_key("Done"): - Subst = {}; - Subst["__ALICIA_ADDRESS__"] = Cnf["Alicia::MyEmailAddress"]; - Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]; - bcc = []; + Subst = {} + Subst["__ALICIA_ADDRESS__"] = Cnf["Alicia::MyEmailAddress"] + Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"] + bcc = [] if Cnf.Find("Dinstall::Bcc") != "": - bcc.append(Cnf["Dinstall::Bcc"]); + bcc.append(Cnf["Dinstall::Bcc"]) if Cnf.Find("Alicia::Bcc") != "": - bcc.append(Cnf["Alicia::Bcc"]); + bcc.append(Cnf["Alicia::Bcc"]) if bcc: - Subst["__BCC__"] = "Bcc: " + ", ".join(bcc); + Subst["__BCC__"] = "Bcc: " + ", ".join(bcc) else: - Subst["__BCC__"] = "X-Filler: 42"; - Subst["__CC__"] = "X-Katie: alicia $Revision: 1.6 $"; - Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]; - Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]; - Subst["__WHOAMI__"] = utils.whoami(); - - summary = "Concerning package %s...\n" % (package); - summary += "Operating on the %s suite\n" % (suite); + Subst["__BCC__"] = "X-Filler: 42" + Subst["__CC__"] = "X-Katie: alicia $Revision: 1.6 $" + Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"] + Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"] + Subst["__WHOAMI__"] = utils.whoami() + + summary = "Concerning package %s...\n" % (package) + summary += "Operating on the %s suite\n" % (suite) if newpriority != oldpriority: - summary += "Changed priority from %s to %s\n" % (oldpriority,newpriority); + summary += "Changed priority from %s to %s\n" % (oldpriority,newpriority) if newsection != oldsection: - summary += "Changed section from %s to %s\n" % (oldsection,newsection); - Subst["__SUMMARY__"] = summary; + summary += "Changed section from %s to %s\n" % (oldsection,newsection) + Subst["__SUMMARY__"] = summary for bug in utils.split_args(Options["Done"]): - Subst["__BUG_NUMBER__"] = bug; + Subst["__BUG_NUMBER__"] = bug mail_message = utils.TemplateSubst( - Subst,Cnf["Dir::Templates"]+"/alicia.bug-close"); - utils.send_mail(mail_message); - Logger.log(["closed bug",bug]); + Subst,Cnf["Dir::Templates"]+"/alicia.bug-close") + utils.send_mail(mail_message) + Logger.log(["closed bug",bug]) - Logger.close(); + Logger.close() - print "Done"; + print "Done" ################################################################################# diff --git a/dak/poolize.py b/dak/poolize.py index 66efca2d..66fe3e0b 100755 --- a/dak/poolize.py +++ b/dak/poolize.py @@ -26,16 +26,16 @@ ################################################################################ -import os, pg, re, stat, sys; -import utils, db_access; -import apt_pkg, apt_inst; +import os, pg, re, stat, sys +import utils, db_access +import apt_pkg, apt_inst ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None -re_isadeb = re.compile (r"(.+?)_(.+?)(_(.+))?\.u?deb$"); +re_isadeb = re.compile (r"(.+?)_(.+?)(_(.+))?\.u?deb$") ################################################################################ @@ -63,114 +63,114 @@ Migrate packages from legacy locations into the pool. # verbose and no_action are booleans def poolize (q, limit, verbose, no_action): - poolized_size = 0L; - poolized_count = 0; + poolized_size = 0L + poolized_count = 0 # Parse -l/--limit argument - qd = q.dictresult(); + qd = q.dictresult() for qid in qd: - legacy_filename = qid["path"]+qid["filename"]; - size = os.stat(legacy_filename)[stat.ST_SIZE]; + legacy_filename = qid["path"]+qid["filename"] + size = os.stat(legacy_filename)[stat.ST_SIZE] if (poolized_size + size) > limit and limit >= 0: - utils.warn("Hit %s limit." % (utils.size_type(limit))); - break; - poolized_size += size; - poolized_count += 1; - base_filename = os.path.basename(legacy_filename); - destination_filename = base_filename; + utils.warn("Hit %s limit." % (utils.size_type(limit))) + break + poolized_size += size + poolized_count += 1 + base_filename = os.path.basename(legacy_filename) + destination_filename = base_filename # Work out the source package name if re_isadeb.match(base_filename): control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(legacy_filename))) - package = control.Find("Package", ""); - source = control.Find("Source", package); + package = control.Find("Package", "") + source = control.Find("Source", package) if source.find("(") != -1: m = utils.re_extract_src_version.match(source) source = m.group(1) # If it's a binary, we need to also rename the file to include the architecture - version = control.Find("Version", ""); - architecture = control.Find("Architecture", ""); + version = control.Find("Version", "") + architecture = control.Find("Architecture", "") if package == "" or version == "" or architecture == "": - utils.fubar("%s: couldn't determine required information to rename .deb file." % (legacy_filename)); - version = utils.re_no_epoch.sub('', version); - destination_filename = "%s_%s_%s.deb" % (package, version, architecture); + utils.fubar("%s: couldn't determine required information to rename .deb file." % (legacy_filename)) + version = utils.re_no_epoch.sub('', version) + destination_filename = "%s_%s_%s.deb" % (package, version, architecture) else: m = utils.re_issource.match(base_filename) if m: - source = m.group(1); + source = m.group(1) else: - utils.fubar("expansion of source filename '%s' failed." % (legacy_filename)); + utils.fubar("expansion of source filename '%s' failed." % (legacy_filename)) # Work out the component name - component = qid["component"]; + component = qid["component"] if component == "": - q = projectB.query("SELECT DISTINCT(c.name) FROM override o, component c WHERE o.package = '%s' AND o.component = c.id;" % (source)); - ql = q.getresult(); + q = projectB.query("SELECT DISTINCT(c.name) FROM override o, component c WHERE o.package = '%s' AND o.component = c.id;" % (source)) + ql = q.getresult() if not ql: - utils.fubar("No override match for '%s' so I can't work out the component." % (source)); + utils.fubar("No override match for '%s' so I can't work out the component." % (source)) if len(ql) > 1: - utils.fubar("Multiple override matches for '%s' so I can't work out the component." % (source)); - component = ql[0][0]; + utils.fubar("Multiple override matches for '%s' so I can't work out the component." % (source)) + component = ql[0][0] # Work out the new location - q = projectB.query("SELECT l.id FROM location l, component c WHERE c.name = '%s' AND c.id = l.component AND l.type = 'pool';" % (component)); - ql = q.getresult(); + q = projectB.query("SELECT l.id FROM location l, component c WHERE c.name = '%s' AND c.id = l.component AND l.type = 'pool';" % (component)) + ql = q.getresult() if len(ql) != 1: - utils.fubar("couldn't determine location ID for '%s'. [query returned %d matches, not 1 as expected]" % (source, len(ql))); - location_id = ql[0][0]; + utils.fubar("couldn't determine location ID for '%s'. [query returned %d matches, not 1 as expected]" % (source, len(ql))) + location_id = ql[0][0] # First move the files to the new location - pool_location = utils.poolify (source, component); - pool_filename = pool_location + destination_filename; - destination = Cnf["Dir::Pool"] + pool_location + destination_filename; + pool_location = utils.poolify (source, component) + pool_filename = pool_location + destination_filename + destination = Cnf["Dir::Pool"] + pool_location + destination_filename if os.path.exists(destination): - utils.fubar("'%s' already exists in the pool; serious FUBARity." % (legacy_filename)); + utils.fubar("'%s' already exists in the pool; serious FUBARity." % (legacy_filename)) if verbose: - print "Moving: %s -> %s" % (legacy_filename, destination); + print "Moving: %s -> %s" % (legacy_filename, destination) if not no_action: - utils.move(legacy_filename, destination); + utils.move(legacy_filename, destination) # Then Update the DB's files table if verbose: - print "SQL: UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, location_id, qid["files_id"]); + print "SQL: UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, location_id, qid["files_id"]) if not no_action: - q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, location_id, qid["files_id"])); + q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, location_id, qid["files_id"])) - sys.stderr.write("Poolized %s in %s files.\n" % (utils.size_type(poolized_size), poolized_count)); + sys.stderr.write("Poolized %s in %s files.\n" % (utils.size_type(poolized_size), poolized_count)) ################################################################################ def main (): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() for i in ["help", "limit", "no-action", "verbose" ]: if not Cnf.has_key("Catherine::Options::%s" % (i)): - Cnf["Catherine::Options::%s" % (i)] = ""; + Cnf["Catherine::Options::%s" % (i)] = "" Arguments = [('h',"help","Catherine::Options::Help"), ('l',"limit", "Catherine::Options::Limit", "HasArg"), ('n',"no-action","Catherine::Options::No-Action"), - ('v',"verbose","Catherine::Options::Verbose")]; + ('v',"verbose","Catherine::Options::Verbose")] - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Catherine::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) if not Options["Limit"]: - limit = -1; + limit = -1 else: - limit = int(Options["Limit"]) * 1024; + limit = int(Options["Limit"]) * 1024 # -n/--no-action implies -v/--verbose if Options["No-Action"]: - Options["Verbose"] = "true"; + Options["Verbose"] = "true" # Sanity check the limit argument if limit > 0 and limit < 1024: - utils.fubar("-l/--limit takes an argument with a value in kilobytes."); + utils.fubar("-l/--limit takes an argument with a value in kilobytes.") # Grab a list of all files not already in the pool q = projectB.query(""" @@ -182,9 +182,9 @@ UNION SELECT l.path, f.filename, f.id as files_id, null as component FROM files f, location l WHERE NOT EXISTS (SELECT 1 FROM location l WHERE l.type = 'pool' AND f.location = l.id) AND NOT (f.filename ~ '^potato') AND f.location = l.id AND NOT EXISTS - (SELECT 1 FROM location l WHERE l.component IS NOT NULL AND f.location = l.id);"""); + (SELECT 1 FROM location l WHERE l.component IS NOT NULL AND f.location = l.id);""") - poolize(q, limit, Options["Verbose"], Options["No-Action"]); + poolize(q, limit, Options["Verbose"], Options["No-Action"]) ####################################################################################### diff --git a/dak/process_accepted.py b/dak/process_accepted.py index d28d1ace..8e48ae1f 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -30,34 +30,34 @@ ############################################################################### -import errno, fcntl, os, sys, time, re; -import apt_pkg; -import db_access, katie, logging, utils; +import errno, fcntl, os, sys, time, re +import apt_pkg +import db_access, katie, logging, utils ############################################################################### # Globals -kelly_version = "$Revision: 1.18 $"; +kelly_version = "$Revision: 1.18 $" -Cnf = None; -Options = None; -Logger = None; -Urgency_Logger = None; -projectB = None; -Katie = None; -pkg = None; +Cnf = None +Options = None +Logger = None +Urgency_Logger = None +projectB = None +Katie = None +pkg = None -reject_message = ""; -changes = None; -dsc = None; -dsc_files = None; -files = None; -Subst = None; +reject_message = "" +changes = None +dsc = None +dsc_files = None +files = None +Subst = None -install_count = 0; -install_bytes = 0.0; +install_count = 0 +install_bytes = 0.0 -installing_to_stable = 0; +installing_to_stable = 0 ############################################################################### @@ -68,40 +68,40 @@ class Urgency_Log: "Urgency Logger object" def __init__ (self, Cnf): "Initialize a new Urgency Logger object" - self.Cnf = Cnf; - self.timestamp = time.strftime("%Y%m%d%H%M%S"); + self.Cnf = Cnf + self.timestamp = time.strftime("%Y%m%d%H%M%S") # Create the log directory if it doesn't exist - self.log_dir = Cnf["Dir::UrgencyLog"]; + self.log_dir = Cnf["Dir::UrgencyLog"] if not os.path.exists(self.log_dir): - umask = os.umask(00000); - os.makedirs(self.log_dir, 02775); + umask = os.umask(00000) + os.makedirs(self.log_dir, 02775) # Open the logfile - self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp); - self.log_file = utils.open_file(self.log_filename, 'w'); - self.writes = 0; + self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp) + self.log_file = utils.open_file(self.log_filename, 'w') + self.writes = 0 def log (self, source, version, urgency): "Log an event" - self.log_file.write(" ".join([source, version, urgency])+'\n'); - self.log_file.flush(); - self.writes += 1; + self.log_file.write(" ".join([source, version, urgency])+'\n') + self.log_file.flush() + self.writes += 1 def close (self): "Close a Logger object" - self.log_file.flush(); - self.log_file.close(); + self.log_file.flush() + self.log_file.close() if self.writes: - new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp); - utils.move(self.log_filename, new_filename); + new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp) + utils.move(self.log_filename, new_filename) else: - os.unlink(self.log_filename); + os.unlink(self.log_filename) ############################################################################### def reject (str, prefix="Rejected: "): - global reject_message; + global reject_message if str: - reject_message += prefix + str + "\n"; + reject_message += prefix + str + "\n" # Recheck anything that relies on the database; since that's not # frozen between accept and our run time. @@ -113,23 +113,23 @@ def check(): # The .orig.tar.gz can disappear out from under us is it's a # duplicate of one in the archive. if not files.has_key(file): - continue; + continue # Check that the source still exists if files[file]["type"] == "deb": - source_version = files[file]["source version"]; - source_package = files[file]["source package"]; + source_version = files[file]["source version"] + source_package = files[file]["source package"] if not changes["architecture"].has_key("source") \ and not Katie.source_exists(source_package, source_version, changes["distribution"].keys()): - reject("no source found for %s %s (%s)." % (source_package, source_version, file)); + reject("no source found for %s %s (%s)." % (source_package, source_version, file)) # Version and file overwrite checks if not installing_to_stable: if files[file]["type"] == "deb": - reject(Katie.check_binary_against_db(file), ""); + reject(Katie.check_binary_against_db(file), "") elif files[file]["type"] == "dsc": - reject(Katie.check_source_against_db(file), ""); - (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); - reject(reject_msg, ""); + reject(Katie.check_source_against_db(file), "") + (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file) + reject(reject_msg, "") # propogate in the case it is in the override tables: if changes.has_key("propdistribution"): @@ -148,12 +148,12 @@ def check(): # Check the package is still in the override tables for suite in changes["distribution"].keys(): if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): - reject("%s is NEW for %s." % (file, suite)); + reject("%s is NEW for %s." % (file, suite)) ############################################################################### def init(): - global Cnf, Options, Katie, projectB, changes, dsc, dsc_files, files, pkg, Subst; + global Cnf, Options, Katie, projectB, changes, dsc, dsc_files, files, pkg, Subst Cnf = utils.get_conf() @@ -162,33 +162,33 @@ def init(): ('n',"no-action","Dinstall::Options::No-Action"), ('p',"no-lock", "Dinstall::Options::No-Lock"), ('s',"no-mail", "Dinstall::Options::No-Mail"), - ('V',"version","Dinstall::Options::Version")]; + ('V',"version","Dinstall::Options::Version")] for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version"]: if not Cnf.has_key("Dinstall::Options::%s" % (i)): - Cnf["Dinstall::Options::%s" % (i)] = ""; + Cnf["Dinstall::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Dinstall::Options") if Options["Help"]: - usage(); + usage() if Options["Version"]: - print "kelly %s" % (kelly_version); - sys.exit(0); + print "kelly %s" % (kelly_version) + sys.exit(0) - Katie = katie.Katie(Cnf); - projectB = Katie.projectB; + Katie = katie.Katie(Cnf) + projectB = Katie.projectB - changes = Katie.pkg.changes; - dsc = Katie.pkg.dsc; - dsc_files = Katie.pkg.dsc_files; - files = Katie.pkg.files; - pkg = Katie.pkg; - Subst = Katie.Subst; + changes = Katie.pkg.changes + dsc = Katie.pkg.dsc + dsc_files = Katie.pkg.dsc_files + files = Katie.pkg.files + pkg = Katie.pkg + Subst = Katie.Subst - return changes_files; + return changes_files ############################################################################### @@ -205,38 +205,38 @@ def usage (exit_code=0): ############################################################################### def action (): - (summary, short_summary) = Katie.build_summaries(); + (summary, short_summary) = Katie.build_summaries() (prompt, answer) = ("", "XXX") if Options["No-Action"] or Options["Automatic"]: answer = 'S' if reject_message.find("Rejected") != -1: - print "REJECT\n" + reject_message,; - prompt = "[R]eject, Skip, Quit ?"; + print "REJECT\n" + reject_message, + prompt = "[R]eject, Skip, Quit ?" if Options["Automatic"]: - answer = 'R'; + answer = 'R' else: print "INSTALL to " + ", ".join(changes["distribution"].keys()) - print reject_message + summary,; - prompt = "[I]nstall, Skip, Quit ?"; + print reject_message + summary, + prompt = "[I]nstall, Skip, Quit ?" if Options["Automatic"]: - answer = 'I'; + answer = 'I' while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.match(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.match(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); + answer = m.group(1) + answer = answer[:1].upper() if answer == 'R': - do_reject (); + do_reject () elif answer == 'I': if not installing_to_stable: - install(); + install() else: - stable_install(summary, short_summary); + stable_install(summary, short_summary) elif answer == 'Q': sys.exit(0) @@ -248,36 +248,36 @@ def action (): # folks... def do_reject (): - Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]; - Subst["__REJECT_MESSAGE__"] = reject_message; - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; - reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/kelly.unaccept"); + Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] + Subst["__REJECT_MESSAGE__"] = reject_message + Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/kelly.unaccept") # Write the rejection email out as the .reason file - reason_filename = os.path.basename(pkg.changes_file[:-8]) + ".reason"; - reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename; + reason_filename = os.path.basename(pkg.changes_file[:-8]) + ".reason" + reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename # If we fail here someone is probably trying to exploit the race # so let's just raise an exception ... if os.path.exists(reject_filename): - os.unlink(reject_filename); - fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); - os.write(fd, reject_mail_message); - os.close(fd); + os.unlink(reject_filename) + fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) + os.write(fd, reject_mail_message) + os.close(fd) - utils.send_mail(reject_mail_message); - Logger.log(["unaccepted", pkg.changes_file]); + utils.send_mail(reject_mail_message) + Logger.log(["unaccepted", pkg.changes_file]) ############################################################################### def install (): - global install_count, install_bytes; + global install_count, install_bytes print "Installing." - Logger.log(["installing changes",pkg.changes_file]); + Logger.log(["installing changes",pkg.changes_file]) # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed. - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") # Add the .dsc file to the DB for file in files.keys(): @@ -286,34 +286,34 @@ def install (): version = dsc["version"] # NB: not files[file]["version"], that has no epoch maintainer = dsc["maintainer"] maintainer = maintainer.replace("'", "\\'") - maintainer_id = db_access.get_or_set_maintainer_id(maintainer); - fingerprint_id = db_access.get_or_set_fingerprint_id(dsc["fingerprint"]); - install_date = time.strftime("%Y-%m-%d"); - filename = files[file]["pool name"] + file; - dsc_component = files[file]["component"]; - dsc_location_id = files[file]["location id"]; + maintainer_id = db_access.get_or_set_maintainer_id(maintainer) + fingerprint_id = db_access.get_or_set_fingerprint_id(dsc["fingerprint"]) + install_date = time.strftime("%Y-%m-%d") + filename = files[file]["pool name"] + file + dsc_component = files[file]["component"] + dsc_location_id = files[file]["location id"] if not files[file].has_key("files id") or not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id) projectB.query("INSERT INTO source (source, version, maintainer, file, install_date, sig_fpr) VALUES ('%s', '%s', %d, %d, '%s', %s)" - % (package, version, maintainer_id, files[file]["files id"], install_date, fingerprint_id)); + % (package, version, maintainer_id, files[file]["files id"], install_date, fingerprint_id)) for suite in changes["distribution"].keys(): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id)) # Add the source files to the DB (files and dsc_files) - projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"])); + projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"])) for dsc_file in dsc_files.keys(): - filename = files[file]["pool name"] + dsc_file; + filename = files[file]["pool name"] + dsc_file # If the .orig.tar.gz is already in the pool, it's # files id is stored in dsc_files by check_dsc(). - files_id = dsc_files[dsc_file].get("files id", None); + files_id = dsc_files[dsc_file].get("files id", None) if files_id == None: - files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id); + files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id) # FIXME: needs to check for -1/-2 and or handle exception if files_id == None: - files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id); - projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id)); + files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id) + projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id)) # Add the .deb files to the DB for file in files.keys(): @@ -322,50 +322,50 @@ def install (): version = files[file]["version"] maintainer = files[file]["maintainer"] maintainer = maintainer.replace("'", "\\'") - maintainer_id = db_access.get_or_set_maintainer_id(maintainer); - fingerprint_id = db_access.get_or_set_fingerprint_id(changes["fingerprint"]); + maintainer_id = db_access.get_or_set_maintainer_id(maintainer) + fingerprint_id = db_access.get_or_set_fingerprint_id(changes["fingerprint"]) architecture = files[file]["architecture"] - architecture_id = db_access.get_architecture_id (architecture); - type = files[file]["dbtype"]; + architecture_id = db_access.get_architecture_id (architecture) + type = files[file]["dbtype"] source = files[file]["source package"] - source_version = files[file]["source version"]; - filename = files[file]["pool name"] + file; + source_version = files[file]["source version"] + filename = files[file]["pool name"] + file if not files[file].has_key("location id") or not files[file]["location id"]: - files[file]["location id"] = db_access.get_location_id(Cnf["Dir::Pool"],files[file]["component"],utils.where_am_i()); + files[file]["location id"] = db_access.get_location_id(Cnf["Dir::Pool"],files[file]["component"],utils.where_am_i()) if not files[file].has_key("files id") or not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"]) - source_id = db_access.get_source_id (source, source_version); + source_id = db_access.get_source_id (source, source_version) if source_id: projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, %d, '%s', %d)" - % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type, fingerprint_id)); + % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type, fingerprint_id)) else: projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, '%s', %d)" - % (package, version, maintainer_id, architecture_id, files[file]["files id"], type, fingerprint_id)); + % (package, version, maintainer_id, architecture_id, files[file]["files id"], type, fingerprint_id)) for suite in changes["distribution"].keys(): - suite_id = db_access.get_suite_id(suite); - projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id)); + suite_id = db_access.get_suite_id(suite) + projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id)) # If the .orig.tar.gz is in a legacy directory we need to poolify # it, so that apt-get source (and anything else that goes by the # "Directory:" field in the Sources.gz file) works. - orig_tar_id = Katie.pkg.orig_tar_id; - orig_tar_location = Katie.pkg.orig_tar_location; - legacy_source_untouchable = Katie.pkg.legacy_source_untouchable; + orig_tar_id = Katie.pkg.orig_tar_id + orig_tar_location = Katie.pkg.orig_tar_location + legacy_source_untouchable = Katie.pkg.legacy_source_untouchable if orig_tar_id and orig_tar_location == "legacy": - q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id)); - qd = q.dictresult(); + q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id)) + qd = q.dictresult() for qid in qd: # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details) if legacy_source_untouchable.has_key(qid["files_id"]): - continue; + continue # First move the files to the new location - legacy_filename = qid["path"] + qid["filename"]; - pool_location = utils.poolify (changes["source"], files[file]["component"]); - pool_filename = pool_location + os.path.basename(qid["filename"]); + legacy_filename = qid["path"] + qid["filename"] + pool_location = utils.poolify (changes["source"], files[file]["component"]) + pool_filename = pool_location + os.path.basename(qid["filename"]) destination = Cnf["Dir::Pool"] + pool_location - utils.move(legacy_filename, destination); + utils.move(legacy_filename, destination) # Then Update the DB's files table - q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"])); + q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"])) # If this is a sourceful diff only upload that is moving non-legacy # cross-component we need to copy the .orig.tar.gz into the new @@ -373,280 +373,280 @@ def install (): # if changes["architecture"].has_key("source") and orig_tar_id and \ orig_tar_location != "legacy" and orig_tar_location != dsc_location_id: - q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id)); - ql = q.getresult()[0]; - old_filename = ql[0] + ql[1]; - file_size = ql[2]; - file_md5sum = ql[3]; - new_filename = utils.poolify(changes["source"], dsc_component) + os.path.basename(old_filename); - new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id); + q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id)) + ql = q.getresult()[0] + old_filename = ql[0] + ql[1] + file_size = ql[2] + file_md5sum = ql[3] + new_filename = utils.poolify(changes["source"], dsc_component) + os.path.basename(old_filename) + new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id) if new_files_id == None: - utils.copy(old_filename, Cnf["Dir::Pool"] + new_filename); - new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id); - projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id)); + utils.copy(old_filename, Cnf["Dir::Pool"] + new_filename) + new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id) + projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id)) # Install the files into the pool for file in files.keys(): - destination = Cnf["Dir::Pool"] + files[file]["pool name"] + file; - utils.move(file, destination); - Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]); - install_bytes += float(files[file]["size"]); + destination = Cnf["Dir::Pool"] + files[file]["pool name"] + file + utils.move(file, destination) + Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]) + install_bytes += float(files[file]["size"]) # Copy the .changes file across for suite which need it. - copy_changes = {}; - copy_katie = {}; + copy_changes = {} + copy_katie = {} for suite in changes["distribution"].keys(): if Cnf.has_key("Suite::%s::CopyChanges" % (suite)): - copy_changes[Cnf["Suite::%s::CopyChanges" % (suite)]] = ""; + copy_changes[Cnf["Suite::%s::CopyChanges" % (suite)]] = "" # and the .katie file... if Cnf.has_key("Suite::%s::CopyKatie" % (suite)): - copy_katie[Cnf["Suite::%s::CopyKatie" % (suite)]] = ""; + copy_katie[Cnf["Suite::%s::CopyKatie" % (suite)]] = "" for dest in copy_changes.keys(): - utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest); + utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest) for dest in copy_katie.keys(): - utils.copy(Katie.pkg.changes_file[:-8]+".katie", dest); + utils.copy(Katie.pkg.changes_file[:-8]+".katie", dest) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") # Move the .changes into the 'done' directory utils.move (pkg.changes_file, - os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file))); + os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file))) # Remove the .katie file - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); + os.unlink(Katie.pkg.changes_file[:-8]+".katie") if changes["architecture"].has_key("source") and Urgency_Logger: - Urgency_Logger.log(dsc["source"], dsc["version"], changes["urgency"]); + Urgency_Logger.log(dsc["source"], dsc["version"], changes["urgency"]) # Undo the work done in katie.py(accept) to help auto-building # from accepted. - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for suite in changes["distribution"].keys(): if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"): - continue; - now_date = time.strftime("%Y-%m-%d %H:%M"); - suite_id = db_access.get_suite_id(suite); - dest_dir = Cnf["Dir::QueueBuild"]; + continue + now_date = time.strftime("%Y-%m-%d %H:%M") + suite_id = db_access.get_suite_id(suite) + dest_dir = Cnf["Dir::QueueBuild"] if Cnf.FindB("Dinstall::SecurityQueueBuild"): - dest_dir = os.path.join(dest_dir, suite); + dest_dir = os.path.join(dest_dir, suite) for file in files.keys(): - dest = os.path.join(dest_dir, file); + dest = os.path.join(dest_dir, file) # Remove it from the list of packages for later processing by apt-ftparchive - projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, dest, suite_id)); + projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, dest, suite_id)) if not Cnf.FindB("Dinstall::SecurityQueueBuild"): # Update the symlink to point to the new location in the pool - pool_location = utils.poolify (changes["source"], files[file]["component"]); - src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(file)); + pool_location = utils.poolify (changes["source"], files[file]["component"]) + src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(file)) if os.path.islink(dest): - os.unlink(dest); - os.symlink(src, dest); + os.unlink(dest) + os.symlink(src, dest) # Update last_used on any non-upload .orig.tar.gz symlink if orig_tar_id: # Determine the .orig.tar.gz file name for dsc_file in dsc_files.keys(): if dsc_file.endswith(".orig.tar.gz"): - orig_tar_gz = os.path.join(dest_dir, dsc_file); + orig_tar_gz = os.path.join(dest_dir, dsc_file) # Remove it from the list of packages for later processing by apt-ftparchive - projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, orig_tar_gz, suite_id)); - projectB.query("COMMIT WORK"); + projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, orig_tar_gz, suite_id)) + projectB.query("COMMIT WORK") # Finally... - install_count += 1; + install_count += 1 ################################################################################ def stable_install (summary, short_summary): - global install_count; + global install_count - print "Installing to stable."; + print "Installing to stable." # Begin a transaction; if we bomb out anywhere between here and # the COMMIT WORK below, the DB won't be changed. - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") # Add the source to stable (and remove it from proposed-updates) for file in files.keys(): if files[file]["type"] == "dsc": - package = dsc["source"]; + package = dsc["source"] version = dsc["version"]; # NB: not files[file]["version"], that has no epoch q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version)) - ql = q.getresult(); + ql = q.getresult() if not ql: - utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version)); - source_id = ql[0][0]; - suite_id = db_access.get_suite_id('proposed-updates'); - projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)); - suite_id = db_access.get_suite_id('stable'); - projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id)); + utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version)) + source_id = ql[0][0] + suite_id = db_access.get_suite_id('proposed-updates') + projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)) + suite_id = db_access.get_suite_id('stable') + projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id)) # Add the binaries to stable (and remove it/them from proposed-updates) for file in files.keys(): if files[file]["type"] == "deb": binNMU = 0 - package = files[file]["package"]; - version = files[file]["version"]; - architecture = files[file]["architecture"]; - q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)); - ql = q.getresult(); + package = files[file]["package"] + version = files[file]["version"] + architecture = files[file]["architecture"] + q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) + ql = q.getresult() if not ql: - suite_id = db_access.get_suite_id('proposed-updates'); - que = "SELECT b.version FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN suite su ON (ba.suite = su.id) WHERE b.package = '%s' AND (ba.suite = '%s')" % (package, suite_id); + suite_id = db_access.get_suite_id('proposed-updates') + que = "SELECT b.version FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN suite su ON (ba.suite = su.id) WHERE b.package = '%s' AND (ba.suite = '%s')" % (package, suite_id) q = projectB.query(que) # Reduce the query results to a list of version numbers - ql = map(lambda x: x[0], q.getresult()); + ql = map(lambda x: x[0], q.getresult()) if not ql: - utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture)); + utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture)) else: for x in ql: if re.match(re.compile(r"%s((\.0)?\.)|(\+b)\d+$" % re.escape(version)),x): binNMU = 1 break if not binNMU: - binary_id = ql[0][0]; - suite_id = db_access.get_suite_id('proposed-updates'); - projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)); - suite_id = db_access.get_suite_id('stable'); - projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id)); + binary_id = ql[0][0] + suite_id = db_access.get_suite_id('proposed-updates') + projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)) + suite_id = db_access.get_suite_id('stable') + projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id)) else: del files[file] - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") - utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(pkg.changes_file)); + utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(pkg.changes_file)) ## Update the Stable ChangeLog file - new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog"; - changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog"; + new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog" + changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog" if os.path.exists(new_changelog_filename): - os.unlink (new_changelog_filename); + os.unlink (new_changelog_filename) - new_changelog = utils.open_file(new_changelog_filename, 'w'); + new_changelog = utils.open_file(new_changelog_filename, 'w') for file in files.keys(): if files[file]["type"] == "deb": - new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file)); + new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file)) elif utils.re_issource.match(file): - new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file)); + new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file)) else: - new_changelog.write("%s\n" % (file)); - chop_changes = katie.re_fdnic.sub("\n", changes["changes"]); - new_changelog.write(chop_changes + '\n\n'); + new_changelog.write("%s\n" % (file)) + chop_changes = katie.re_fdnic.sub("\n", changes["changes"]) + new_changelog.write(chop_changes + '\n\n') if os.access(changelog_filename, os.R_OK) != 0: - changelog = utils.open_file(changelog_filename); - new_changelog.write(changelog.read()); - new_changelog.close(); + changelog = utils.open_file(changelog_filename) + new_changelog.write(changelog.read()) + new_changelog.close() if os.access(changelog_filename, os.R_OK) != 0: - os.unlink(changelog_filename); - utils.move(new_changelog_filename, changelog_filename); + os.unlink(changelog_filename) + utils.move(new_changelog_filename, changelog_filename) - install_count += 1; + install_count += 1 if not Options["No-Mail"] and changes["architecture"].has_key("source"): - Subst["__SUITE__"] = " into stable"; - Subst["__SUMMARY__"] = summary; - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/kelly.installed"); - utils.send_mail(mail_message); + Subst["__SUITE__"] = " into stable" + Subst["__SUMMARY__"] = summary + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/kelly.installed") + utils.send_mail(mail_message) Katie.announce(short_summary, 1) # Finally remove the .katie file - katie_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyKatie"], os.path.basename(Katie.pkg.changes_file[:-8]+".katie")); - os.unlink(katie_file); + katie_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyKatie"], os.path.basename(Katie.pkg.changes_file[:-8]+".katie")) + os.unlink(katie_file) ################################################################################ def process_it (changes_file): - global reject_message; + global reject_message - reject_message = ""; + reject_message = "" # Absolutize the filename to avoid the requirement of being in the # same directory as the .changes file. - pkg.changes_file = os.path.abspath(changes_file); + pkg.changes_file = os.path.abspath(changes_file) - # And since handling of installs to stable munges with the CWD; + # And since handling of installs to stable munges with the CWD # save and restore it. - pkg.directory = os.getcwd(); + pkg.directory = os.getcwd() if installing_to_stable: - old = Katie.pkg.changes_file; - Katie.pkg.changes_file = os.path.basename(old); - os.chdir(Cnf["Suite::Proposed-Updates::CopyKatie"]); + old = Katie.pkg.changes_file + Katie.pkg.changes_file = os.path.basename(old) + os.chdir(Cnf["Suite::Proposed-Updates::CopyKatie"]) - Katie.init_vars(); - Katie.update_vars(); - Katie.update_subst(); + Katie.init_vars() + Katie.update_vars() + Katie.update_subst() if installing_to_stable: - Katie.pkg.changes_file = old; + Katie.pkg.changes_file = old - check(); - action(); + check() + action() # Restore CWD - os.chdir(pkg.directory); + os.chdir(pkg.directory) ############################################################################### def main(): - global projectB, Logger, Urgency_Logger, installing_to_stable; + global projectB, Logger, Urgency_Logger, installing_to_stable - changes_files = init(); + changes_files = init() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: - Options["Automatic"] = ""; + Options["Automatic"] = "" # Check that we aren't going to clash with the daily cron job if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: - utils.fubar("Archive maintenance in progress. Try again later."); + utils.fubar("Archive maintenance in progress. Try again later.") # If running from within proposed-updates; assume an install to stable if os.getcwd().find('proposed-updates') != -1: - installing_to_stable = 1; + installing_to_stable = 1 # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: - lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT); + lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT) try: - fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB); + fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError, e: if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN': - utils.fubar("Couldn't obtain lock; assuming another kelly is already running."); + utils.fubar("Couldn't obtain lock; assuming another kelly is already running.") else: - raise; - Logger = Katie.Logger = logging.Logger(Cnf, "kelly"); + raise + Logger = Katie.Logger = logging.Logger(Cnf, "kelly") if not installing_to_stable and Cnf.get("Dir::UrgencyLog"): - Urgency_Logger = Urgency_Log(Cnf); + Urgency_Logger = Urgency_Log(Cnf) # Initialize the substitution template mapping global - bcc = "X-Katie: %s" % (kelly_version); + bcc = "X-Katie: %s" % (kelly_version) if Cnf.has_key("Dinstall::Bcc"): - Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); + Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) else: - Subst["__BCC__"] = bcc; + Subst["__BCC__"] = bcc # Sort the .changes files so that we process sourceful ones first - changes_files.sort(utils.changes_compare); + changes_files.sort(utils.changes_compare) # Process the changes files for changes_file in changes_files: - print "\n" + changes_file; - process_it (changes_file); + print "\n" + changes_file + process_it (changes_file) if install_count: sets = "set" if install_count > 1: sets = "sets" - sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes)))); - Logger.log(["total",install_count,install_bytes]); + sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes)))) + Logger.log(["total",install_count,install_bytes]) if not Options["No-Action"]: - Logger.close(); + Logger.close() if Urgency_Logger: - Urgency_Logger.close(); + Urgency_Logger.close() ############################################################################### if __name__ == '__main__': - main(); + main() diff --git a/dak/process_new.py b/dak/process_new.py index d98c0df6..e1e30789 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -37,152 +37,152 @@ ################################################################################ -import copy, errno, os, readline, stat, sys, time; -import apt_pkg, apt_inst; -import db_access, fernanda, katie, logging, utils; +import copy, errno, os, readline, stat, sys, time +import apt_pkg, apt_inst +import db_access, fernanda, katie, logging, utils # Globals -lisa_version = "$Revision: 1.31 $"; +lisa_version = "$Revision: 1.31 $" -Cnf = None; -Options = None; -Katie = None; -projectB = None; -Logger = None; +Cnf = None +Options = None +Katie = None +projectB = None +Logger = None -Priorities = None; -Sections = None; +Priorities = None +Sections = None -reject_message = ""; +reject_message = "" ################################################################################ ################################################################################ ################################################################################ def reject (str, prefix="Rejected: "): - global reject_message; + global reject_message if str: - reject_message += prefix + str + "\n"; + reject_message += prefix + str + "\n" def recheck(): - global reject_message; - files = Katie.pkg.files; - reject_message = ""; + global reject_message + files = Katie.pkg.files + reject_message = "" for file in files.keys(): # The .orig.tar.gz can disappear out from under us is it's a # duplicate of one in the archive. if not files.has_key(file): - continue; + continue # Check that the source still exists if files[file]["type"] == "deb": - source_version = files[file]["source version"]; - source_package = files[file]["source package"]; + source_version = files[file]["source version"] + source_package = files[file]["source package"] if not Katie.pkg.changes["architecture"].has_key("source") \ and not Katie.source_exists(source_package, source_version, Katie.pkg.changes["distribution"].keys()): - source_epochless_version = utils.re_no_epoch.sub('', source_version); - dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version); + source_epochless_version = utils.re_no_epoch.sub('', source_version) + dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): - reject("no source found for %s %s (%s)." % (source_package, source_version, file)); + reject("no source found for %s %s (%s)." % (source_package, source_version, file)) # Version and file overwrite checks if files[file]["type"] == "deb": - reject(Katie.check_binary_against_db(file)); + reject(Katie.check_binary_against_db(file)) elif files[file]["type"] == "dsc": - reject(Katie.check_source_against_db(file)); - (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); - reject(reject_msg); + reject(Katie.check_source_against_db(file)) + (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file) + reject(reject_msg) if reject_message: - answer = "XXX"; + answer = "XXX" if Options["No-Action"] or Options["Automatic"]: answer = 'S' - print "REJECT\n" + reject_message,; - prompt = "[R]eject, Skip, Quit ?"; + print "REJECT\n" + reject_message, + prompt = "[R]eject, Skip, Quit ?" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.match(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.match(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); + answer = m.group(1) + answer = answer[:1].upper() if answer == 'R': - Katie.do_reject(0, reject_message); - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - return 0; + Katie.do_reject(0, reject_message) + os.unlink(Katie.pkg.changes_file[:-8]+".katie") + return 0 elif answer == 'S': - return 0; + return 0 elif answer == 'Q': - sys.exit(0); + sys.exit(0) - return 1; + return 1 ################################################################################ def determine_new (changes, files): - new = {}; + new = {} # Build up a list of potentially new things for file in files.keys(): - f = files[file]; + f = files[file] # Skip byhand elements if f["type"] == "byhand": - continue; - pkg = f["package"]; - priority = f["priority"]; - section = f["section"]; + continue + pkg = f["package"] + priority = f["priority"] + section = f["section"] # FIXME: unhardcode if section == "non-US/main": - section = "non-US"; - type = get_type(f); - component = f["component"]; + section = "non-US" + type = get_type(f) + component = f["component"] if type == "dsc": - priority = "source"; + priority = "source" if not new.has_key(pkg): - new[pkg] = {}; - new[pkg]["priority"] = priority; - new[pkg]["section"] = section; - new[pkg]["type"] = type; - new[pkg]["component"] = component; - new[pkg]["files"] = []; + new[pkg] = {} + new[pkg]["priority"] = priority + new[pkg]["section"] = section + new[pkg]["type"] = type + new[pkg]["component"] = component + new[pkg]["files"] = [] else: - old_type = new[pkg]["type"]; + old_type = new[pkg]["type"] if old_type != type: # source gets trumped by deb or udeb if old_type == "dsc": - new[pkg]["priority"] = priority; - new[pkg]["section"] = section; - new[pkg]["type"] = type; - new[pkg]["component"] = component; - new[pkg]["files"].append(file); + new[pkg]["priority"] = priority + new[pkg]["section"] = section + new[pkg]["type"] = type + new[pkg]["component"] = component + new[pkg]["files"].append(file) if f.has_key("othercomponents"): - new[pkg]["othercomponents"] = f["othercomponents"]; + new[pkg]["othercomponents"] = f["othercomponents"] for suite in changes["suite"].keys(): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) for pkg in new.keys(): - component_id = db_access.get_component_id(new[pkg]["component"]); - type_id = db_access.get_override_type_id(new[pkg]["type"]); - q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id)); - ql = q.getresult(); + component_id = db_access.get_component_id(new[pkg]["component"]) + type_id = db_access.get_override_type_id(new[pkg]["type"]) + q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id)) + ql = q.getresult() if ql: for file in new[pkg]["files"]: if files[file].has_key("new"): - del files[file]["new"]; - del new[pkg]; + del files[file]["new"] + del new[pkg] if changes["suite"].has_key("stable"): - print "WARNING: overrides will be added for stable!"; + print "WARNING: overrides will be added for stable!" if changes["suite"].has_key("oldstable"): - print "WARNING: overrides will be added for OLDstable!"; + print "WARNING: overrides will be added for OLDstable!" for pkg in new.keys(): if new[pkg].has_key("othercomponents"): - print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]); + print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]) - return new; + return new ################################################################################ @@ -190,107 +190,107 @@ def indiv_sg_compare (a, b): """Sort by source name, source, version, 'have source', and finally by filename.""" # Sort by source version - q = apt_pkg.VersionCompare(a["version"], b["version"]); + q = apt_pkg.VersionCompare(a["version"], b["version"]) if q: - return -q; + return -q # Sort by 'have source' - a_has_source = a["architecture"].get("source"); - b_has_source = b["architecture"].get("source"); + a_has_source = a["architecture"].get("source") + b_has_source = b["architecture"].get("source") if a_has_source and not b_has_source: - return -1; + return -1 elif b_has_source and not a_has_source: - return 1; + return 1 - return cmp(a["filename"], b["filename"]); + return cmp(a["filename"], b["filename"]) ############################################################ def sg_compare (a, b): - a = a[1]; - b = b[1]; + a = a[1] + b = b[1] """Sort by have note, time of oldest upload.""" # Sort by have note - a_note_state = a["note_state"]; - b_note_state = b["note_state"]; + a_note_state = a["note_state"] + b_note_state = b["note_state"] if a_note_state < b_note_state: - return -1; + return -1 elif a_note_state > b_note_state: - return 1; + return 1 # Sort by time of oldest upload - return cmp(a["oldest"], b["oldest"]); + return cmp(a["oldest"], b["oldest"]) def sort_changes(changes_files): """Sort into source groups, then sort each source group by version, have source, filename. Finally, sort the source groups by have note, time of oldest upload of each source upload.""" if len(changes_files) == 1: - return changes_files; + return changes_files - sorted_list = []; - cache = {}; + sorted_list = [] + cache = {} # Read in all the .changes files for filename in changes_files: try: - Katie.pkg.changes_file = filename; - Katie.init_vars(); - Katie.update_vars(); - cache[filename] = copy.copy(Katie.pkg.changes); - cache[filename]["filename"] = filename; + Katie.pkg.changes_file = filename + Katie.init_vars() + Katie.update_vars() + cache[filename] = copy.copy(Katie.pkg.changes) + cache[filename]["filename"] = filename except: - sorted_list.append(filename); - break; + sorted_list.append(filename) + break # Divide the .changes into per-source groups - per_source = {}; + per_source = {} for filename in cache.keys(): - source = cache[filename]["source"]; + source = cache[filename]["source"] if not per_source.has_key(source): - per_source[source] = {}; - per_source[source]["list"] = []; - per_source[source]["list"].append(cache[filename]); + per_source[source] = {} + per_source[source]["list"] = [] + per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): - source_list = per_source[source]["list"]; - first = source_list[0]; - oldest = os.stat(first["filename"])[stat.ST_MTIME]; - have_note = 0; + source_list = per_source[source]["list"] + first = source_list[0] + oldest = os.stat(first["filename"])[stat.ST_MTIME] + have_note = 0 for d in per_source[source]["list"]: - mtime = os.stat(d["filename"])[stat.ST_MTIME]; + mtime = os.stat(d["filename"])[stat.ST_MTIME] if mtime < oldest: - oldest = mtime; - have_note += (d.has_key("lisa note")); - per_source[source]["oldest"] = oldest; + oldest = mtime + have_note += (d.has_key("lisa note")) + per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none elif have_note < len(source_list): per_source[source]["note_state"] = 1; # some else: per_source[source]["note_state"] = 2; # all - per_source[source]["list"].sort(indiv_sg_compare); - per_source_items = per_source.items(); - per_source_items.sort(sg_compare); + per_source[source]["list"].sort(indiv_sg_compare) + per_source_items = per_source.items() + per_source_items.sort(sg_compare) for i in per_source_items: for j in i[1]["list"]: - sorted_list.append(j["filename"]); - return sorted_list; + sorted_list.append(j["filename"]) + return sorted_list ################################################################################ class Section_Completer: def __init__ (self): - self.sections = []; - q = projectB.query("SELECT section FROM section"); + self.sections = [] + q = projectB.query("SELECT section FROM section") for i in q.getresult(): - self.sections.append(i[0]); + self.sections.append(i[0]) def complete(self, text, state): if state == 0: - self.matches = []; - n = len(text); + self.matches = [] + n = len(text) for word in self.sections: if word[:n] == text: - self.matches.append(word); + self.matches.append(word) try: return self.matches[state] except IndexError: @@ -300,18 +300,18 @@ class Section_Completer: class Priority_Completer: def __init__ (self): - self.priorities = []; - q = projectB.query("SELECT priority FROM priority"); + self.priorities = [] + q = projectB.query("SELECT priority FROM priority") for i in q.getresult(): - self.priorities.append(i[0]); + self.priorities.append(i[0]) def complete(self, text, state): if state == 0: - self.matches = []; - n = len(text); + self.matches = [] + n = len(text) for word in self.priorities: if word[:n] == text: - self.matches.append(word); + self.matches.append(word) try: return self.matches[state] except IndexError: @@ -321,447 +321,447 @@ class Priority_Completer: def check_valid (new): for pkg in new.keys(): - section = new[pkg]["section"]; - priority = new[pkg]["priority"]; - type = new[pkg]["type"]; - new[pkg]["section id"] = db_access.get_section_id(section); - new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]); + section = new[pkg]["section"] + priority = new[pkg]["priority"] + type = new[pkg]["type"] + new[pkg]["section id"] = db_access.get_section_id(section) + new[pkg]["priority id"] = db_access.get_priority_id(new[pkg]["priority"]) # Sanity checks if (section == "debian-installer" and type != "udeb") or \ (section != "debian-installer" and type == "udeb"): - new[pkg]["section id"] = -1; + new[pkg]["section id"] = -1 if (priority == "source" and type != "dsc") or \ (priority != "source" and type == "dsc"): - new[pkg]["priority id"] = -1; + new[pkg]["priority id"] = -1 ################################################################################ def print_new (new, indexed, file=sys.stdout): - check_valid(new); - broken = 0; - index = 0; + check_valid(new) + broken = 0 + index = 0 for pkg in new.keys(): - index += 1; - section = new[pkg]["section"]; - priority = new[pkg]["priority"]; + index += 1 + section = new[pkg]["section"] + priority = new[pkg]["priority"] if new[pkg]["section id"] == -1: - section += "[!]"; - broken = 1; + section += "[!]" + broken = 1 if new[pkg]["priority id"] == -1: - priority += "[!]"; - broken = 1; + priority += "[!]" + broken = 1 if indexed: - line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section); + line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section) else: - line = "%-20s %-20s %-20s" % (pkg, priority, section); - line = line.strip()+'\n'; - file.write(line); - note = Katie.pkg.changes.get("lisa note"); + line = "%-20s %-20s %-20s" % (pkg, priority, section) + line = line.strip()+'\n' + file.write(line) + note = Katie.pkg.changes.get("lisa note") if note: - print "*"*75; - print note; - print "*"*75; - return broken, note; + print "*"*75 + print note + print "*"*75 + return broken, note ################################################################################ def get_type (f): # Determine the type if f.has_key("dbtype"): - type = f["dbtype"]; + type = f["dbtype"] elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc": - type = "dsc"; + type = "dsc" else: - utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type)); + utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type)) # Validate the override type - type_id = db_access.get_override_type_id(type); + type_id = db_access.get_override_type_id(type) if type_id == -1: - utils.fubar("invalid type (%s) for new. Say wha?" % (type)); + utils.fubar("invalid type (%s) for new. Say wha?" % (type)) - return type; + return type ################################################################################ def index_range (index): if index == 1: - return "1"; + return "1" else: - return "1-%s" % (index); + return "1-%s" % (index) ################################################################################ ################################################################################ def edit_new (new): # Write the current data to a temporary file - temp_filename = utils.temp_filename(); - temp_file = utils.open_file(temp_filename, 'w'); - print_new (new, 0, temp_file); - temp_file.close(); + temp_filename = utils.temp_filename() + temp_file = utils.open_file(temp_filename, 'w') + print_new (new, 0, temp_file) + temp_file.close() # Spawn an editor on that file editor = os.environ.get("EDITOR","vi") result = os.system("%s %s" % (editor, temp_filename)) if result != 0: utils.fubar ("%s invocation failed for %s." % (editor, temp_filename), result) # Read the edited data back in - temp_file = utils.open_file(temp_filename); - lines = temp_file.readlines(); - temp_file.close(); - os.unlink(temp_filename); + temp_file = utils.open_file(temp_filename) + lines = temp_file.readlines() + temp_file.close() + os.unlink(temp_filename) # Parse the new data for line in lines: - line = line.strip(); + line = line.strip() if line == "": - continue; - s = line.split(); + continue + s = line.split() # Pad the list if necessary - s[len(s):3] = [None] * (3-len(s)); - (pkg, priority, section) = s[:3]; + s[len(s):3] = [None] * (3-len(s)) + (pkg, priority, section) = s[:3] if not new.has_key(pkg): - utils.warn("Ignoring unknown package '%s'" % (pkg)); + utils.warn("Ignoring unknown package '%s'" % (pkg)) else: # Strip off any invalid markers, print_new will readd them. if section.endswith("[!]"): - section = section[:-3]; + section = section[:-3] if priority.endswith("[!]"): - priority = priority[:-3]; + priority = priority[:-3] for file in new[pkg]["files"]: - Katie.pkg.files[file]["section"] = section; - Katie.pkg.files[file]["priority"] = priority; - new[pkg]["section"] = section; - new[pkg]["priority"] = priority; + Katie.pkg.files[file]["section"] = section + Katie.pkg.files[file]["priority"] = priority + new[pkg]["section"] = section + new[pkg]["priority"] = priority ################################################################################ def edit_index (new, index): priority = new[index]["priority"] section = new[index]["section"] - type = new[index]["type"]; + type = new[index]["type"] done = 0 while not done: - print "\t".join([index, priority, section]); + print "\t".join([index, priority, section]) - answer = "XXX"; + answer = "XXX" if type != "dsc": - prompt = "[B]oth, Priority, Section, Done ? "; + prompt = "[B]oth, Priority, Section, Done ? " else: - prompt = "[S]ection, Done ? "; - edit_priority = edit_section = 0; + prompt = "[S]ection, Done ? " + edit_priority = edit_section = 0 while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); + answer = utils.our_raw_input(prompt) m = katie.re_default_answer.match(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() if answer == 'P': - edit_priority = 1; + edit_priority = 1 elif answer == 'S': - edit_section = 1; + edit_section = 1 elif answer == 'B': - edit_priority = edit_section = 1; + edit_priority = edit_section = 1 elif answer == 'D': - done = 1; + done = 1 # Edit the priority if edit_priority: - readline.set_completer(Priorities.complete); - got_priority = 0; + readline.set_completer(Priorities.complete) + got_priority = 0 while not got_priority: - new_priority = utils.our_raw_input("New priority: ").strip(); + new_priority = utils.our_raw_input("New priority: ").strip() if new_priority not in Priorities.priorities: - print "E: '%s' is not a valid priority, try again." % (new_priority); + print "E: '%s' is not a valid priority, try again." % (new_priority) else: - got_priority = 1; - priority = new_priority; + got_priority = 1 + priority = new_priority # Edit the section if edit_section: - readline.set_completer(Sections.complete); - got_section = 0; + readline.set_completer(Sections.complete) + got_section = 0 while not got_section: - new_section = utils.our_raw_input("New section: ").strip(); + new_section = utils.our_raw_input("New section: ").strip() if new_section not in Sections.sections: - print "E: '%s' is not a valid section, try again." % (new_section); + print "E: '%s' is not a valid section, try again." % (new_section) else: - got_section = 1; - section = new_section; + got_section = 1 + section = new_section # Reset the readline completer - readline.set_completer(None); + readline.set_completer(None) for file in new[index]["files"]: - Katie.pkg.files[file]["section"] = section; - Katie.pkg.files[file]["priority"] = priority; - new[index]["priority"] = priority; - new[index]["section"] = section; - return new; + Katie.pkg.files[file]["section"] = section + Katie.pkg.files[file]["priority"] = priority + new[index]["priority"] = priority + new[index]["section"] = section + return new ################################################################################ def edit_overrides (new): - print; + print done = 0 while not done: - print_new (new, 1); - new_index = {}; - index = 0; + print_new (new, 1) + new_index = {} + index = 0 for i in new.keys(): - index += 1; - new_index[index] = i; + index += 1 + new_index[index] = i - prompt = "(%s) edit override , Editor, Done ? " % (index_range(index)); + prompt = "(%s) edit override , Editor, Done ? " % (index_range(index)) got_answer = 0 while not got_answer: - answer = utils.our_raw_input(prompt); + answer = utils.our_raw_input(prompt) if not utils.str_isnum(answer): - answer = answer[:1].upper(); + answer = answer[:1].upper() if answer == "E" or answer == "D": - got_answer = 1; + got_answer = 1 elif katie.re_isanum.match (answer): - answer = int(answer); + answer = int(answer) if (answer < 1) or (answer > index): - print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)); + print "%s is not a valid index (%s). Please retry." % (answer, index_range(index)) else: - got_answer = 1; + got_answer = 1 if answer == 'E': - edit_new(new); + edit_new(new) elif answer == 'D': - done = 1; + done = 1 else: - edit_index (new, new_index[answer]); + edit_index (new, new_index[answer]) - return new; + return new ################################################################################ def edit_note(note): # Write the current data to a temporary file - temp_filename = utils.temp_filename(); - temp_file = utils.open_file(temp_filename, 'w'); - temp_file.write(note); - temp_file.close(); + temp_filename = utils.temp_filename() + temp_file = utils.open_file(temp_filename, 'w') + temp_file.write(note) + temp_file.close() editor = os.environ.get("EDITOR","vi") - answer = 'E'; + answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - temp_file = utils.open_file(temp_filename); - note = temp_file.read().rstrip(); - temp_file.close(); - print "Note:"; - print utils.prefix_multi_line_string(note," "); + temp_file = utils.open_file(temp_filename) + note = temp_file.read().rstrip() + temp_file.close() + print "Note:" + print utils.prefix_multi_line_string(note," ") prompt = "[D]one, Edit, Abandon, Quit ?" - answer = "XXX"; + answer = "XXX" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.search(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - os.unlink(temp_filename); + answer = m.group(1) + answer = answer[:1].upper() + os.unlink(temp_filename) if answer == 'A': - return; + return elif answer == 'Q': - sys.exit(0); - Katie.pkg.changes["lisa note"] = note; - Katie.dump_vars(Cnf["Dir::Queue::New"]); + sys.exit(0) + Katie.pkg.changes["lisa note"] = note + Katie.dump_vars(Cnf["Dir::Queue::New"]) ################################################################################ def check_pkg (): try: - less_fd = os.popen("less -R -", 'w', 0); - stdout_fd = sys.stdout; + less_fd = os.popen("less -R -", 'w', 0) + stdout_fd = sys.stdout try: - sys.stdout = less_fd; - fernanda.display_changes(Katie.pkg.changes_file); - files = Katie.pkg.files; + sys.stdout = less_fd + fernanda.display_changes(Katie.pkg.changes_file) + files = Katie.pkg.files for file in files.keys(): if files[file].has_key("new"): - type = files[file]["type"]; + type = files[file]["type"] if type == "deb": - fernanda.check_deb(file); + fernanda.check_deb(file) elif type == "dsc": - fernanda.check_dsc(file); + fernanda.check_dsc(file) finally: - sys.stdout = stdout_fd; + sys.stdout = stdout_fd except IOError, e: if errno.errorcode[e.errno] == 'EPIPE': - utils.warn("[fernanda] Caught EPIPE; skipping."); - pass; + utils.warn("[fernanda] Caught EPIPE; skipping.") + pass else: - raise; + raise except KeyboardInterrupt: - utils.warn("[fernanda] Caught C-c; skipping."); - pass; + utils.warn("[fernanda] Caught C-c; skipping.") + pass ################################################################################ ## FIXME: horribly Debian specific def do_bxa_notification(): - files = Katie.pkg.files; - summary = ""; + files = Katie.pkg.files + summary = "" for file in files.keys(): if files[file]["type"] == "deb": - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file))); - summary += "\n"; - summary += "Package: %s\n" % (control.Find("Package")); - summary += "Description: %s\n" % (control.Find("Description")); - Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary; - bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification"); - utils.send_mail(bxa_mail); + control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file))) + summary += "\n" + summary += "Package: %s\n" % (control.Find("Package")) + summary += "Description: %s\n" % (control.Find("Description")) + Katie.Subst["__BINARY_DESCRIPTIONS__"] = summary + bxa_mail = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lisa.bxa_notification") + utils.send_mail(bxa_mail) ################################################################################ def add_overrides (new): - changes = Katie.pkg.changes; - files = Katie.pkg.files; + changes = Katie.pkg.changes + files = Katie.pkg.files - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") for suite in changes["suite"].keys(): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) for pkg in new.keys(): - component_id = db_access.get_component_id(new[pkg]["component"]); - type_id = db_access.get_override_type_id(new[pkg]["type"]); - priority_id = new[pkg]["priority id"]; - section_id = new[pkg]["section id"]; - projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)); + component_id = db_access.get_component_id(new[pkg]["component"]) + type_id = db_access.get_override_type_id(new[pkg]["type"]) + priority_id = new[pkg]["priority id"] + section_id = new[pkg]["section id"] + projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)) for file in new[pkg]["files"]: if files[file].has_key("new"): - del files[file]["new"]; - del new[pkg]; + del files[file]["new"] + del new[pkg] - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") if Cnf.FindB("Dinstall::BXANotify"): - do_bxa_notification(); + do_bxa_notification() ################################################################################ def prod_maintainer (): # Here we prepare an editor and get them ready to prod... - temp_filename = utils.temp_filename(); + temp_filename = utils.temp_filename() editor = os.environ.get("EDITOR","vi") - answer = 'E'; + answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - file = utils.open_file(temp_filename); - prod_message = "".join(file.readlines()); - file.close(); - print "Prod message:"; - print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1); + file = utils.open_file(temp_filename) + prod_message = "".join(file.readlines()) + file.close() + print "Prod message:" + print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1) prompt = "[P]rod, Edit, Abandon, Quit ?" - answer = "XXX"; + answer = "XXX" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.search(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - os.unlink(temp_filename); + answer = m.group(1) + answer = answer[:1].upper() + os.unlink(temp_filename) if answer == 'A': - return; + return elif answer == 'Q': - sys.exit(0); + sys.exit(0) # Otherwise, do the proding... user_email_address = utils.whoami() + " <%s>" % ( - Cnf["Dinstall::MyAdminAddress"]); + Cnf["Dinstall::MyAdminAddress"]) - Subst = Katie.Subst; + Subst = Katie.Subst - Subst["__FROM_ADDRESS__"] = user_email_address; - Subst["__PROD_MESSAGE__"] = prod_message; - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; + Subst["__FROM_ADDRESS__"] = user_email_address + Subst["__PROD_MESSAGE__"] = prod_message + Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] prod_mail_message = utils.TemplateSubst( - Subst,Cnf["Dir::Templates"]+"/lisa.prod"); + Subst,Cnf["Dir::Templates"]+"/lisa.prod") # Send the prod mail if appropriate if not Cnf["Dinstall::Options::No-Mail"]: - utils.send_mail(prod_mail_message); + utils.send_mail(prod_mail_message) - print "Sent proding message"; + print "Sent proding message" ################################################################################ def do_new(): - print "NEW\n"; - files = Katie.pkg.files; - changes = Katie.pkg.changes; + print "NEW\n" + files = Katie.pkg.files + changes = Katie.pkg.changes # Make a copy of distribution we can happily trample on - changes["suite"] = copy.copy(changes["distribution"]); + changes["suite"] = copy.copy(changes["distribution"]) # Fix up the list of target suites for suite in changes["suite"].keys(): - override = Cnf.Find("Suite::%s::OverrideSuite" % (suite)); + override = Cnf.Find("Suite::%s::OverrideSuite" % (suite)) if override: - del changes["suite"][suite]; - changes["suite"][override] = 1; + del changes["suite"][suite] + changes["suite"][override] = 1 # Validate suites for suite in changes["suite"].keys(): - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id == -1: - utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)); + utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) # The main NEW processing loop - done = 0; + done = 0 while not done: # Find out what's new - new = determine_new(changes, files); + new = determine_new(changes, files) if not new: - break; + break - answer = "XXX"; + answer = "XXX" if Options["No-Action"] or Options["Automatic"]: - answer = 'S'; + answer = 'S' - (broken, note) = print_new(new, 0); - prompt = ""; + (broken, note) = print_new(new, 0) + prompt = "" if not broken and not note: - prompt = "Add overrides, "; + prompt = "Add overrides, " if broken: - print "W: [!] marked entries must be fixed before package can be processed."; + print "W: [!] marked entries must be fixed before package can be processed." if note: - print "W: note must be removed before package can be processed."; - prompt += "Remove note, "; + print "W: note must be removed before package can be processed." + prompt += "Remove note, " - prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?"; + prompt += "Edit overrides, Check, Manual reject, Note edit, Prod, [S]kip, Quit ?" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() if answer == 'A': - done = add_overrides (new); + done = add_overrides (new) elif answer == 'C': - check_pkg(); + check_pkg() elif answer == 'E': - new = edit_overrides (new); + new = edit_overrides (new) elif answer == 'M': - aborted = Katie.do_reject(1, Options["Manual-Reject"]); + aborted = Katie.do_reject(1, Options["Manual-Reject"]) if not aborted: - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - done = 1; + os.unlink(Katie.pkg.changes_file[:-8]+".katie") + done = 1 elif answer == 'N': - edit_note(changes.get("lisa note", "")); + edit_note(changes.get("lisa note", "")) elif answer == 'P': - prod_maintainer(); + prod_maintainer() elif answer == 'R': - confirm = utils.our_raw_input("Really clear note (y/N)? ").lower(); + confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - del changes["lisa note"]; + del changes["lisa note"] elif answer == 'S': - done = 1; + done = 1 elif answer == 'Q': sys.exit(0) @@ -781,185 +781,185 @@ def usage (exit_code=0): ################################################################################ def init(): - global Cnf, Options, Logger, Katie, projectB, Sections, Priorities; + global Cnf, Options, Logger, Katie, projectB, Sections, Priorities - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('a',"automatic","Lisa::Options::Automatic"), ('h',"help","Lisa::Options::Help"), ('m',"manual-reject","Lisa::Options::Manual-Reject", "HasArg"), ('n',"no-action","Lisa::Options::No-Action"), - ('V',"version","Lisa::Options::Version")]; + ('V',"version","Lisa::Options::Version")] for i in ["automatic", "help", "manual-reject", "no-action", "version"]: if not Cnf.has_key("Lisa::Options::%s" % (i)): - Cnf["Lisa::Options::%s" % (i)] = ""; + Cnf["Lisa::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Lisa::Options") if Options["Help"]: - usage(); + usage() if Options["Version"]: - print "lisa %s" % (lisa_version); - sys.exit(0); + print "lisa %s" % (lisa_version) + sys.exit(0) - Katie = katie.Katie(Cnf); + Katie = katie.Katie(Cnf) if not Options["No-Action"]: - Logger = Katie.Logger = logging.Logger(Cnf, "lisa"); + Logger = Katie.Logger = logging.Logger(Cnf, "lisa") - projectB = Katie.projectB; + projectB = Katie.projectB - Sections = Section_Completer(); - Priorities = Priority_Completer(); - readline.parse_and_bind("tab: complete"); + Sections = Section_Completer() + Priorities = Priority_Completer() + readline.parse_and_bind("tab: complete") - return changes_files; + return changes_files ################################################################################ def do_byhand(): - done = 0; + done = 0 while not done: - files = Katie.pkg.files; - will_install = 1; - byhand = []; + files = Katie.pkg.files + will_install = 1 + byhand = [] for file in files.keys(): if files[file]["type"] == "byhand": if os.path.exists(file): - print "W: %s still present; please process byhand components and try again." % (file); - will_install = 0; + print "W: %s still present; please process byhand components and try again." % (file) + will_install = 0 else: - byhand.append(file); + byhand.append(file) - answer = "XXXX"; + answer = "XXXX" if Options["No-Action"]: - answer = "S"; + answer = "S" if will_install: if Options["Automatic"] and not Options["No-Action"]: - answer = 'A'; - prompt = "[A]ccept, Manual reject, Skip, Quit ?"; + answer = 'A' + prompt = "[A]ccept, Manual reject, Skip, Quit ?" else: - prompt = "Manual reject, [S]kip, Quit ?"; + prompt = "Manual reject, [S]kip, Quit ?" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.search(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); + answer = m.group(1) + answer = answer[:1].upper() if answer == 'A': - done = 1; + done = 1 for file in byhand: - del files[file]; + del files[file] elif answer == 'M': - Katie.do_reject(1, Options["Manual-Reject"]); - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - done = 1; + Katie.do_reject(1, Options["Manual-Reject"]) + os.unlink(Katie.pkg.changes_file[:-8]+".katie") + done = 1 elif answer == 'S': - done = 1; + done = 1 elif answer == 'Q': - sys.exit(0); + sys.exit(0) ################################################################################ def do_accept(): - print "ACCEPT"; + print "ACCEPT" if not Options["No-Action"]: - retry = 0; + retry = 0 while retry < 10: try: - lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL); - retry = 10; + lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) + retry = 10 except OSError, e: if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': - retry += 1; + retry += 1 if (retry >= 10): - utils.fubar("Couldn't obtain lock; assuming jennifer is already running."); + utils.fubar("Couldn't obtain lock; assuming jennifer is already running.") else: - print("Unable to get accepted lock (try %d of 10)" % retry); - time.sleep(60); + print("Unable to get accepted lock (try %d of 10)" % retry) + time.sleep(60) else: - raise; - (summary, short_summary) = Katie.build_summaries(); - Katie.accept(summary, short_summary); - os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - os.unlink(Cnf["Lisa::AcceptedLockFile"]); + raise + (summary, short_summary) = Katie.build_summaries() + Katie.accept(summary, short_summary) + os.unlink(Katie.pkg.changes_file[:-8]+".katie") + os.unlink(Cnf["Lisa::AcceptedLockFile"]) def check_status(files): - new = byhand = 0; + new = byhand = 0 for file in files.keys(): if files[file]["type"] == "byhand": - byhand = 1; + byhand = 1 elif files[file].has_key("new"): - new = 1; - return (new, byhand); + new = 1 + return (new, byhand) def do_pkg(changes_file): - Katie.pkg.changes_file = changes_file; - Katie.init_vars(); - Katie.update_vars(); - Katie.update_subst(); - files = Katie.pkg.files; + Katie.pkg.changes_file = changes_file + Katie.init_vars() + Katie.update_vars() + Katie.update_subst() + files = Katie.pkg.files if not recheck(): - return; + return - (new, byhand) = check_status(files); + (new, byhand) = check_status(files) if new or byhand: if new: - do_new(); + do_new() if byhand: - do_byhand(); - (new, byhand) = check_status(files); + do_byhand() + (new, byhand) = check_status(files) if not new and not byhand: - do_accept(); + do_accept() ################################################################################ def end(): - accept_count = Katie.accept_count; - accept_bytes = Katie.accept_bytes; + accept_count = Katie.accept_count + accept_bytes = Katie.accept_bytes if accept_count: sets = "set" if accept_count > 1: sets = "sets" - sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))); - Logger.log(["total",accept_count,accept_bytes]); + sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) + Logger.log(["total",accept_count,accept_bytes]) if not Options["No-Action"]: - Logger.close(); + Logger.close() ################################################################################ def main(): - changes_files = init(); + changes_files = init() if len(changes_files) > 50: - sys.stderr.write("Sorting changes...\n"); - changes_files = sort_changes(changes_files); + sys.stderr.write("Sorting changes...\n") + changes_files = sort_changes(changes_files) # Kill me now? **FIXME** - Cnf["Dinstall::Options::No-Mail"] = ""; - bcc = "X-Katie: lisa %s" % (lisa_version); + Cnf["Dinstall::Options::No-Mail"] = "" + bcc = "X-Katie: lisa %s" % (lisa_version) if Cnf.has_key("Dinstall::Bcc"): - Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); + Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) else: - Katie.Subst["__BCC__"] = bcc; + Katie.Subst["__BCC__"] = bcc for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file, 0); + changes_file = utils.validate_changes_file_arg(changes_file, 0) if not changes_file: - continue; - print "\n" + changes_file; - do_pkg (changes_file); + continue + print "\n" + changes_file + do_pkg (changes_file) - end(); + end() ################################################################################ diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index e11153fc..1d87447a 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -29,79 +29,79 @@ ################################################################################ -import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback; -import apt_inst, apt_pkg; -import db_access, katie, logging, utils; +import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback +import apt_inst, apt_pkg +import db_access, katie, logging, utils -from types import *; +from types import * ################################################################################ -re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$"); -re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$"); -re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)"); -re_strip_revision = re.compile(r"-([^-]+)$"); +re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$") +re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$") +re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)") +re_strip_revision = re.compile(r"-([^-]+)$") ################################################################################ # Globals -jennifer_version = "$Revision: 1.65 $"; +jennifer_version = "$Revision: 1.65 $" -Cnf = None; -Options = None; -Logger = None; -Katie = None; +Cnf = None +Options = None +Logger = None +Katie = None -reprocess = 0; -in_holding = {}; +reprocess = 0 +in_holding = {} # Aliases to the real vars in the Katie class; hysterical raisins. -reject_message = ""; -changes = {}; -dsc = {}; -dsc_files = {}; -files = {}; -pkg = {}; +reject_message = "" +changes = {} +dsc = {} +dsc_files = {} +files = {} +pkg = {} ############################################################################### def init(): - global Cnf, Options, Katie, changes, dsc, dsc_files, files, pkg; + global Cnf, Options, Katie, changes, dsc, dsc_files, files, pkg - apt_pkg.init(); + apt_pkg.init() - Cnf = apt_pkg.newConfiguration(); - apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + Cnf = apt_pkg.newConfiguration() + apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()) Arguments = [('a',"automatic","Dinstall::Options::Automatic"), ('h',"help","Dinstall::Options::Help"), ('n',"no-action","Dinstall::Options::No-Action"), ('p',"no-lock", "Dinstall::Options::No-Lock"), ('s',"no-mail", "Dinstall::Options::No-Mail"), - ('V',"version","Dinstall::Options::Version")]; + ('V',"version","Dinstall::Options::Version")] for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "override-distribution", "version"]: - Cnf["Dinstall::Options::%s" % (i)] = ""; + Cnf["Dinstall::Options::%s" % (i)] = "" - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Dinstall::Options") if Options["Help"]: - usage(); + usage() elif Options["Version"]: - print "jennifer %s" % (jennifer_version); - sys.exit(0); + print "jennifer %s" % (jennifer_version) + sys.exit(0) - Katie = katie.Katie(Cnf); + Katie = katie.Katie(Cnf) - changes = Katie.pkg.changes; - dsc = Katie.pkg.dsc; - dsc_files = Katie.pkg.dsc_files; - files = Katie.pkg.files; - pkg = Katie.pkg; + changes = Katie.pkg.changes + dsc = Katie.pkg.dsc + dsc_files = Katie.pkg.dsc_files + files = Katie.pkg.files + pkg = Katie.pkg - return changes_files; + return changes_files ################################################################################ @@ -118,92 +118,92 @@ def usage (exit_code=0): ################################################################################ def reject (str, prefix="Rejected: "): - global reject_message; + global reject_message if str: - reject_message += prefix + str + "\n"; + reject_message += prefix + str + "\n" ################################################################################ def copy_to_holding(filename): - global in_holding; + global in_holding - base_filename = os.path.basename(filename); + base_filename = os.path.basename(filename) - dest = Cnf["Dir::Queue::Holding"] + '/' + base_filename; + dest = Cnf["Dir::Queue::Holding"] + '/' + base_filename try: - fd = os.open(dest, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0640); - os.close(fd); + fd = os.open(dest, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0640) + os.close(fd) except OSError, e: # Shouldn't happen, but will if, for example, someone lists a # file twice in the .changes. if errno.errorcode[e.errno] == 'EEXIST': - reject("%s: already exists in holding area; can not overwrite." % (base_filename)); - return; - raise; + reject("%s: already exists in holding area; can not overwrite." % (base_filename)) + return + raise try: - shutil.copy(filename, dest); + shutil.copy(filename, dest) except IOError, e: # In either case (ENOENT or EACCES) we want to remove the # O_CREAT | O_EXCLed ghost file, so add the file to the list # of 'in holding' even if it's not the real file. if errno.errorcode[e.errno] == 'ENOENT': - reject("%s: can not copy to holding area: file not found." % (base_filename)); - os.unlink(dest); - return; + reject("%s: can not copy to holding area: file not found." % (base_filename)) + os.unlink(dest) + return elif errno.errorcode[e.errno] == 'EACCES': - reject("%s: can not copy to holding area: read permission denied." % (base_filename)); - os.unlink(dest); - return; - raise; + reject("%s: can not copy to holding area: read permission denied." % (base_filename)) + os.unlink(dest) + return + raise - in_holding[base_filename] = ""; + in_holding[base_filename] = "" ################################################################################ def clean_holding(): - global in_holding; + global in_holding - cwd = os.getcwd(); - os.chdir(Cnf["Dir::Queue::Holding"]); + cwd = os.getcwd() + os.chdir(Cnf["Dir::Queue::Holding"]) for file in in_holding.keys(): if os.path.exists(file): if file.find('/') != -1: - utils.fubar("WTF? clean_holding() got a file ('%s') with / in it!" % (file)); + utils.fubar("WTF? clean_holding() got a file ('%s') with / in it!" % (file)) else: - os.unlink(file); - in_holding = {}; - os.chdir(cwd); + os.unlink(file) + in_holding = {} + os.chdir(cwd) ################################################################################ def check_changes(): - filename = pkg.changes_file; + filename = pkg.changes_file # Parse the .changes field into a dictionary try: - changes.update(utils.parse_changes(filename)); + changes.update(utils.parse_changes(filename)) except utils.cant_open_exc: - reject("%s: can't read file." % (filename)); - return 0; + reject("%s: can't read file." % (filename)) + return 0 except utils.changes_parse_error_exc, line: - reject("%s: parse error, can't grok: %s." % (filename, line)); - return 0; + reject("%s: parse error, can't grok: %s." % (filename, line)) + return 0 # Parse the Files field from the .changes into another dictionary try: - files.update(utils.build_file_list(changes)); + files.update(utils.build_file_list(changes)) except utils.changes_parse_error_exc, line: - reject("%s: parse error, can't grok: %s." % (filename, line)); + reject("%s: parse error, can't grok: %s." % (filename, line)) except utils.nk_format_exc, format: - reject("%s: unknown format '%s'." % (filename, format)); - return 0; + reject("%s: unknown format '%s'." % (filename, format)) + return 0 # Check for mandatory fields for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files", "changes", "description"): if not changes.has_key(i): - reject("%s: Missing mandatory field `%s'." % (filename, i)); + reject("%s: Missing mandatory field `%s'." % (filename, i)) return 0 # Avoid errors during later tests # Split multi-value fields into a lower-level dictionary @@ -219,28 +219,28 @@ def check_changes(): try: (changes["maintainer822"], changes["maintainer2047"], changes["maintainername"], changes["maintaineremail"]) = \ - utils.fix_maintainer (changes["maintainer"]); + utils.fix_maintainer (changes["maintainer"]) except utils.ParseMaintError, msg: reject("%s: Maintainer field ('%s') failed to parse: %s" \ - % (filename, changes["maintainer"], msg)); + % (filename, changes["maintainer"], msg)) # ...likewise for the Changed-By: field if it exists. try: (changes["changedby822"], changes["changedby2047"], changes["changedbyname"], changes["changedbyemail"]) = \ - utils.fix_maintainer (changes.get("changed-by", "")); + utils.fix_maintainer (changes.get("changed-by", "")) except utils.ParseMaintError, msg: (changes["changedby822"], changes["changedby2047"], changes["changedbyname"], changes["changedbyemail"]) = \ ("", "", "", "") reject("%s: Changed-By field ('%s') failed to parse: %s" \ - % (filename, changes["changed-by"], msg)); + % (filename, changes["changed-by"], msg)) # Ensure all the values in Closes: are numbers if changes.has_key("closes"): for i in changes["closes"].keys(): if katie.re_isanum.match (i) == None: - reject("%s: `%s' from Closes field isn't a number." % (filename, i)); + reject("%s: `%s' from Closes field isn't a number." % (filename, i)) # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison) @@ -249,17 +249,17 @@ def check_changes(): # Check there isn't already a changes file of the same name in one # of the queue directories. - base_filename = os.path.basename(filename); + base_filename = os.path.basename(filename) for dir in [ "Accepted", "Byhand", "Done", "New" ]: if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+base_filename): - reject("%s: a file with this name already exists in the %s directory." % (base_filename, dir)); + reject("%s: a file with this name already exists in the %s directory." % (base_filename, dir)) # Check the .changes is non-empty if not files: reject("%s: nothing to do (Files field is empty)." % (base_filename)) - return 0; + return 0 - return 1; + return 1 ################################################################################ @@ -268,36 +268,36 @@ def check_distributions(): # Handle suite mappings for map in Cnf.ValueList("SuiteMappings"): - args = map.split(); - type = args[0]; + args = map.split() + type = args[0] if type == "map" or type == "silent-map": - (source, dest) = args[1:3]; + (source, dest) = args[1:3] if changes["distribution"].has_key(source): del changes["distribution"][source] - changes["distribution"][dest] = 1; + changes["distribution"][dest] = 1 if type != "silent-map": - reject("Mapping %s to %s." % (source, dest),""); + reject("Mapping %s to %s." % (source, dest),"") if changes.has_key("distribution-version"): if changes["distribution-version"].has_key(source): changes["distribution-version"][source]=dest elif type == "map-unreleased": - (source, dest) = args[1:3]; + (source, dest) = args[1:3] if changes["distribution"].has_key(source): for arch in changes["architecture"].keys(): if arch not in Cnf.ValueList("Suite::%s::Architectures" % (source)): - reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),""); - del changes["distribution"][source]; - changes["distribution"][dest] = 1; - break; + reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),"") + del changes["distribution"][source] + changes["distribution"][dest] = 1 + break elif type == "ignore": - suite = args[1]; + suite = args[1] if changes["distribution"].has_key(suite): - del changes["distribution"][suite]; - reject("Ignoring %s as a target suite." % (suite), "Warning: "); + del changes["distribution"][suite] + reject("Ignoring %s as a target suite." % (suite), "Warning: ") elif type == "reject": - suite = args[1]; + suite = args[1] if changes["distribution"].has_key(suite): - reject("Uploads to %s are not accepted." % (suite)); + reject("Uploads to %s are not accepted." % (suite)) elif type == "propup-version": # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes" # @@ -308,12 +308,12 @@ def check_distributions(): # Ensure there is (still) a target distribution if changes["distribution"].keys() == []: - reject("no valid distribution."); + reject("no valid distribution.") # Ensure target distributions exist for suite in changes["distribution"].keys(): if not Cnf.has_key("Suite::%s" % (suite)): - reject("Unknown distribution `%s'." % (suite)); + reject("Unknown distribution `%s'." % (suite)) ################################################################################ @@ -362,269 +362,269 @@ Pre-Depends on dpkg (>= 1.10.24).""" def check_files(): global reprocess - archive = utils.where_am_i(); - file_keys = files.keys(); + archive = utils.where_am_i() + file_keys = files.keys() # if reprocess is 2 we've already done this and we're checking # things again for the new .orig.tar.gz. # [Yes, I'm fully aware of how disgusting this is] if not Options["No-Action"] and reprocess < 2: - cwd = os.getcwd(); - os.chdir(pkg.directory); + cwd = os.getcwd() + os.chdir(pkg.directory) for file in file_keys: - copy_to_holding(file); - os.chdir(cwd); + copy_to_holding(file) + os.chdir(cwd) # Check there isn't already a .changes or .katie file of the same name in # the proposed-updates "CopyChanges" or "CopyKatie" storage directories. # [NB: this check must be done post-suite mapping] - base_filename = os.path.basename(pkg.changes_file); + base_filename = os.path.basename(pkg.changes_file) katie_filename = base_filename[:-8]+".katie" for suite in changes["distribution"].keys(): - copychanges = "Suite::%s::CopyChanges" % (suite); + copychanges = "Suite::%s::CopyChanges" % (suite) if Cnf.has_key(copychanges) and \ os.path.exists(Cnf[copychanges]+"/"+base_filename): reject("%s: a file with this name already exists in %s" \ - % (base_filename, Cnf[copychanges])); + % (base_filename, Cnf[copychanges])) - copykatie = "Suite::%s::CopyKatie" % (suite); + copykatie = "Suite::%s::CopyKatie" % (suite) if Cnf.has_key(copykatie) and \ os.path.exists(Cnf[copykatie]+"/"+katie_filename): reject("%s: a file with this name already exists in %s" \ - % (katie_filename, Cnf[copykatie])); + % (katie_filename, Cnf[copykatie])) - reprocess = 0; - has_binaries = 0; - has_source = 0; + reprocess = 0 + has_binaries = 0 + has_source = 0 for file in file_keys: # Ensure the file does not already exist in one of the accepted directories for dir in [ "Accepted", "Byhand", "New" ]: if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+file): - reject("%s file already exists in the %s directory." % (file, dir)); + reject("%s file already exists in the %s directory." % (file, dir)) if not utils.re_taint_free.match(file): - reject("!!WARNING!! tainted filename: '%s'." % (file)); + reject("!!WARNING!! tainted filename: '%s'." % (file)) # Check the file is readable if os.access(file,os.R_OK) == 0: # When running in -n, copy_to_holding() won't have # generated the reject_message, so we need to. if Options["No-Action"]: if os.path.exists(file): - reject("Can't read `%s'. [permission denied]" % (file)); + reject("Can't read `%s'. [permission denied]" % (file)) else: - reject("Can't read `%s'. [file not found]" % (file)); - files[file]["type"] = "unreadable"; - continue; + reject("Can't read `%s'. [file not found]" % (file)) + files[file]["type"] = "unreadable" + continue # If it's byhand skip remaining checks if files[file]["section"] == "byhand" or files[file]["section"] == "raw-installer": - files[file]["byhand"] = 1; - files[file]["type"] = "byhand"; + files[file]["byhand"] = 1 + files[file]["type"] = "byhand" # Checks for a binary package... elif utils.re_isadeb.match(file): - has_binaries = 1; - files[file]["type"] = "deb"; + has_binaries = 1 + files[file]["type"] = "deb" # Extract package control information - deb_file = utils.open_file(file); + deb_file = utils.open_file(file) try: - control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file)); + control = apt_pkg.ParseSection(apt_inst.debExtractControl(deb_file)) except: - reject("%s: debExtractControl() raised %s." % (file, sys.exc_type)); - deb_file.close(); + reject("%s: debExtractControl() raised %s." % (file, sys.exc_type)) + deb_file.close() # Can't continue, none of the checks on control would work. - continue; - deb_file.close(); + continue + deb_file.close() # Check for mandatory fields for field in [ "Package", "Architecture", "Version" ]: if control.Find(field) == None: - reject("%s: No %s field in control." % (file, field)); + reject("%s: No %s field in control." % (file, field)) # Can't continue - continue; + continue # Ensure the package name matches the one give in the .changes if not changes["binary"].has_key(control.Find("Package", "")): - reject("%s: control file lists name as `%s', which isn't in changes file." % (file, control.Find("Package", ""))); + reject("%s: control file lists name as `%s', which isn't in changes file." % (file, control.Find("Package", ""))) # Validate the package field - package = control.Find("Package"); + package = control.Find("Package") if not re_valid_pkg_name.match(package): - reject("%s: invalid package name '%s'." % (file, package)); + reject("%s: invalid package name '%s'." % (file, package)) # Validate the version field - version = control.Find("Version"); + version = control.Find("Version") if not re_valid_version.match(version): - reject("%s: invalid version number '%s'." % (file, version)); + reject("%s: invalid version number '%s'." % (file, version)) # Ensure the architecture of the .deb is one we know about. default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable") - architecture = control.Find("Architecture"); + architecture = control.Find("Architecture") if architecture not in Cnf.ValueList("Suite::%s::Architectures" % (default_suite)): - reject("Unknown architecture '%s'." % (architecture)); + reject("Unknown architecture '%s'." % (architecture)) # Ensure the architecture of the .deb is one of the ones # listed in the .changes. if not changes["architecture"].has_key(architecture): - reject("%s: control file lists arch as `%s', which isn't in changes file." % (file, architecture)); + reject("%s: control file lists arch as `%s', which isn't in changes file." % (file, architecture)) # Sanity-check the Depends field - depends = control.Find("Depends"); + depends = control.Find("Depends") if depends == '': - reject("%s: Depends field is empty." % (file)); + reject("%s: Depends field is empty." % (file)) # Check the section & priority match those given in the .changes (non-fatal) if control.Find("Section") and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"): - reject("%s control file lists section as `%s', but changes file has `%s'." % (file, control.Find("Section", ""), files[file]["section"]), "Warning: "); + reject("%s control file lists section as `%s', but changes file has `%s'." % (file, control.Find("Section", ""), files[file]["section"]), "Warning: ") if control.Find("Priority") and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"): - reject("%s control file lists priority as `%s', but changes file has `%s'." % (file, control.Find("Priority", ""), files[file]["priority"]),"Warning: "); + reject("%s control file lists priority as `%s', but changes file has `%s'." % (file, control.Find("Priority", ""), files[file]["priority"]),"Warning: ") - files[file]["package"] = package; - files[file]["architecture"] = architecture; - files[file]["version"] = version; - files[file]["maintainer"] = control.Find("Maintainer", ""); + files[file]["package"] = package + files[file]["architecture"] = architecture + files[file]["version"] = version + files[file]["maintainer"] = control.Find("Maintainer", "") if file.endswith(".udeb"): - files[file]["dbtype"] = "udeb"; + files[file]["dbtype"] = "udeb" elif file.endswith(".deb"): - files[file]["dbtype"] = "deb"; + files[file]["dbtype"] = "deb" else: - reject("%s is neither a .deb or a .udeb." % (file)); - files[file]["source"] = control.Find("Source", files[file]["package"]); + reject("%s is neither a .deb or a .udeb." % (file)) + files[file]["source"] = control.Find("Source", files[file]["package"]) # Get the source version - source = files[file]["source"]; - source_version = ""; + source = files[file]["source"] + source_version = "" if source.find("(") != -1: - m = utils.re_extract_src_version.match(source); - source = m.group(1); - source_version = m.group(2); + m = utils.re_extract_src_version.match(source) + source = m.group(1) + source_version = m.group(2) if not source_version: - source_version = files[file]["version"]; - files[file]["source package"] = source; - files[file]["source version"] = source_version; + source_version = files[file]["version"] + files[file]["source package"] = source + files[file]["source version"] = source_version # Ensure the filename matches the contents of the .deb - m = utils.re_isadeb.match(file); + m = utils.re_isadeb.match(file) # package name - file_package = m.group(1); + file_package = m.group(1) if files[file]["package"] != file_package: - reject("%s: package part of filename (%s) does not match package name in the %s (%s)." % (file, file_package, files[file]["dbtype"], files[file]["package"])); - epochless_version = utils.re_no_epoch.sub('', control.Find("Version")); + reject("%s: package part of filename (%s) does not match package name in the %s (%s)." % (file, file_package, files[file]["dbtype"], files[file]["package"])) + epochless_version = utils.re_no_epoch.sub('', control.Find("Version")) # version - file_version = m.group(2); + file_version = m.group(2) if epochless_version != file_version: - reject("%s: version part of filename (%s) does not match package version in the %s (%s)." % (file, file_version, files[file]["dbtype"], epochless_version)); + reject("%s: version part of filename (%s) does not match package version in the %s (%s)." % (file, file_version, files[file]["dbtype"], epochless_version)) # architecture - file_architecture = m.group(3); + file_architecture = m.group(3) if files[file]["architecture"] != file_architecture: - reject("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % (file, file_architecture, files[file]["dbtype"], files[file]["architecture"])); + reject("%s: architecture part of filename (%s) does not match package architecture in the %s (%s)." % (file, file_architecture, files[file]["dbtype"], files[file]["architecture"])) # Check for existent source - source_version = files[file]["source version"]; - source_package = files[file]["source package"]; + source_version = files[file]["source version"] + source_package = files[file]["source package"] if changes["architecture"].has_key("source"): if source_version != changes["version"]: - reject("source version (%s) for %s doesn't match changes version %s." % (source_version, file, changes["version"])); + reject("source version (%s) for %s doesn't match changes version %s." % (source_version, file, changes["version"])) else: # Check in the SQL database if not Katie.source_exists(source_package, source_version, changes["distribution"].keys()): # Check in one of the other directories - source_epochless_version = utils.re_no_epoch.sub('', source_version); - dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version); + source_epochless_version = utils.re_no_epoch.sub('', source_version) + dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) if os.path.exists(Cnf["Dir::Queue::Byhand"] + '/' + dsc_filename): - files[file]["byhand"] = 1; + files[file]["byhand"] = 1 elif os.path.exists(Cnf["Dir::Queue::New"] + '/' + dsc_filename): - files[file]["new"] = 1; + files[file]["new"] = 1 elif not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): - reject("no source found for %s %s (%s)." % (source_package, source_version, file)); + reject("no source found for %s %s (%s)." % (source_package, source_version, file)) # Check the version and for file overwrites - reject(Katie.check_binary_against_db(file),""); + reject(Katie.check_binary_against_db(file),"") check_deb_ar(file, control) # Checks for a source package... else: - m = utils.re_issource.match(file); + m = utils.re_issource.match(file) if m: - has_source = 1; - files[file]["package"] = m.group(1); - files[file]["version"] = m.group(2); - files[file]["type"] = m.group(3); + has_source = 1 + files[file]["package"] = m.group(1) + files[file]["version"] = m.group(2) + files[file]["type"] = m.group(3) # Ensure the source package name matches the Source filed in the .changes if changes["source"] != files[file]["package"]: - reject("%s: changes file doesn't say %s for Source" % (file, files[file]["package"])); + reject("%s: changes file doesn't say %s for Source" % (file, files[file]["package"])) # Ensure the source version matches the version in the .changes file if files[file]["type"] == "orig.tar.gz": - changes_version = changes["chopversion2"]; + changes_version = changes["chopversion2"] else: - changes_version = changes["chopversion"]; + changes_version = changes["chopversion"] if changes_version != files[file]["version"]: - reject("%s: should be %s according to changes file." % (file, changes_version)); + reject("%s: should be %s according to changes file." % (file, changes_version)) # Ensure the .changes lists source in the Architecture field if not changes["architecture"].has_key("source"): - reject("%s: changes file doesn't list `source' in Architecture field." % (file)); + reject("%s: changes file doesn't list `source' in Architecture field." % (file)) # Check the signature of a .dsc file if files[file]["type"] == "dsc": - dsc["fingerprint"] = utils.check_signature(file, reject); + dsc["fingerprint"] = utils.check_signature(file, reject) - files[file]["architecture"] = "source"; + files[file]["architecture"] = "source" # Not a binary or source package? Assume byhand... else: - files[file]["byhand"] = 1; - files[file]["type"] = "byhand"; + files[file]["byhand"] = 1 + files[file]["type"] = "byhand" # Per-suite file checks - files[file]["oldfiles"] = {}; + files[file]["oldfiles"] = {} for suite in changes["distribution"].keys(): # Skip byhand if files[file].has_key("byhand"): - continue; + continue # Handle component mappings for map in Cnf.ValueList("ComponentMappings"): - (source, dest) = map.split(); + (source, dest) = map.split() if files[file]["component"] == source: - files[file]["original component"] = source; - files[file]["component"] = dest; + files[file]["original component"] = source + files[file]["component"] = dest # Ensure the component is valid for the target suite if Cnf.has_key("Suite:%s::Components" % (suite)) and \ files[file]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)): - reject("unknown component `%s' for suite `%s'." % (files[file]["component"], suite)); - continue; + reject("unknown component `%s' for suite `%s'." % (files[file]["component"], suite)) + continue # Validate the component - component = files[file]["component"]; - component_id = db_access.get_component_id(component); + component = files[file]["component"] + component_id = db_access.get_component_id(component) if component_id == -1: - reject("file '%s' has unknown component '%s'." % (file, component)); - continue; + reject("file '%s' has unknown component '%s'." % (file, component)) + continue # See if the package is NEW if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): - files[file]["new"] = 1; + files[file]["new"] = 1 # Validate the priority if files[file]["priority"].find('/') != -1: - reject("file '%s' has invalid priority '%s' [contains '/']." % (file, files[file]["priority"])); + reject("file '%s' has invalid priority '%s' [contains '/']." % (file, files[file]["priority"])) # Determine the location - location = Cnf["Dir::Pool"]; - location_id = db_access.get_location_id (location, component, archive); + location = Cnf["Dir::Pool"] + location_id = db_access.get_location_id (location, component, archive) if location_id == -1: - reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive)); - files[file]["location id"] = location_id; + reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive)) + files[file]["location id"] = location_id # Check the md5sum & size against existing files (if any) - files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]); - files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]); + files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]) + files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]) if files_id == -1: - reject("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (file)); + reject("INTERNAL ERROR, get_files_id() returned multiple matches for %s." % (file)) elif files_id == -2: - reject("md5sum and/or size mismatch on existing copy of %s." % (file)); + reject("md5sum and/or size mismatch on existing copy of %s." % (file)) files[file]["files id"] = files_id # Check for packages that have moved from one component to another @@ -636,140 +636,140 @@ SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id""" % (files[file]["package"], suite, - files[file]["architecture"])); - ql = q.getresult(); + files[file]["architecture"])) + ql = q.getresult() if ql: - files[file]["othercomponents"] = ql[0][0]; + files[file]["othercomponents"] = ql[0][0] # If the .changes file says it has source, it must have source. if changes["architecture"].has_key("source"): if not has_source: - reject("no source found and Architecture line in changes mention source."); + reject("no source found and Architecture line in changes mention source.") if not has_binaries and Cnf.FindB("Dinstall::Reject::NoSourceOnly"): - reject("source only uploads are not supported."); + reject("source only uploads are not supported.") ############################################################################### def check_dsc(): - global reprocess; + global reprocess # Ensure there is source to check if not changes["architecture"].has_key("source"): - return 1; + return 1 # Find the .dsc - dsc_filename = None; + dsc_filename = None for file in files.keys(): if files[file]["type"] == "dsc": if dsc_filename: - reject("can not process a .changes file with multiple .dsc's."); - return 0; + reject("can not process a .changes file with multiple .dsc's.") + return 0 else: - dsc_filename = file; + dsc_filename = file # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: - reject("source uploads must contain a dsc file"); - return 0; + reject("source uploads must contain a dsc file") + return 0 # Parse the .dsc file try: - dsc.update(utils.parse_changes(dsc_filename, signing_rules=1)); + dsc.update(utils.parse_changes(dsc_filename, signing_rules=1)) except utils.cant_open_exc: # if not -n copy_to_holding() will have done this for us... if Options["No-Action"]: - reject("%s: can't read file." % (dsc_filename)); + reject("%s: can't read file." % (dsc_filename)) except utils.changes_parse_error_exc, line: - reject("%s: parse error, can't grok: %s." % (dsc_filename, line)); + reject("%s: parse error, can't grok: %s." % (dsc_filename, line)) except utils.invalid_dsc_format_exc, line: - reject("%s: syntax error on line %s." % (dsc_filename, line)); + reject("%s: syntax error on line %s." % (dsc_filename, line)) # Build up the file list of files mentioned by the .dsc try: - dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)); + dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)) except utils.no_files_exc: - reject("%s: no Files: field." % (dsc_filename)); - return 0; + reject("%s: no Files: field." % (dsc_filename)) + return 0 except utils.changes_parse_error_exc, line: - reject("%s: parse error, can't grok: %s." % (dsc_filename, line)); - return 0; + reject("%s: parse error, can't grok: %s." % (dsc_filename, line)) + return 0 # Enforce mandatory fields for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"): if not dsc.has_key(i): - reject("%s: missing mandatory field `%s'." % (dsc_filename, i)); - return 0; + reject("%s: missing mandatory field `%s'." % (dsc_filename, i)) + return 0 # Validate the source and version fields if not re_valid_pkg_name.match(dsc["source"]): - reject("%s: invalid source name '%s'." % (dsc_filename, dsc["source"])); + reject("%s: invalid source name '%s'." % (dsc_filename, dsc["source"])) if not re_valid_version.match(dsc["version"]): - reject("%s: invalid version number '%s'." % (dsc_filename, dsc["version"])); + reject("%s: invalid version number '%s'." % (dsc_filename, dsc["version"])) # Bumping the version number of the .dsc breaks extraction by stable's # dpkg-source. So let's not do that... if dsc["format"] != "1.0": - reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename)); + reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename)) # Validate the Maintainer field try: - utils.fix_maintainer (dsc["maintainer"]); + utils.fix_maintainer (dsc["maintainer"]) except utils.ParseMaintError, msg: reject("%s: Maintainer field ('%s') failed to parse: %s" \ - % (dsc_filename, dsc["maintainer"], msg)); + % (dsc_filename, dsc["maintainer"], msg)) # Validate the build-depends field(s) for field_name in [ "build-depends", "build-depends-indep" ]: - field = dsc.get(field_name); + field = dsc.get(field_name) if field: # Check for broken dpkg-dev lossage... if field.startswith("ARRAY"): - reject("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % (dsc_filename, field_name.title())); + reject("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % (dsc_filename, field_name.title())) # Have apt try to parse them... try: - apt_pkg.ParseSrcDepends(field); + apt_pkg.ParseSrcDepends(field) except: - reject("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title())); - pass; + reject("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title())) + pass # Ensure the version number in the .dsc matches the version number in the .changes - epochless_dsc_version = utils.re_no_epoch.sub('', dsc["version"]); - changes_version = files[dsc_filename]["version"]; + epochless_dsc_version = utils.re_no_epoch.sub('', dsc["version"]) + changes_version = files[dsc_filename]["version"] if epochless_dsc_version != files[dsc_filename]["version"]: - reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version)); + reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version)) # Ensure there is a .tar.gz in the .dsc file - has_tar = 0; + has_tar = 0 for f in dsc_files.keys(): - m = utils.re_issource.match(f); + m = utils.re_issource.match(f) if not m: - reject("%s: %s in Files field not recognised as source." % (dsc_filename, f)); - type = m.group(3); + reject("%s: %s in Files field not recognised as source." % (dsc_filename, f)) + type = m.group(3) if type == "orig.tar.gz" or type == "tar.gz": - has_tar = 1; + has_tar = 1 if not has_tar: - reject("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename)); + reject("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename)) # Ensure source is newer than existing source in target suites - reject(Katie.check_source_against_db(dsc_filename),""); + reject(Katie.check_source_against_db(dsc_filename),"") - (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(dsc_filename); - reject(reject_msg, ""); + (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(dsc_filename) + reject(reject_msg, "") if is_in_incoming: if not Options["No-Action"]: - copy_to_holding(is_in_incoming); - orig_tar_gz = os.path.basename(is_in_incoming); - files[orig_tar_gz] = {}; - files[orig_tar_gz]["size"] = os.stat(orig_tar_gz)[stat.ST_SIZE]; - files[orig_tar_gz]["md5sum"] = dsc_files[orig_tar_gz]["md5sum"]; - files[orig_tar_gz]["section"] = files[dsc_filename]["section"]; - files[orig_tar_gz]["priority"] = files[dsc_filename]["priority"]; - files[orig_tar_gz]["component"] = files[dsc_filename]["component"]; - files[orig_tar_gz]["type"] = "orig.tar.gz"; - reprocess = 2; - - return 1; + copy_to_holding(is_in_incoming) + orig_tar_gz = os.path.basename(is_in_incoming) + files[orig_tar_gz] = {} + files[orig_tar_gz]["size"] = os.stat(orig_tar_gz)[stat.ST_SIZE] + files[orig_tar_gz]["md5sum"] = dsc_files[orig_tar_gz]["md5sum"] + files[orig_tar_gz]["section"] = files[dsc_filename]["section"] + files[orig_tar_gz]["priority"] = files[dsc_filename]["priority"] + files[orig_tar_gz]["component"] = files[dsc_filename]["component"] + files[orig_tar_gz]["type"] = "orig.tar.gz" + reprocess = 2 + + return 1 ################################################################################ @@ -778,69 +778,69 @@ def get_changelog_versions(source_dir): version history out of debian/changelog for the BTS.""" # Find the .dsc (again) - dsc_filename = None; + dsc_filename = None for file in files.keys(): if files[file]["type"] == "dsc": - dsc_filename = file; + dsc_filename = file # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: - return; + return # Create a symlink mirror of the source files in our temporary directory for f in files.keys(): - m = utils.re_issource.match(f); + m = utils.re_issource.match(f) if m: - src = os.path.join(source_dir, f); + src = os.path.join(source_dir, f) # If a file is missing for whatever reason, give up. if not os.path.exists(src): - return; - type = m.group(3); + return + type = m.group(3) if type == "orig.tar.gz" and pkg.orig_tar_gz: - continue; - dest = os.path.join(os.getcwd(), f); - os.symlink(src, dest); + continue + dest = os.path.join(os.getcwd(), f) + os.symlink(src, dest) # If the orig.tar.gz is not a part of the upload, create a symlink to the # existing copy. if pkg.orig_tar_gz: - dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz)); - os.symlink(pkg.orig_tar_gz, dest); + dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz)) + os.symlink(pkg.orig_tar_gz, dest) # Extract the source - cmd = "dpkg-source -sn -x %s" % (dsc_filename); - (result, output) = commands.getstatusoutput(cmd); + cmd = "dpkg-source -sn -x %s" % (dsc_filename) + (result, output) = commands.getstatusoutput(cmd) if (result != 0): - reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result)); - reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), ""); - return; + reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result)) + reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "") + return if not Cnf.Find("Dir::Queue::BTSVersionTrack"): - return; + return # Get the upstream version - upstr_version = utils.re_no_epoch.sub('', dsc["version"]); + upstr_version = utils.re_no_epoch.sub('', dsc["version"]) if re_strip_revision.search(upstr_version): - upstr_version = re_strip_revision.sub('', upstr_version); + upstr_version = re_strip_revision.sub('', upstr_version) # Ensure the changelog file exists - changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version); + changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version) if not os.path.exists(changelog_filename): - reject("%s: debian/changelog not found in extracted source." % (dsc_filename)); - return; + reject("%s: debian/changelog not found in extracted source." % (dsc_filename)) + return # Parse the changelog - dsc["bts changelog"] = ""; - changelog_file = utils.open_file(changelog_filename); + dsc["bts changelog"] = "" + changelog_file = utils.open_file(changelog_filename) for line in changelog_file.readlines(): - m = re_changelog_versions.match(line); + m = re_changelog_versions.match(line) if m: - dsc["bts changelog"] += line; - changelog_file.close(); + dsc["bts changelog"] += line + changelog_file.close() # Check we found at least one revision in the changelog if not dsc["bts changelog"]: - reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename)); + reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename)) ######################################## @@ -851,41 +851,41 @@ def check_source(): # or c) the orig.tar.gz is MIA if not changes["architecture"].has_key("source") or reprocess == 2 \ or pkg.orig_tar_gz == -1: - return; + return # Create a temporary directory to extract the source into if Options["No-Action"]: - tmpdir = tempfile.mktemp(); + tmpdir = tempfile.mktemp() else: # We're in queue/holding and can create a random directory. - tmpdir = "%s" % (os.getpid()); - os.mkdir(tmpdir); + tmpdir = "%s" % (os.getpid()) + os.mkdir(tmpdir) # Move into the temporary directory - cwd = os.getcwd(); - os.chdir(tmpdir); + cwd = os.getcwd() + os.chdir(tmpdir) # Get the changelog version history - get_changelog_versions(cwd); + get_changelog_versions(cwd) # Move back and cleanup the temporary tree - os.chdir(cwd); + os.chdir(cwd) try: - shutil.rmtree(tmpdir); + shutil.rmtree(tmpdir) except OSError, e: if errno.errorcode[e.errno] != 'EACCES': - utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"])); + utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"])) - reject("%s: source tree could not be cleanly removed." % (dsc["source"])); + reject("%s: source tree could not be cleanly removed." % (dsc["source"])) # We probably have u-r or u-w directories so chmod everything # and try again. cmd = "chmod -R u+rwx %s" % (tmpdir) result = os.system(cmd) if result != 0: - utils.fubar("'%s' failed with result %s." % (cmd, result)); - shutil.rmtree(tmpdir); + utils.fubar("'%s' failed with result %s." % (cmd, result)) + shutil.rmtree(tmpdir) except: - utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"])); + utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"])) ################################################################################ @@ -894,48 +894,48 @@ def check_source(): def check_urgency (): if changes["architecture"].has_key("source"): if not changes.has_key("urgency"): - changes["urgency"] = Cnf["Urgency::Default"]; + changes["urgency"] = Cnf["Urgency::Default"] if changes["urgency"] not in Cnf.ValueList("Urgency::Valid"): - reject("%s is not a valid urgency; it will be treated as %s by testing." % (changes["urgency"], Cnf["Urgency::Default"]), "Warning: "); - changes["urgency"] = Cnf["Urgency::Default"]; - changes["urgency"] = changes["urgency"].lower(); + reject("%s is not a valid urgency; it will be treated as %s by testing." % (changes["urgency"], Cnf["Urgency::Default"]), "Warning: ") + changes["urgency"] = Cnf["Urgency::Default"] + changes["urgency"] = changes["urgency"].lower() ################################################################################ def check_md5sums (): for file in files.keys(): try: - file_handle = utils.open_file(file); + file_handle = utils.open_file(file) except utils.cant_open_exc: - continue; + continue # Check md5sum if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]: - reject("%s: md5sum check failed." % (file)); - file_handle.close(); + reject("%s: md5sum check failed." % (file)) + file_handle.close() # Check size - actual_size = os.stat(file)[stat.ST_SIZE]; - size = int(files[file]["size"]); + actual_size = os.stat(file)[stat.ST_SIZE] + size = int(files[file]["size"]) if size != actual_size: reject("%s: actual file size (%s) does not match size (%s) in .changes" - % (file, actual_size, size)); + % (file, actual_size, size)) for file in dsc_files.keys(): try: - file_handle = utils.open_file(file); + file_handle = utils.open_file(file) except utils.cant_open_exc: - continue; + continue # Check md5sum if apt_pkg.md5sum(file_handle) != dsc_files[file]["md5sum"]: - reject("%s: md5sum check failed." % (file)); - file_handle.close(); + reject("%s: md5sum check failed." % (file)) + file_handle.close() # Check size - actual_size = os.stat(file)[stat.ST_SIZE]; - size = int(dsc_files[file]["size"]); + actual_size = os.stat(file)[stat.ST_SIZE] + size = int(dsc_files[file]["size"]) if size != actual_size: reject("%s: actual file size (%s) does not match size (%s) in .dsc" - % (file, actual_size, size)); + % (file, actual_size, size)) ################################################################################ @@ -946,31 +946,31 @@ def check_md5sums (): def check_timestamps(): class Tar: def __init__(self, future_cutoff, past_cutoff): - self.reset(); - self.future_cutoff = future_cutoff; - self.past_cutoff = past_cutoff; + self.reset() + self.future_cutoff = future_cutoff + self.past_cutoff = past_cutoff def reset(self): - self.future_files = {}; - self.ancient_files = {}; + self.future_files = {} + self.ancient_files = {} def callback(self, Kind,Name,Link,Mode,UID,GID,Size,MTime,Major,Minor): if MTime > self.future_cutoff: - self.future_files[Name] = MTime; + self.future_files[Name] = MTime if MTime < self.past_cutoff: - self.ancient_files[Name] = MTime; + self.ancient_files[Name] = MTime #### - future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"]); - past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y")); - tar = Tar(future_cutoff, past_cutoff); + future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"]) + past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y")) + tar = Tar(future_cutoff, past_cutoff) for filename in files.keys(): if files[filename]["type"] == "deb": - tar.reset(); + tar.reset() try: - deb_file = utils.open_file(filename); - apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz"); - deb_file.seek(0); + deb_file = utils.open_file(filename) + apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz") + deb_file.seek(0) try: apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz") except SystemError, e: @@ -979,27 +979,27 @@ def check_timestamps(): raise deb_file.seek(0) apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2") - deb_file.close(); + deb_file.close() # - future_files = tar.future_files.keys(); + future_files = tar.future_files.keys() if future_files: - num_future_files = len(future_files); - future_file = future_files[0]; - future_date = tar.future_files[future_file]; + num_future_files = len(future_files) + future_file = future_files[0] + future_date = tar.future_files[future_file] reject("%s: has %s file(s) with a time stamp too far into the future (e.g. %s [%s])." % (filename, num_future_files, future_file, - time.ctime(future_date))); + time.ctime(future_date))) # - ancient_files = tar.ancient_files.keys(); + ancient_files = tar.ancient_files.keys() if ancient_files: - num_ancient_files = len(ancient_files); - ancient_file = ancient_files[0]; - ancient_date = tar.ancient_files[ancient_file]; + num_ancient_files = len(ancient_files) + ancient_file = ancient_files[0] + ancient_date = tar.ancient_files[ancient_file] reject("%s: has %s file(s) with a time stamp too ancient (e.g. %s [%s])." % (filename, num_ancient_files, ancient_file, - time.ctime(ancient_date))); + time.ctime(ancient_date))) except: - reject("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value)); + reject("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value)) ################################################################################ ################################################################################ @@ -1008,23 +1008,23 @@ def check_timestamps(): # the file is still being uploaded. def upload_too_new(): - too_new = 0; + too_new = 0 # Move back to the original directory to get accurate time stamps - cwd = os.getcwd(); - os.chdir(pkg.directory); - file_list = pkg.files.keys(); - file_list.extend(pkg.dsc_files.keys()); - file_list.append(pkg.changes_file); + cwd = os.getcwd() + os.chdir(pkg.directory) + file_list = pkg.files.keys() + file_list.extend(pkg.dsc_files.keys()) + file_list.append(pkg.changes_file) for file in file_list: try: - last_modified = time.time()-os.path.getmtime(file); + last_modified = time.time()-os.path.getmtime(file) if last_modified < int(Cnf["Dinstall::SkipTime"]): - too_new = 1; - break; + too_new = 1 + break except: - pass; - os.chdir(cwd); - return too_new; + pass + os.chdir(cwd) + return too_new ################################################################################ @@ -1032,9 +1032,9 @@ def action (): # changes["distribution"] may not exist in corner cases # (e.g. unreadable changes files) if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType): - changes["distribution"] = {}; + changes["distribution"] = {} - (summary, short_summary) = Katie.build_summaries(); + (summary, short_summary) = Katie.build_summaries() # q-unapproved hax0ring queue_info = { @@ -1055,13 +1055,13 @@ def action (): if reject_message.find("Rejected") != -1: if upload_too_new(): - print "SKIP (too new)\n" + reject_message,; - prompt = "[S]kip, Quit ?"; + print "SKIP (too new)\n" + reject_message, + prompt = "[S]kip, Quit ?" else: - print "REJECT\n" + reject_message,; - prompt = "[R]eject, Skip, Quit ?"; + print "REJECT\n" + reject_message, + prompt = "[R]eject, Skip, Quit ?" if Options["Automatic"]: - answer = 'R'; + answer = 'R' else: queue = None for q in queues: @@ -1081,23 +1081,23 @@ def action (): if Options["Automatic"]: answer = queuekey else: - print "ACCEPT\n" + reject_message + summary,; - prompt = "[A]ccept, Skip, Quit ?"; + print "ACCEPT\n" + reject_message + summary, + prompt = "[A]ccept, Skip, Quit ?" if Options["Automatic"]: - answer = 'A'; + answer = 'A' while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.match(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.match(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); + answer = m.group(1) + answer = answer[:1].upper() if answer == 'R': - os.chdir (pkg.directory); - Katie.do_reject(0, reject_message); + os.chdir (pkg.directory) + Katie.do_reject(0, reject_message) elif answer == 'A': - accept(summary, short_summary); + accept(summary, short_summary) remove_from_unchecked() elif answer == queuekey: queue_info[queue]["process"](summary) @@ -1106,24 +1106,24 @@ def action (): sys.exit(0) def remove_from_unchecked(): - os.chdir (pkg.directory); + os.chdir (pkg.directory) for file in files.keys(): - os.unlink(file); - os.unlink(pkg.changes_file); + os.unlink(file) + os.unlink(pkg.changes_file) ################################################################################ def accept (summary, short_summary): - Katie.accept(summary, short_summary); - Katie.check_override(); + Katie.accept(summary, short_summary) + Katie.check_override() ################################################################################ def move_to_dir (dest, perms=0660, changesperms=0664): - utils.move (pkg.changes_file, dest, perms=changesperms); - file_keys = files.keys(); + utils.move (pkg.changes_file, dest, perms=changesperms) + file_keys = files.keys() for file in file_keys: - utils.move (file, dest, perms=perms); + utils.move (file, dest, perms=perms) ################################################################################ @@ -1148,15 +1148,15 @@ def is_unembargo (): def queue_unembargo (summary): print "Moving to UNEMBARGOED holding area." - Logger.log(["Moving to unembargoed", pkg.changes_file]); + Logger.log(["Moving to unembargoed", pkg.changes_file]) - Katie.dump_vars(Cnf["Dir::Queue::Unembargoed"]); + Katie.dump_vars(Cnf["Dir::Queue::Unembargoed"]) move_to_dir(Cnf["Dir::Queue::Unembargoed"]) Katie.queue_build("unembargoed", Cnf["Dir::Queue::Unembargoed"]) # Check for override disparities - Katie.Subst["__SUMMARY__"] = summary; - Katie.check_override(); + Katie.Subst["__SUMMARY__"] = summary + Katie.check_override() ################################################################################ @@ -1165,15 +1165,15 @@ def is_embargo (): def queue_embargo (summary): print "Moving to EMBARGOED holding area." - Logger.log(["Moving to embargoed", pkg.changes_file]); + Logger.log(["Moving to embargoed", pkg.changes_file]) - Katie.dump_vars(Cnf["Dir::Queue::Embargoed"]); + Katie.dump_vars(Cnf["Dir::Queue::Embargoed"]) move_to_dir(Cnf["Dir::Queue::Embargoed"]) Katie.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) # Check for override disparities - Katie.Subst["__SUMMARY__"] = summary; - Katie.check_override(); + Katie.Subst["__SUMMARY__"] = summary + Katie.check_override() ################################################################################ @@ -1185,14 +1185,14 @@ def is_byhand (): def do_byhand (summary): print "Moving to BYHAND holding area." - Logger.log(["Moving to byhand", pkg.changes_file]); + Logger.log(["Moving to byhand", pkg.changes_file]) - Katie.dump_vars(Cnf["Dir::Queue::Byhand"]); + Katie.dump_vars(Cnf["Dir::Queue::Byhand"]) move_to_dir(Cnf["Dir::Queue::Byhand"]) # Check for override disparities - Katie.Subst["__SUMMARY__"] = summary; - Katie.check_override(); + Katie.Subst["__SUMMARY__"] = summary + Katie.check_override() ################################################################################ @@ -1203,19 +1203,19 @@ def is_new (): return 0 def acknowledge_new (summary): - Subst = Katie.Subst; + Subst = Katie.Subst print "Moving to NEW holding area." - Logger.log(["Moving to new", pkg.changes_file]); + Logger.log(["Moving to new", pkg.changes_file]) - Katie.dump_vars(Cnf["Dir::Queue::New"]); + Katie.dump_vars(Cnf["Dir::Queue::New"]) move_to_dir(Cnf["Dir::Queue::New"]) if not Options["No-Mail"]: - print "Sending new ack."; - Subst["__SUMMARY__"] = summary; - new_ack_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.new"); - utils.send_mail(new_ack_message); + print "Sending new ack." + Subst["__SUMMARY__"] = summary + new_ack_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.new") + utils.send_mail(new_ack_message) ################################################################################ @@ -1229,129 +1229,129 @@ def acknowledge_new (summary): # file. def process_it (changes_file): - global reprocess, reject_message; + global reprocess, reject_message # Reset some globals - reprocess = 1; - Katie.init_vars(); + reprocess = 1 + Katie.init_vars() # Some defaults in case we can't fully process the .changes file - changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]; - changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]; - reject_message = ""; + changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"] + changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"] + reject_message = "" # Absolutize the filename to avoid the requirement of being in the # same directory as the .changes file. - pkg.changes_file = os.path.abspath(changes_file); + pkg.changes_file = os.path.abspath(changes_file) # Remember where we are so we can come back after cd-ing into the # holding directory. - pkg.directory = os.getcwd(); + pkg.directory = os.getcwd() try: # If this is the Real Thing(tm), copy things into a private # holding directory first to avoid replacable file races. if not Options["No-Action"]: - os.chdir(Cnf["Dir::Queue::Holding"]); - copy_to_holding(pkg.changes_file); + os.chdir(Cnf["Dir::Queue::Holding"]) + copy_to_holding(pkg.changes_file) # Relativize the filename so we use the copy in holding # rather than the original... - pkg.changes_file = os.path.basename(pkg.changes_file); - changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject); + pkg.changes_file = os.path.basename(pkg.changes_file) + changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject) if changes["fingerprint"]: - valid_changes_p = check_changes(); + valid_changes_p = check_changes() else: - valid_changes_p = 0; + valid_changes_p = 0 if valid_changes_p: while reprocess: - check_distributions(); - check_files(); - valid_dsc_p = check_dsc(); + check_distributions() + check_files() + valid_dsc_p = check_dsc() if valid_dsc_p: - check_source(); - check_md5sums(); - check_urgency(); - check_timestamps(); - Katie.update_subst(reject_message); - action(); + check_source() + check_md5sums() + check_urgency() + check_timestamps() + Katie.update_subst(reject_message) + action() except SystemExit: - raise; + raise except: - print "ERROR"; - traceback.print_exc(file=sys.stderr); - pass; + print "ERROR" + traceback.print_exc(file=sys.stderr) + pass # Restore previous WD - os.chdir(pkg.directory); + os.chdir(pkg.directory) ############################################################################### def main(): - global Cnf, Options, Logger; + global Cnf, Options, Logger - changes_files = init(); + changes_files = init() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: - Options["Automatic"] = ""; + Options["Automatic"] = "" # Ensure all the arguments we were given are .changes files for file in changes_files: if not file.endswith(".changes"): - utils.warn("Ignoring '%s' because it's not a .changes file." % (file)); - changes_files.remove(file); + utils.warn("Ignoring '%s' because it's not a .changes file." % (file)) + changes_files.remove(file) if changes_files == []: - utils.fubar("Need at least one .changes file as an argument."); + utils.fubar("Need at least one .changes file as an argument.") # Check that we aren't going to clash with the daily cron job if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]: - utils.fubar("Archive maintenance in progress. Try again later."); + utils.fubar("Archive maintenance in progress. Try again later.") # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: - lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT); + lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT) try: - fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB); + fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError, e: if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN': - utils.fubar("Couldn't obtain lock; assuming another jennifer is already running."); + utils.fubar("Couldn't obtain lock; assuming another jennifer is already running.") else: - raise; - Logger = Katie.Logger = logging.Logger(Cnf, "jennifer"); + raise + Logger = Katie.Logger = logging.Logger(Cnf, "jennifer") # debian-{devel-,}-changes@lists.debian.org toggles writes access based on this header - bcc = "X-Katie: %s" % (jennifer_version); + bcc = "X-Katie: %s" % (jennifer_version) if Cnf.has_key("Dinstall::Bcc"): - Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); + Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) else: - Katie.Subst["__BCC__"] = bcc; + Katie.Subst["__BCC__"] = bcc # Sort the .changes files so that we process sourceful ones first - changes_files.sort(utils.changes_compare); + changes_files.sort(utils.changes_compare) # Process the changes files for changes_file in changes_files: - print "\n" + changes_file; + print "\n" + changes_file try: - process_it (changes_file); + process_it (changes_file) finally: if not Options["No-Action"]: - clean_holding(); + clean_holding() - accept_count = Katie.accept_count; - accept_bytes = Katie.accept_bytes; + accept_count = Katie.accept_count + accept_bytes = Katie.accept_bytes if accept_count: sets = "set" if accept_count > 1: - sets = "sets"; - print "Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes))); - Logger.log(["total",accept_count,accept_bytes]); + sets = "sets" + print "Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes))) + Logger.log(["total",accept_count,accept_bytes]) if not Options["No-Action"]: - Logger.close(); + Logger.close() ################################################################################ diff --git a/dak/queue_report.py b/dak/queue_report.py index d33b19e3..d2630447 100755 --- a/dak/queue_report.py +++ b/dak/queue_report.py @@ -35,15 +35,15 @@ ################################################################################ -import copy, glob, os, stat, sys, time; -import apt_pkg; -import katie, utils; -import encodings.utf_8, encodings.latin_1, string; +import copy, glob, os, stat, sys, time +import apt_pkg +import katie, utils +import encodings.utf_8, encodings.latin_1, string -Cnf = None; -Katie = None; -direction = []; -row_number = 0; +Cnf = None +Katie = None +direction = [] +row_number = 0 ################################################################################ @@ -70,58 +70,58 @@ Prints a report of packages in queue directories (usually new and byhand). def plural(x): if x > 1: - return "s"; + return "s" else: - return ""; + return "" ################################################################################ def time_pp(x): if x < 60: - unit="second"; + unit="second" elif x < 3600: - x /= 60; - unit="minute"; + x /= 60 + unit="minute" elif x < 86400: - x /= 3600; - unit="hour"; + x /= 3600 + unit="hour" elif x < 604800: - x /= 86400; - unit="day"; + x /= 86400 + unit="day" elif x < 2419200: - x /= 604800; - unit="week"; + x /= 604800 + unit="week" elif x < 29030400: - x /= 2419200; - unit="month"; + x /= 2419200 + unit="month" else: - x /= 29030400; - unit="year"; - x = int(x); - return "%s %s%s" % (x, unit, plural(x)); + x /= 29030400 + unit="year" + x = int(x) + return "%s %s%s" % (x, unit, plural(x)) ################################################################################ def sg_compare (a, b): - a = a[1]; - b = b[1]; + a = a[1] + b = b[1] """Sort by have note, time of oldest upload.""" # Sort by have note - a_note_state = a["note_state"]; - b_note_state = b["note_state"]; + a_note_state = a["note_state"] + b_note_state = b["note_state"] if a_note_state < b_note_state: - return -1; + return -1 elif a_note_state > b_note_state: - return 1; + return 1 # Sort by time of oldest upload - return cmp(a["oldest"], b["oldest"]); + return cmp(a["oldest"], b["oldest"]) ############################################################ def sortfunc(a,b): for sorting in direction: - (sortkey, way, time) = sorting; + (sortkey, way, time) = sorting ret = 0 if time == "m": x=int(a[sortkey]/60) @@ -225,13 +225,13 @@ def table_footer(type, source_count, total_count): def force_to_latin(s): """Forces a string to Latin-1.""" - latin1_s = unicode(s,'utf-8'); - return latin1_s.encode('iso8859-1', 'replace'); + latin1_s = unicode(s,'utf-8') + return latin1_s.encode('iso8859-1', 'replace') def table_row(source, version, arch, last_mod, maint, distribution, closes): - global row_number; + global row_number if row_number % 2 != 0: print "" @@ -241,119 +241,119 @@ def table_row(source, version, arch, last_mod, maint, distribution, closes): tdclass = "sid" for dist in distribution: if dist == "experimental": - tdclass = "exp"; - print "%s" % (tdclass, source); + tdclass = "exp" + print "%s" % (tdclass, source) print "" % (tdclass) for vers in version.split(): - print "%s
" % (vers); - print "%s" % (tdclass, arch, tdclass); + print "%s
" % (vers) + print "%s" % (tdclass, arch, tdclass) for dist in distribution: - print "%s
" % (dist); - print "%s" % (tdclass, last_mod); - (name, mail) = maint.split(":"); - name = force_to_latin(name); + print "%s
" % (dist) + print "%s" % (tdclass, last_mod) + (name, mail) = maint.split(":") + name = force_to_latin(name) - print "%s" % (tdclass, mail, name); + print "%s" % (tdclass, mail, name) print "" % (tdclass) for close in closes: - print "#%s
" % (close, close); - print ""; - row_number+=1; + print "#%s
" % (close, close) + print "" + row_number+=1 ############################################################ def process_changes_files(changes_files, type): - msg = ""; - cache = {}; + msg = "" + cache = {} # Read in all the .changes files for filename in changes_files: try: - Katie.pkg.changes_file = filename; - Katie.init_vars(); - Katie.update_vars(); - cache[filename] = copy.copy(Katie.pkg.changes); - cache[filename]["filename"] = filename; + Katie.pkg.changes_file = filename + Katie.init_vars() + Katie.update_vars() + cache[filename] = copy.copy(Katie.pkg.changes) + cache[filename]["filename"] = filename except: - break; + break # Divide the .changes into per-source groups - per_source = {}; + per_source = {} for filename in cache.keys(): - source = cache[filename]["source"]; + source = cache[filename]["source"] if not per_source.has_key(source): - per_source[source] = {}; - per_source[source]["list"] = []; - per_source[source]["list"].append(cache[filename]); + per_source[source] = {} + per_source[source]["list"] = [] + per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): - source_list = per_source[source]["list"]; - first = source_list[0]; - oldest = os.stat(first["filename"])[stat.ST_MTIME]; - have_note = 0; + source_list = per_source[source]["list"] + first = source_list[0] + oldest = os.stat(first["filename"])[stat.ST_MTIME] + have_note = 0 for d in per_source[source]["list"]: - mtime = os.stat(d["filename"])[stat.ST_MTIME]; + mtime = os.stat(d["filename"])[stat.ST_MTIME] if Cnf.has_key("Helena::Options::New"): if mtime > oldest: - oldest = mtime; + oldest = mtime else: if mtime < oldest: - oldest = mtime; - have_note += (d.has_key("lisa note")); - per_source[source]["oldest"] = oldest; + oldest = mtime + have_note += (d.has_key("lisa note")) + per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none elif have_note < len(source_list): per_source[source]["note_state"] = 1; # some else: per_source[source]["note_state"] = 2; # all - per_source_items = per_source.items(); - per_source_items.sort(sg_compare); - - entries = []; - max_source_len = 0; - max_version_len = 0; - max_arch_len = 0; - maintainer = {}; - maint=""; - distribution=""; - closes=""; - source_exists=""; + per_source_items = per_source.items() + per_source_items.sort(sg_compare) + + entries = [] + max_source_len = 0 + max_version_len = 0 + max_arch_len = 0 + maintainer = {} + maint="" + distribution="" + closes="" + source_exists="" for i in per_source_items: - last_modified = time.time()-i[1]["oldest"]; - source = i[1]["list"][0]["source"]; + last_modified = time.time()-i[1]["oldest"] + source = i[1]["list"][0]["source"] if len(source) > max_source_len: - max_source_len = len(source); - arches = {}; - versions = {}; + max_source_len = len(source) + arches = {} + versions = {} for j in i[1]["list"]: if Cnf.has_key("Helena::Options::New"): try: (maintainer["maintainer822"], maintainer["maintainer2047"], maintainer["maintainername"], maintainer["maintaineremail"]) = \ - utils.fix_maintainer (j["maintainer"]); + utils.fix_maintainer (j["maintainer"]) except utils.ParseMaintError, msg: - print "Problems while parsing maintainer address\n"; - maintainer["maintainername"] = "Unknown"; - maintainer["maintaineremail"] = "Unknown"; - maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"]); - distribution=j["distribution"].keys(); - closes=j["closes"].keys(); + print "Problems while parsing maintainer address\n" + maintainer["maintainername"] = "Unknown" + maintainer["maintaineremail"] = "Unknown" + maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"]) + distribution=j["distribution"].keys() + closes=j["closes"].keys() for arch in j["architecture"].keys(): - arches[arch] = ""; - version = j["version"]; - versions[version] = ""; - arches_list = arches.keys(); - arches_list.sort(utils.arch_compare_sw); - arch_list = " ".join(arches_list); - version_list = " ".join(versions.keys()); + arches[arch] = "" + version = j["version"] + versions[version] = "" + arches_list = arches.keys() + arches_list.sort(utils.arch_compare_sw) + arch_list = " ".join(arches_list) + version_list = " ".join(versions.keys()) if len(version_list) > max_version_len: - max_version_len = len(version_list); + max_version_len = len(version_list) if len(arch_list) > max_arch_len: - max_arch_len = len(arch_list); + max_arch_len = len(arch_list) if i[1]["note_state"]: - note = " | [N]"; + note = " | [N]" else: - note = ""; - entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes]); + note = "" + entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes]) # direction entry consists of "Which field, which direction, time-consider" where # time-consider says how we should treat last_modified. Thats all. @@ -364,28 +364,28 @@ def process_changes_files(changes_files, type): age = Cnf["Helena::Options::Age"] if Cnf.has_key("Helena::Options::New"): # If we produce html we always have oldest first. - direction.append([4,-1,"ao"]); + direction.append([4,-1,"ao"]) else: if Cnf.has_key("Helena::Options::Sort"): for i in Cnf["Helena::Options::Sort"].split(","): if i == "ao": # Age, oldest first. - direction.append([4,-1,age]); + direction.append([4,-1,age]) elif i == "an": # Age, newest first. - direction.append([4,1,age]); + direction.append([4,1,age]) elif i == "na": # Name, Ascending. - direction.append([0,1,0]); + direction.append([0,1,0]) elif i == "nd": # Name, Descending. - direction.append([0,-1,0]); + direction.append([0,-1,0]) elif i == "nl": # Notes last. - direction.append([3,1,0]); + direction.append([3,1,0]) elif i == "nf": # Notes first. - direction.append([3,-1,0]); + direction.append([3,-1,0]) entries.sort(lambda x, y: sortfunc(x, y)) # Yes, in theory you can add several sort options at the commandline with. But my mind is to small # at the moment to come up with a real good sorting function that considers all the sidesteps you @@ -393,75 +393,75 @@ def process_changes_files(changes_files, type): # Will be enhanced in the future. if Cnf.has_key("Helena::Options::New"): - direction.append([4,1,"ao"]); + direction.append([4,1,"ao"]) entries.sort(lambda x, y: sortfunc(x, y)) # Output for a html file. First table header. then table_footer. # Any line between them is then a printed from subroutine table_row. if len(entries) > 0: - table_header(type.upper()); + table_header(type.upper()) for entry in entries: - (source, version_list, arch_list, note, last_modified, maint, distribution, closes) = entry; - table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes); - total_count = len(changes_files); - source_count = len(per_source_items); - table_footer(type.upper(), source_count, total_count); + (source, version_list, arch_list, note, last_modified, maint, distribution, closes) = entry + table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes) + total_count = len(changes_files) + source_count = len(per_source_items) + table_footer(type.upper(), source_count, total_count) else: # The "normal" output without any formatting. format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len) - msg = ""; + msg = "" for entry in entries: - (source, version_list, arch_list, note, last_modified, undef, undef, undef) = entry; - msg += format % (source, version_list, arch_list, note, time_pp(last_modified)); + (source, version_list, arch_list, note, last_modified, undef, undef, undef) = entry + msg += format % (source, version_list, arch_list, note, time_pp(last_modified)) if msg: - total_count = len(changes_files); - source_count = len(per_source_items); - print type.upper(); - print "-"*len(type); + total_count = len(changes_files) + source_count = len(per_source_items) + print type.upper() + print "-"*len(type) print - print msg; - print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)); + print msg + print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)) print ################################################################################ def main(): - global Cnf, Katie; + global Cnf, Katie - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('h',"help","Helena::Options::Help"), ('n',"new","Helena::Options::New"), ('s',"sort","Helena::Options::Sort", "HasArg"), - ('a',"age","Helena::Options::Age", "HasArg")]; + ('a',"age","Helena::Options::Age", "HasArg")] for i in [ "help" ]: if not Cnf.has_key("Helena::Options::%s" % (i)): - Cnf["Helena::Options::%s" % (i)] = ""; + Cnf["Helena::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Helena::Options") if Options["Help"]: - usage(); + usage() - Katie = katie.Katie(Cnf); + Katie = katie.Katie(Cnf) if Cnf.has_key("Helena::Options::New"): - header(); + header() - directories = Cnf.ValueList("Helena::Directories"); + directories = Cnf.ValueList("Helena::Directories") if not directories: - directories = [ "byhand", "new" ]; + directories = [ "byhand", "new" ] for directory in directories: - changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)])); - process_changes_files(changes_files, directory); + changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)])) + process_changes_files(changes_files, directory) if Cnf.has_key("Helena::Options::New"): - footer(); + footer() ################################################################################ if __name__ == '__main__': - main(); + main() diff --git a/dak/reject_proposed_updates.py b/dak/reject_proposed_updates.py index 72bd8800..dfcbd31e 100755 --- a/dak/reject_proposed_updates.py +++ b/dak/reject_proposed_updates.py @@ -20,20 +20,20 @@ ################################################################################ -import os, pg, sys; -import db_access, katie, logging, utils; -import apt_pkg; +import os, pg, sys +import db_access, katie, logging, utils +import apt_pkg ################################################################################ # Globals -lauren_version = "$Revision: 1.4 $"; +lauren_version = "$Revision: 1.4 $" -Cnf = None; -Options = None; -projectB = None; -Katie = None; -Logger = None; +Cnf = None +Options = None +projectB = None +Katie = None +Logger = None ################################################################################ @@ -49,154 +49,154 @@ Manually reject the .CHANGES file(s). ################################################################################ def main(): - global Cnf, Logger, Options, projectB, Katie; + global Cnf, Logger, Options, projectB, Katie - Cnf = utils.get_conf(); + Cnf = utils.get_conf() Arguments = [('h',"help","Lauren::Options::Help"), ('m',"manual-reject","Lauren::Options::Manual-Reject", "HasArg"), - ('s',"no-mail", "Lauren::Options::No-Mail")]; + ('s',"no-mail", "Lauren::Options::No-Mail")] for i in [ "help", "manual-reject", "no-mail" ]: if not Cnf.has_key("Lauren::Options::%s" % (i)): - Cnf["Lauren::Options::%s" % (i)] = ""; + Cnf["Lauren::Options::%s" % (i)] = "" - arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Lauren::Options"); + Options = Cnf.SubTree("Lauren::Options") if Options["Help"]: - usage(); + usage() if not arguments: - utils.fubar("need at least one .changes filename as an argument."); + utils.fubar("need at least one .changes filename as an argument.") - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) - Katie = katie.Katie(Cnf); - Logger = Katie.Logger = logging.Logger(Cnf, "lauren"); + Katie = katie.Katie(Cnf) + Logger = Katie.Logger = logging.Logger(Cnf, "lauren") - bcc = "X-Katie: lauren %s" % (lauren_version); + bcc = "X-Katie: lauren %s" % (lauren_version) if Cnf.has_key("Dinstall::Bcc"): - Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); + Katie.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) else: - Katie.Subst["__BCC__"] = bcc; + Katie.Subst["__BCC__"] = bcc for arg in arguments: - arg = utils.validate_changes_file_arg(arg); - Katie.pkg.changes_file = arg; - Katie.init_vars(); - cwd = os.getcwd(); - os.chdir(Cnf["Suite::Proposed-Updates::CopyKatie"]); - Katie.update_vars(); - os.chdir(cwd); - Katie.update_subst(); + arg = utils.validate_changes_file_arg(arg) + Katie.pkg.changes_file = arg + Katie.init_vars() + cwd = os.getcwd() + os.chdir(Cnf["Suite::Proposed-Updates::CopyKatie"]) + Katie.update_vars() + os.chdir(cwd) + Katie.update_subst() print arg - done = 0; - prompt = "Manual reject, [S]kip, Quit ?"; + done = 0 + prompt = "Manual reject, [S]kip, Quit ?" while not done: - answer = "XXX"; + answer = "XXX" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.search(prompt) if answer == "": answer = m.group(1) answer = answer[:1].upper() if answer == 'M': - aborted = reject(Options["Manual-Reject"]); + aborted = reject(Options["Manual-Reject"]) if not aborted: - done = 1; + done = 1 elif answer == 'S': - done = 1; + done = 1 elif answer == 'Q': sys.exit(0) - Logger.close(); + Logger.close() ################################################################################ def reject (reject_message = ""): - files = Katie.pkg.files; - dsc = Katie.pkg.dsc; - changes_file = Katie.pkg.changes_file; + files = Katie.pkg.files + dsc = Katie.pkg.dsc + changes_file = Katie.pkg.changes_file # If we weren't given a manual rejection message, spawn an editor # so the user can add one in... if not reject_message: - temp_filename = utils.temp_filename(); + temp_filename = utils.temp_filename() editor = os.environ.get("EDITOR","vi") - answer = 'E'; + answer = 'E' while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - file = utils.open_file(temp_filename); - reject_message = "".join(file.readlines()); - file.close(); - print "Reject message:"; - print utils.prefix_multi_line_string(reject_message," ", include_blank_lines=1); + file = utils.open_file(temp_filename) + reject_message = "".join(file.readlines()) + file.close() + print "Reject message:" + print utils.prefix_multi_line_string(reject_message," ", include_blank_lines=1) prompt = "[R]eject, Edit, Abandon, Quit ?" - answer = "XXX"; + answer = "XXX" while prompt.find(answer) == -1: - answer = utils.our_raw_input(prompt); - m = katie.re_default_answer.search(prompt); + answer = utils.our_raw_input(prompt) + m = katie.re_default_answer.search(prompt) if answer == "": - answer = m.group(1); - answer = answer[:1].upper(); - os.unlink(temp_filename); + answer = m.group(1) + answer = answer[:1].upper() + os.unlink(temp_filename) if answer == 'A': - return 1; + return 1 elif answer == 'Q': - sys.exit(0); + sys.exit(0) print "Rejecting.\n" # Reject the .changes file - Katie.force_reject([changes_file]); + Katie.force_reject([changes_file]) # Setup the .reason file - reason_filename = changes_file[:-8] + ".reason"; - reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename; + reason_filename = changes_file[:-8] + ".reason" + reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename # If we fail here someone is probably trying to exploit the race # so let's just raise an exception ... if os.path.exists(reject_filename): - os.unlink(reject_filename); - reject_fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + os.unlink(reject_filename) + reject_fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) # Build up the rejection email - user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]); + user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]) - Katie.Subst["__REJECTOR_ADDRESS__"] = user_email_address; - Katie.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message; - Katie.Subst["__STABLE_REJECTOR__"] = Cnf["Lauren::StableRejector"]; - Katie.Subst["__MORE_INFO_URL__"] = Cnf["Lauren::MoreInfoURL"]; - Katie.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; - reject_mail_message = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lauren.stable-rejected"); + Katie.Subst["__REJECTOR_ADDRESS__"] = user_email_address + Katie.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message + Katie.Subst["__STABLE_REJECTOR__"] = Cnf["Lauren::StableRejector"] + Katie.Subst["__MORE_INFO_URL__"] = Cnf["Lauren::MoreInfoURL"] + Katie.Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + reject_mail_message = utils.TemplateSubst(Katie.Subst,Cnf["Dir::Templates"]+"/lauren.stable-rejected") # Write the rejection email out as the .reason file - os.write(reject_fd, reject_mail_message); - os.close(reject_fd); + os.write(reject_fd, reject_mail_message) + os.close(reject_fd) # Remove the packages from proposed-updates - suite_id = db_access.get_suite_id('proposed-updates'); + suite_id = db_access.get_suite_id('proposed-updates') - projectB.query("BEGIN WORK"); + projectB.query("BEGIN WORK") # Remove files from proposed-updates suite for file in files.keys(): if files[file]["type"] == "dsc": - package = dsc["source"]; + package = dsc["source"] version = dsc["version"]; # NB: not files[file]["version"], that has no epoch - q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version)); - ql = q.getresult(); + q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version)) + ql = q.getresult() if not ql: - utils.fubar("reject: Couldn't find %s_%s in source table." % (package, version)); - source_id = ql[0][0]; - projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)); + utils.fubar("reject: Couldn't find %s_%s in source table." % (package, version)) + source_id = ql[0][0] + projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)) elif files[file]["type"] == "deb": - package = files[file]["package"]; - version = files[file]["version"]; - architecture = files[file]["architecture"]; - q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)); - ql = q.getresult(); + package = files[file]["package"] + version = files[file]["version"] + architecture = files[file]["architecture"] + q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) + ql = q.getresult() # Horrible hack to work around partial replacement of # packages with newer versions (from different source @@ -204,24 +204,24 @@ def reject (reject_message = ""): # newer version of the package and only do the # warn&continue thing if it finds one. if not ql: - utils.warn("reject: Couldn't find %s_%s_%s in binaries table." % (package, version, architecture)); + utils.warn("reject: Couldn't find %s_%s_%s in binaries table." % (package, version, architecture)) else: - binary_id = ql[0][0]; - projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)); - projectB.query("COMMIT WORK"); + binary_id = ql[0][0] + projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)) + projectB.query("COMMIT WORK") # Send the rejection mail if appropriate if not Options["No-Mail"]: - utils.send_mail(reject_mail_message); + utils.send_mail(reject_mail_message) # Finally remove the .katie file - katie_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyKatie"], os.path.basename(changes_file[:-8]+".katie")); - os.unlink(katie_file); + katie_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyKatie"], os.path.basename(changes_file[:-8]+".katie")) + os.unlink(katie_file) - Logger.log(["rejected", changes_file]); - return 0; + Logger.log(["rejected", changes_file]) + return 0 ################################################################################ if __name__ == '__main__': - main(); + main() diff --git a/dak/rm.py b/dak/rm.py index 3b2596cc..10d1dd01 100755 --- a/dak/rm.py +++ b/dak/rm.py @@ -40,20 +40,20 @@ ################################################################################ -import commands, os, pg, re, sys; -import utils, db_access; -import apt_pkg, apt_inst; +import commands, os, pg, re, sys +import utils, db_access +import apt_pkg, apt_inst ################################################################################ -re_strip_source_version = re.compile (r'\s+.*$'); -re_build_dep_arch = re.compile(r"\[[^]]+\]"); +re_strip_source_version = re.compile (r'\s+.*$') +re_build_dep_arch = re.compile(r"\[[^]]+\]") ################################################################################ -Cnf = None; -Options = None; -projectB = None; +Cnf = None +Options = None +projectB = None ################################################################################ @@ -87,10 +87,10 @@ ARCH, BUG#, COMPONENT and SUITE can be comma (or space) separated lists, e.g. # the fuck are we gonna do now? What are we gonna do?" def game_over(): - answer = utils.our_raw_input("Continue (y/N)? ").lower(); + answer = utils.our_raw_input("Continue (y/N)? ").lower() if answer != "y": print "Aborted." - sys.exit(1); + sys.exit(1) ################################################################################ @@ -98,20 +98,20 @@ def reverse_depends_check(removals, suites): print "Checking reverse dependencies..." components = Cnf.ValueList("Suite::%s::Components" % suites[0]) dep_problem = 0 - p2c = {}; + p2c = {} for architecture in Cnf.ValueList("Suite::%s::Architectures" % suites[0]): if architecture in ["source", "all"]: continue - deps = {}; - virtual_packages = {}; + deps = {} + virtual_packages = {} for component in components: filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suites[0], component, architecture) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename(); - (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)); + temp_filename = utils.temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): - utils.fubar("Gunzip invocation failed!\n%s\n" % (output), result); - packages = utils.open_file(temp_filename); + utils.fubar("Gunzip invocation failed!\n%s\n" % (output), result) + packages = utils.open_file(temp_filename) Packages = apt_pkg.ParseTagFile(packages) while Packages.Step(): package = Packages.Section.Find("Package") @@ -133,9 +133,9 @@ def reverse_depends_check(removals, suites): virtual_packages[virtual_pkg] = 0 if package not in removals: virtual_packages[virtual_pkg] += 1 - p2c[package] = component; + p2c[package] = component packages.close() - os.unlink(temp_filename); + os.unlink(temp_filename) # If a virtual package is only provided by the to-be-removed # packages, treat the virtual package as to-be-removed too. @@ -160,19 +160,19 @@ def reverse_depends_check(removals, suites): if dep_package in removals: unsat += 1 if unsat == len(dep): - component = p2c[package]; + component = p2c[package] if component != "main": - what = "%s/%s" % (package, component); + what = "%s/%s" % (package, component) else: - what = "** %s" % (package); - print "%s has an unsatisfied dependency on %s: %s" % (what, architecture, utils.pp_deps(dep)); + what = "** %s" % (package) + print "%s has an unsatisfied dependency on %s: %s" % (what, architecture, utils.pp_deps(dep)) dep_problem = 1 # Check source dependencies (Build-Depends and Build-Depends-Indep) for component in components: filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suites[0], component) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename(); + temp_filename = utils.temp_filename() result, output = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if result != 0: sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) @@ -199,9 +199,9 @@ def reverse_depends_check(removals, suites): unsat += 1 if unsat == len(dep): if component != "main": - source = "%s/%s" % (source, component); + source = "%s/%s" % (source, component) else: - source = "** %s" % (source); + source = "** %s" % (source) print "%s has an unsatisfied build-dependency: %s" % (source, utils.pp_deps(dep)) dep_problem = 1 sources.close() @@ -218,7 +218,7 @@ def reverse_depends_check(removals, suites): ################################################################################ def main (): - global Cnf, Options, projectB; + global Cnf, Options, projectB Cnf = utils.get_conf() @@ -234,43 +234,43 @@ def main (): ('p',"partial", "Melanie::Options::Partial"), ('s',"suite","Melanie::Options::Suite", "HasArg"), ('S',"source-only", "Melanie::Options::Source-Only"), - ]; + ] for i in [ "architecture", "binary-only", "carbon-copy", "component", "done", "help", "no-action", "partial", "rdep-check", "reason", "source-only" ]: if not Cnf.has_key("Melanie::Options::%s" % (i)): - Cnf["Melanie::Options::%s" % (i)] = ""; + Cnf["Melanie::Options::%s" % (i)] = "" if not Cnf.has_key("Melanie::Options::Suite"): - Cnf["Melanie::Options::Suite"] = "unstable"; + Cnf["Melanie::Options::Suite"] = "unstable" - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Melanie::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + db_access.init(Cnf, projectB) # Sanity check options if not arguments: - utils.fubar("need at least one package name as an argument."); + utils.fubar("need at least one package name as an argument.") if Options["Architecture"] and Options["Source-Only"]: - utils.fubar("can't use -a/--architecutre and -S/--source-only options simultaneously."); + utils.fubar("can't use -a/--architecutre and -S/--source-only options simultaneously.") if Options["Binary-Only"] and Options["Source-Only"]: - utils.fubar("can't use -b/--binary-only and -S/--source-only options simultaneously."); + utils.fubar("can't use -b/--binary-only and -S/--source-only options simultaneously.") if Options.has_key("Carbon-Copy") and not Options.has_key("Done"): - utils.fubar("can't use -C/--carbon-copy without also using -d/--done option."); + utils.fubar("can't use -C/--carbon-copy without also using -d/--done option.") if Options["Architecture"] and not Options["Partial"]: - utils.warn("-a/--architecture implies -p/--partial."); - Options["Partial"] = "true"; + utils.warn("-a/--architecture implies -p/--partial.") + Options["Partial"] = "true" # Force the admin to tell someone if we're not doing a rene-led removal # (or closing a bug, which counts as telling someone). if not Options["No-Action"] and not Options["Carbon-Copy"] \ and not Options["Done"] and Options["Reason"].find("[rene]") == -1: - utils.fubar("Need a -C/--carbon-copy if not closing a bug and not doing a rene-led removal."); + utils.fubar("Need a -C/--carbon-copy if not closing a bug and not doing a rene-led removal.") # Process -C/--carbon-copy # @@ -279,60 +279,60 @@ def main (): # 2) the keyword 'package' - cc's $package@packages.debian.org for every argument # 3) contains a '@' - assumed to be an email address, used unmofidied # - carbon_copy = []; + carbon_copy = [] for copy_to in utils.split_args(Options.get("Carbon-Copy")): if utils.str_isnum(copy_to): - carbon_copy.append(copy_to + "@" + Cnf["Dinstall::BugServer"]); + carbon_copy.append(copy_to + "@" + Cnf["Dinstall::BugServer"]) elif copy_to == 'package': for package in arguments: - carbon_copy.append(package + "@" + Cnf["Dinstall::PackagesServer"]); + carbon_copy.append(package + "@" + Cnf["Dinstall::PackagesServer"]) if Cnf.has_key("Dinstall::TrackingServer"): - carbon_copy.append(package + "@" + Cnf["Dinstall::TrackingServer"]); + carbon_copy.append(package + "@" + Cnf["Dinstall::TrackingServer"]) elif '@' in copy_to: - carbon_copy.append(copy_to); + carbon_copy.append(copy_to) else: - utils.fubar("Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address." % (copy_to)); + utils.fubar("Invalid -C/--carbon-copy argument '%s'; not a bug number, 'package' or email address." % (copy_to)) if Options["Binary-Only"]: - field = "b.package"; + field = "b.package" else: - field = "s.source"; - con_packages = "AND %s IN (%s)" % (field, ", ".join(map(repr, arguments))); + field = "s.source" + con_packages = "AND %s IN (%s)" % (field, ", ".join(map(repr, arguments))) (con_suites, con_architectures, con_components, check_source) = \ - utils.parse_args(Options); + utils.parse_args(Options) # Additional suite checks - suite_ids_list = []; - suites = utils.split_args(Options["Suite"]); - suites_list = utils.join_with_commas_and(suites); + suite_ids_list = [] + suites = utils.split_args(Options["Suite"]) + suites_list = utils.join_with_commas_and(suites) if not Options["No-Action"]: for suite in suites: - suite_id = db_access.get_suite_id(suite); + suite_id = db_access.get_suite_id(suite) if suite_id != -1: - suite_ids_list.append(suite_id); + suite_ids_list.append(suite_id) if suite == "stable": print "**WARNING** About to remove from the stable suite!" print "This should only be done just prior to a (point) release and not at" print "any other time." - game_over(); + game_over() elif suite == "testing": print "**WARNING About to remove from the testing suite!" print "There's no need to do this normally as removals from unstable will" print "propogate to testing automagically." - game_over(); + game_over() # Additional architecture checks if Options["Architecture"] and check_source: - utils.warn("'source' in -a/--argument makes no sense and is ignored."); + utils.warn("'source' in -a/--argument makes no sense and is ignored.") # Additional component processing - over_con_components = con_components.replace("c.id", "component"); + over_con_components = con_components.replace("c.id", "component") print "Working...", - sys.stdout.flush(); - to_remove = []; - maintainers = {}; + sys.stdout.flush() + to_remove = [] + maintainers = {} # We have 3 modes of package selection: binary-only, source-only # and source+binary. The first two are trivial and obvious; the @@ -341,193 +341,193 @@ def main (): if Options["Binary-Only"]: # Binary-only - q = projectB.query("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures)); + q = projectB.query("SELECT b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s %s" % (con_packages, con_suites, con_components, con_architectures)) for i in q.getresult(): - to_remove.append(i); + to_remove.append(i) else: # Source-only - source_packages = {}; - q = projectB.query("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components)); + source_packages = {} + q = projectB.query("SELECT l.path, f.filename, s.source, s.version, 'source', s.id, s.maintainer FROM source s, src_associations sa, suite su, files f, location l, component c WHERE sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components)) for i in q.getresult(): - source_packages[i[2]] = i[:2]; - to_remove.append(i[2:]); + source_packages[i[2]] = i[:2] + to_remove.append(i[2:]) if not Options["Source-Only"]: # Source + Binary - binary_packages = {}; + binary_packages = {} # First get a list of binary package names we suspect are linked to the source - q = projectB.query("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components)); + q = projectB.query("SELECT DISTINCT b.package FROM binaries b, source s, src_associations sa, suite su, files f, location l, component c WHERE b.source = s.id AND sa.source = s.id AND sa.suite = su.id AND s.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s" % (con_packages, con_suites, con_components)) for i in q.getresult(): - binary_packages[i[0]] = ""; + binary_packages[i[0]] = "" # Then parse each .dsc that we found earlier to see what binary packages it thinks it produces for i in source_packages.keys(): - filename = "/".join(source_packages[i]); + filename = "/".join(source_packages[i]) try: - dsc = utils.parse_changes(filename); + dsc = utils.parse_changes(filename) except utils.cant_open_exc: - utils.warn("couldn't open '%s'." % (filename)); - continue; + utils.warn("couldn't open '%s'." % (filename)) + continue for package in dsc.get("binary").split(','): - package = package.strip(); - binary_packages[package] = ""; + package = package.strip() + binary_packages[package] = "" # Then for each binary package: find any version in # unstable, check the Source: field in the deb matches our # source package and if so add it to the list of packages # to be removed. for package in binary_packages.keys(): - q = projectB.query("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package)); + q = projectB.query("SELECT l.path, f.filename, b.package, b.version, a.arch_string, b.id, b.maintainer FROM binaries b, bin_associations ba, architecture a, suite su, files f, location l, component c WHERE ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id AND b.file = f.id AND f.location = l.id AND l.component = c.id %s %s %s AND b.package = '%s'" % (con_suites, con_components, con_architectures, package)) for i in q.getresult(): - filename = "/".join(i[:2]); + filename = "/".join(i[:2]) control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(filename))) - source = control.Find("Source", control.Find("Package")); - source = re_strip_source_version.sub('', source); + source = control.Find("Source", control.Find("Package")) + source = re_strip_source_version.sub('', source) if source_packages.has_key(source): - to_remove.append(i[2:]); + to_remove.append(i[2:]) print "done." if not to_remove: print "Nothing to do." - sys.exit(0); + sys.exit(0) # If we don't have a reason; spawn an editor so the user can add one # Write the rejection email out as the .reason file if not Options["Reason"] and not Options["No-Action"]: - temp_filename = utils.temp_filename(); + temp_filename = utils.temp_filename() editor = os.environ.get("EDITOR","vi") result = os.system("%s %s" % (editor, temp_filename)) if result != 0: utils.fubar ("vi invocation failed for `%s'!" % (temp_filename), result) - temp_file = utils.open_file(temp_filename); + temp_file = utils.open_file(temp_filename) for line in temp_file.readlines(): - Options["Reason"] += line; - temp_file.close(); - os.unlink(temp_filename); + Options["Reason"] += line + temp_file.close() + os.unlink(temp_filename) # Generate the summary of what's to be removed - d = {}; + d = {} for i in to_remove: - package = i[0]; - version = i[1]; - architecture = i[2]; - maintainer = i[4]; - maintainers[maintainer] = ""; + package = i[0] + version = i[1] + architecture = i[2] + maintainer = i[4] + maintainers[maintainer] = "" if not d.has_key(package): - d[package] = {}; + d[package] = {} if not d[package].has_key(version): - d[package][version] = []; + d[package][version] = [] if architecture not in d[package][version]: - d[package][version].append(architecture); + d[package][version].append(architecture) - maintainer_list = []; + maintainer_list = [] for maintainer_id in maintainers.keys(): - maintainer_list.append(db_access.get_maintainer(maintainer_id)); - summary = ""; - removals = d.keys(); - removals.sort(); + maintainer_list.append(db_access.get_maintainer(maintainer_id)) + summary = "" + removals = d.keys() + removals.sort() for package in removals: - versions = d[package].keys(); - versions.sort(apt_pkg.VersionCompare); + versions = d[package].keys() + versions.sort(apt_pkg.VersionCompare) for version in versions: - d[package][version].sort(utils.arch_compare_sw); - summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])); - print "Will remove the following packages from %s:" % (suites_list); + d[package][version].sort(utils.arch_compare_sw) + summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])) + print "Will remove the following packages from %s:" % (suites_list) print print summary print "Maintainer: %s" % ", ".join(maintainer_list) if Options["Done"]: - print "Will also close bugs: "+Options["Done"]; + print "Will also close bugs: "+Options["Done"] if carbon_copy: print "Will also send CCs to: " + ", ".join(carbon_copy) print print "------------------- Reason -------------------" - print Options["Reason"]; + print Options["Reason"] print "----------------------------------------------" print if Options["Rdep-Check"]: - reverse_depends_check(removals, suites); + reverse_depends_check(removals, suites) # If -n/--no-action, drop out here if Options["No-Action"]: - sys.exit(0); + sys.exit(0) print "Going to remove the packages now." - game_over(); + game_over() - whoami = utils.whoami(); - date = commands.getoutput('date -R'); + whoami = utils.whoami() + date = commands.getoutput('date -R') # Log first; if it all falls apart I want a record that we at least tried. - logfile = utils.open_file(Cnf["Melanie::LogFile"], 'a'); - logfile.write("=========================================================================\n"); - logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)); - logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)); + logfile = utils.open_file(Cnf["Melanie::LogFile"], 'a') + logfile.write("=========================================================================\n") + logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) + logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) if Options["Done"]: - logfile.write("Closed bugs: %s\n" % (Options["Done"])); - logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"])); - logfile.write("----------------------------------------------\n"); - logfile.flush(); + logfile.write("Closed bugs: %s\n" % (Options["Done"])) + logfile.write("\n------------------- Reason -------------------\n%s\n" % (Options["Reason"])) + logfile.write("----------------------------------------------\n") + logfile.flush() - dsc_type_id = db_access.get_override_type_id('dsc'); - deb_type_id = db_access.get_override_type_id('deb'); + dsc_type_id = db_access.get_override_type_id('dsc') + deb_type_id = db_access.get_override_type_id('deb') # Do the actual deletion print "Deleting...", - sys.stdout.flush(); - projectB.query("BEGIN WORK"); + sys.stdout.flush() + projectB.query("BEGIN WORK") for i in to_remove: - package = i[0]; - architecture = i[2]; - package_id = i[3]; + package = i[0] + architecture = i[2] + package_id = i[3] for suite_id in suite_ids_list: if architecture == "source": - projectB.query("DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)); - #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id); + projectB.query("DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id)) + #print "DELETE FROM src_associations WHERE source = %s AND suite = %s" % (package_id, suite_id) else: - projectB.query("DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)); - #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id); + projectB.query("DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id)) + #print "DELETE FROM bin_associations WHERE bin = %s AND suite = %s" % (package_id, suite_id) # Delete from the override file if not Options["Partial"]: if architecture == "source": - type_id = dsc_type_id; + type_id = dsc_type_id else: - type_id = deb_type_id; - projectB.query("DELETE FROM override WHERE package = '%s' AND type = %s AND suite = %s %s" % (package, type_id, suite_id, over_con_components)); - projectB.query("COMMIT WORK"); + type_id = deb_type_id + projectB.query("DELETE FROM override WHERE package = '%s' AND type = %s AND suite = %s %s" % (package, type_id, suite_id, over_con_components)) + projectB.query("COMMIT WORK") print "done." # Send the bug closing messages if Options["Done"]: - Subst = {}; - Subst["__MELANIE_ADDRESS__"] = Cnf["Melanie::MyEmailAddress"]; - Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]; - bcc = []; + Subst = {} + Subst["__MELANIE_ADDRESS__"] = Cnf["Melanie::MyEmailAddress"] + Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"] + bcc = [] if Cnf.Find("Dinstall::Bcc") != "": - bcc.append(Cnf["Dinstall::Bcc"]); + bcc.append(Cnf["Dinstall::Bcc"]) if Cnf.Find("Melanie::Bcc") != "": - bcc.append(Cnf["Melanie::Bcc"]); + bcc.append(Cnf["Melanie::Bcc"]) if bcc: - Subst["__BCC__"] = "Bcc: " + ", ".join(bcc); + Subst["__BCC__"] = "Bcc: " + ", ".join(bcc) else: - Subst["__BCC__"] = "X-Filler: 42"; - Subst["__CC__"] = "X-Katie: melanie $Revision: 1.44 $"; + Subst["__BCC__"] = "X-Filler: 42" + Subst["__CC__"] = "X-Katie: melanie $Revision: 1.44 $" if carbon_copy: - Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy); - Subst["__SUITE_LIST__"] = suites_list; - Subst["__SUMMARY__"] = summary; - Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]; - Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]; - Subst["__WHOAMI__"] = whoami; - whereami = utils.where_am_i(); - Archive = Cnf.SubTree("Archive::%s" % (whereami)); - Subst["__MASTER_ARCHIVE__"] = Archive["OriginServer"]; - Subst["__PRIMARY_MIRROR__"] = Archive["PrimaryMirror"]; + Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy) + Subst["__SUITE_LIST__"] = suites_list + Subst["__SUMMARY__"] = summary + Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"] + Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"] + Subst["__WHOAMI__"] = whoami + whereami = utils.where_am_i() + Archive = Cnf.SubTree("Archive::%s" % (whereami)) + Subst["__MASTER_ARCHIVE__"] = Archive["OriginServer"] + Subst["__PRIMARY_MIRROR__"] = Archive["PrimaryMirror"] for bug in utils.split_args(Options["Done"]): - Subst["__BUG_NUMBER__"] = bug; - mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/melanie.bug-close"); - utils.send_mail(mail_message); + Subst["__BUG_NUMBER__"] = bug + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/melanie.bug-close") + utils.send_mail(mail_message) - logfile.write("=========================================================================\n"); - logfile.close(); + logfile.write("=========================================================================\n") + logfile.close() ####################################################################################### diff --git a/dak/security_install.py b/dak/security_install.py index 4f1832ad..eb79bb4a 100755 --- a/dak/security_install.py +++ b/dak/security_install.py @@ -31,17 +31,17 @@ ################################################################################ -import commands, os, pwd, re, sys, time; -import apt_pkg; -import katie, utils; +import commands, os, pwd, re, sys, time +import apt_pkg +import katie, utils ################################################################################ -Cnf = None; -Options = None; -Katie = None; +Cnf = None +Options = None +Katie = None -re_taint_free = re.compile(r"^['/;\-\+\.\s\w]+$"); +re_taint_free = re.compile(r"^['/;\-\+\.\s\w]+$") ################################################################################ @@ -58,157 +58,157 @@ Install CHANGES_FILE(s) as security advisory ADV_NUMBER ################################################################################ def do_upload(changes_files): - file_list = ""; - suites = {}; - component_mapping = {}; + file_list = "" + suites = {} + component_mapping = {} for component in Cnf.SubTree("Amber::ComponentMappings").List(): - component_mapping[component] = Cnf["Amber::ComponentMappings::%s" % (component)]; - uploads = {}; # uploads[uri] = file_list; - changesfiles = {}; # changesfiles[uri] = file_list; - package_list = {} # package_list[source_name][version]; - changes_files.sort(utils.changes_compare); + component_mapping[component] = Cnf["Amber::ComponentMappings::%s" % (component)] + uploads = {}; # uploads[uri] = file_list + changesfiles = {}; # changesfiles[uri] = file_list + package_list = {} # package_list[source_name][version] + changes_files.sort(utils.changes_compare) for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file); + changes_file = utils.validate_changes_file_arg(changes_file) # Reset variables - components = {}; - upload_uris = {}; - file_list = []; - Katie.init_vars(); + components = {} + upload_uris = {} + file_list = [] + Katie.init_vars() # Parse the .katie file for the .changes file - Katie.pkg.changes_file = changes_file; - Katie.update_vars(); - files = Katie.pkg.files; - changes = Katie.pkg.changes; - dsc = Katie.pkg.dsc; + Katie.pkg.changes_file = changes_file + Katie.update_vars() + files = Katie.pkg.files + changes = Katie.pkg.changes + dsc = Katie.pkg.dsc # We have the changes, now return if its amd64, to not upload them to ftp-master if changes["architecture"].has_key("amd64"): - print "Not uploading amd64 part to ftp-master\n"; + print "Not uploading amd64 part to ftp-master\n" continue if changes["distribution"].has_key("oldstable-security"): - print "Not uploading oldstable-security changes to ftp-master\n"; + print "Not uploading oldstable-security changes to ftp-master\n" continue # Build the file list for this .changes file for file in files.keys(): poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"], utils.poolify(changes["source"], files[file]["component"]), - file); - file_list.append(poolname); - orig_component = files[file].get("original component", files[file]["component"]); - components[orig_component] = ""; + file) + file_list.append(poolname) + orig_component = files[file].get("original component", files[file]["component"]) + components[orig_component] = "" # Determine the upload uri for this .changes file for component in components.keys(): - upload_uri = component_mapping.get(component); + upload_uri = component_mapping.get(component) if upload_uri: - upload_uris[upload_uri] = ""; - num_upload_uris = len(upload_uris.keys()); + upload_uris[upload_uri] = "" + num_upload_uris = len(upload_uris.keys()) if num_upload_uris == 0: utils.fubar("%s: No valid upload URI found from components (%s)." - % (changes_file, ", ".join(components.keys()))); + % (changes_file, ", ".join(components.keys()))) elif num_upload_uris > 1: utils.fubar("%s: more than one upload URI (%s) from components (%s)." % (changes_file, ", ".join(upload_uris.keys()), - ", ".join(components.keys()))); - upload_uri = upload_uris.keys()[0]; + ", ".join(components.keys()))) + upload_uri = upload_uris.keys()[0] # Update the file list for the upload uri if not uploads.has_key(upload_uri): - uploads[upload_uri] = []; - uploads[upload_uri].extend(file_list); + uploads[upload_uri] = [] + uploads[upload_uri].extend(file_list) # Update the changes list for the upload uri if not changes.has_key(upload_uri): - changesfiles[upload_uri] = []; - changesfiles[upload_uri].append(changes_file); + changesfiles[upload_uri] = [] + changesfiles[upload_uri].append(changes_file) # Remember the suites and source name/version for suite in changes["distribution"].keys(): - suites[suite] = ""; + suites[suite] = "" # Remember the source name and version if changes["architecture"].has_key("source") and \ changes["distribution"].has_key("testing"): if not package_list.has_key(dsc["source"]): - package_list[dsc["source"]] = {}; - package_list[dsc["source"]][dsc["version"]] = ""; + package_list[dsc["source"]] = {} + package_list[dsc["source"]][dsc["version"]] = "" if not Options["No-Action"]: - answer = yes_no("Upload to files to main archive (Y/n)?"); + answer = yes_no("Upload to files to main archive (Y/n)?") if answer != "y": - return; + return for uri in uploads.keys(): - uploads[uri].extend(changesfiles[uri]); - (host, path) = uri.split(":"); - file_list = " ".join(uploads[uri]); - print "Uploading files to %s..." % (host); - spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list)); + uploads[uri].extend(changesfiles[uri]) + (host, path) = uri.split(":") + file_list = " ".join(uploads[uri]) + print "Uploading files to %s..." % (host) + spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list)) if not Options["No-Action"]: - filename = "%s/testing-processed" % (Cnf["Dir::Log"]); - file = utils.open_file(filename, 'a'); + filename = "%s/testing-processed" % (Cnf["Dir::Log"]) + file = utils.open_file(filename, 'a') for source in package_list.keys(): for version in package_list[source].keys(): - file.write(" ".join([source, version])+'\n'); - file.close(); + file.write(" ".join([source, version])+'\n') + file.close() ###################################################################### # This function was originally written by aj and NIHishly merged into # amber by me. def make_advisory(advisory_nr, changes_files): - adv_packages = []; + adv_packages = [] updated_pkgs = {}; # updated_pkgs[distro][arch][file] = {path,md5,size} for arg in changes_files: - arg = utils.validate_changes_file_arg(arg); - Katie.pkg.changes_file = arg; - Katie.init_vars(); - Katie.update_vars(); + arg = utils.validate_changes_file_arg(arg) + Katie.pkg.changes_file = arg + Katie.init_vars() + Katie.update_vars() - src = Katie.pkg.changes["source"]; + src = Katie.pkg.changes["source"] if src not in adv_packages: - adv_packages += [src]; + adv_packages += [src] - suites = Katie.pkg.changes["distribution"].keys(); + suites = Katie.pkg.changes["distribution"].keys() for suite in suites: if not updated_pkgs.has_key(suite): - updated_pkgs[suite] = {}; + updated_pkgs[suite] = {} - files = Katie.pkg.files; + files = Katie.pkg.files for file in files.keys(): - arch = files[file]["architecture"]; - md5 = files[file]["md5sum"]; - size = files[file]["size"]; + arch = files[file]["architecture"] + md5 = files[file]["md5sum"] + size = files[file]["size"] poolname = Cnf["Dir::PoolRoot"] + \ - utils.poolify(src, files[file]["component"]); + utils.poolify(src, files[file]["component"]) if arch == "source" and file.endswith(".dsc"): - dscpoolname = poolname; + dscpoolname = poolname for suite in suites: if not updated_pkgs[suite].has_key(arch): updated_pkgs[suite][arch] = {} updated_pkgs[suite][arch][file] = { "md5": md5, "size": size, - "poolname": poolname }; + "poolname": poolname } - dsc_files = Katie.pkg.dsc_files; + dsc_files = Katie.pkg.dsc_files for file in dsc_files.keys(): arch = "source" if not dsc_files[file].has_key("files id"): - continue; + continue # otherwise, it's already in the pool and needs to be # listed specially - md5 = dsc_files[file]["md5sum"]; - size = dsc_files[file]["size"]; + md5 = dsc_files[file]["md5sum"] + size = dsc_files[file]["size"] for suite in suites: if not updated_pkgs[suite].has_key(arch): - updated_pkgs[suite][arch] = {}; + updated_pkgs[suite][arch] = {} updated_pkgs[suite][arch][file] = { "md5": md5, "size": size, - "poolname": dscpoolname }; + "poolname": dscpoolname } if os.environ.has_key("SUDO_UID"): - whoami = long(os.environ["SUDO_UID"]); + whoami = long(os.environ["SUDO_UID"]) else: - whoami = os.getuid(); - whoamifull = pwd.getpwuid(whoami); - username = whoamifull[4].split(",")[0]; + whoami = os.getuid() + whoamifull = pwd.getpwuid(whoami) + username = whoamifull[4].split(",")[0] Subst = { "__ADVISORY__": advisory_nr, @@ -216,154 +216,154 @@ def make_advisory(advisory_nr, changes_files): "__DATE__": time.strftime("%B %d, %Y", time.gmtime(time.time())), "__PACKAGE__": ", ".join(adv_packages), "__KATIE_ADDRESS__": Cnf["Dinstall::MyEmailAddress"] - }; + } if Cnf.has_key("Dinstall::Bcc"): - Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"]); + Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"]) - adv = ""; - archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())]; + adv = "" + archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())] for suite in updated_pkgs.keys(): suite_header = "%s %s (%s)" % (Cnf["Dinstall::MyDistribution"], - Cnf["Suite::%s::Version" % suite], suite); - adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header)); + Cnf["Suite::%s::Version" % suite], suite) + adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header)) - arches = Cnf.ValueList("Suite::%s::Architectures" % suite); + arches = Cnf.ValueList("Suite::%s::Architectures" % suite) if "source" in arches: - arches.remove("source"); + arches.remove("source") if "all" in arches: - arches.remove("all"); - arches.sort(); + arches.remove("all") + arches.sort() adv += " %s was released for %s.\n\n" % ( - suite.capitalize(), utils.join_with_commas_and(arches)); + suite.capitalize(), utils.join_with_commas_and(arches)) for a in ["source", "all"] + arches: if not updated_pkgs[suite].has_key(a): - continue; + continue if a == "source": - adv += " Source archives:\n\n"; + adv += " Source archives:\n\n" elif a == "all": - adv += " Architecture independent packages:\n\n"; + adv += " Architecture independent packages:\n\n" else: adv += " %s architecture (%s)\n\n" % (a, - Cnf["Architectures::%s" % a]); + Cnf["Architectures::%s" % a]) for file in updated_pkgs[suite][a].keys(): adv += " http://%s/%s%s\n" % ( - archive, updated_pkgs[suite][a][file]["poolname"], file); + archive, updated_pkgs[suite][a][file]["poolname"], file) adv += " Size/MD5 checksum: %8s %s\n" % ( updated_pkgs[suite][a][file]["size"], - updated_pkgs[suite][a][file]["md5"]); - adv += "\n"; - adv = adv.rstrip(); + updated_pkgs[suite][a][file]["md5"]) + adv += "\n" + adv = adv.rstrip() - Subst["__ADVISORY_TEXT__"] = adv; + Subst["__ADVISORY_TEXT__"] = adv - adv = utils.TemplateSubst(Subst, Cnf["Dir::Templates"]+"/amber.advisory"); + adv = utils.TemplateSubst(Subst, Cnf["Dir::Templates"]+"/amber.advisory") if not Options["No-Action"]: - utils.send_mail (adv); + utils.send_mail (adv) else: - print "[]"; + print "[]" ###################################################################### def init(): - global Cnf, Katie, Options; + global Cnf, Katie, Options - apt_pkg.init(); - Cnf = utils.get_conf(); + apt_pkg.init() + Cnf = utils.get_conf() Arguments = [('h', "help", "Amber::Options::Help"), - ('n', "no-action", "Amber::Options::No-Action")]; + ('n', "no-action", "Amber::Options::No-Action")] for i in [ "help", "no-action" ]: - Cnf["Amber::Options::%s" % (i)] = ""; + Cnf["Amber::Options::%s" % (i)] = "" - arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Amber::Options") - Katie = katie.Katie(Cnf); + Katie = katie.Katie(Cnf) if Options["Help"]: - usage(0); + usage(0) if not arguments: - usage(1); + usage(1) - advisory_number = arguments[0]; - changes_files = arguments[1:]; + advisory_number = arguments[0] + changes_files = arguments[1:] if advisory_number.endswith(".changes"): - utils.warn("first argument must be the advisory number."); - usage(1); + utils.warn("first argument must be the advisory number.") + usage(1) for file in changes_files: - file = utils.validate_changes_file_arg(file); - return (advisory_number, changes_files); + file = utils.validate_changes_file_arg(file) + return (advisory_number, changes_files) ###################################################################### def yes_no(prompt): while 1: - answer = utils.our_raw_input(prompt+" ").lower(); + answer = utils.our_raw_input(prompt+" ").lower() if answer == "y" or answer == "n": - break; + break else: - print "Invalid answer; please try again."; - return answer; + print "Invalid answer; please try again." + return answer ###################################################################### def spawn(command): if not re_taint_free.match(command): - utils.fubar("Invalid character in \"%s\"." % (command)); + utils.fubar("Invalid character in \"%s\"." % (command)) if Options["No-Action"]: - print "[%s]" % (command); + print "[%s]" % (command) else: - (result, output) = commands.getstatusoutput(command); + (result, output) = commands.getstatusoutput(command) if (result != 0): - utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result); + utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result) ###################################################################### def main(): - (advisory_number, changes_files) = init(); + (advisory_number, changes_files) = init() if not Options["No-Action"]: print "About to install the following files: " for file in changes_files: - print " %s" % (file); - answer = yes_no("Continue (Y/n)?"); + print " %s" % (file) + answer = yes_no("Continue (Y/n)?") if answer == "n": - sys.exit(0); - - os.chdir(Cnf["Dir::Queue::Accepted"]); - print "Installing packages into the archive..."; - spawn("%s/kelly -pa %s" % (Cnf["Dir::Katie"], " ".join(changes_files))); - os.chdir(Cnf["Dir::Katie"]); - print "Updating file lists for apt-ftparchive..."; - spawn("./jenna"); - print "Updating Packages and Sources files..."; - spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file())); - print "Updating Release files..."; - spawn("./ziyi"); + sys.exit(0) + + os.chdir(Cnf["Dir::Queue::Accepted"]) + print "Installing packages into the archive..." + spawn("%s/kelly -pa %s" % (Cnf["Dir::Katie"], " ".join(changes_files))) + os.chdir(Cnf["Dir::Katie"]) + print "Updating file lists for apt-ftparchive..." + spawn("./jenna") + print "Updating Packages and Sources files..." + spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file())) + print "Updating Release files..." + spawn("./ziyi") if not Options["No-Action"]: - os.chdir(Cnf["Dir::Queue::Done"]); + os.chdir(Cnf["Dir::Queue::Done"]) else: - os.chdir(Cnf["Dir::Queue::Accepted"]); - print "Generating template advisory..."; - make_advisory(advisory_number, changes_files); + os.chdir(Cnf["Dir::Queue::Accepted"]) + print "Generating template advisory..." + make_advisory(advisory_number, changes_files) # Trigger security mirrors - spawn("sudo -u archvsync /home/archvsync/signal_security"); + spawn("sudo -u archvsync /home/archvsync/signal_security") - do_upload(changes_files); + do_upload(changes_files) ################################################################################ if __name__ == '__main__': - main(); + main() ################################################################################ diff --git a/dak/shell.py b/dak/shell.py index 8a329689..7e692ee4 100755 --- a/dak/shell.py +++ b/dak/shell.py @@ -121,12 +121,12 @@ for f in functionality: def main(): if len(sys.argv) == 0: print "err, argc == 0? how is that possible?" - sys.exit(1); + sys.exit(1) elif len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] == "--help"): print "Sub commands:" for f in functionality: print " %-23s %s" % (f[0], f[1]) - sys.exit(0); + sys.exit(0) else: # should set PATH based on sys.argv[0] maybe # possibly should set names based on sys.argv[0] too @@ -148,10 +148,10 @@ def main(): cmdname = match[0] elif len(match) > 1: print "ambiguous command: %s" % ", ".join(match) - sys.exit(1); + sys.exit(1) else: print "unknown command \"%s\"" % (cmdname) - sys.exit(1); + sys.exit(1) func = names[cmdname] x = __import__(func[0]) diff --git a/dak/split_done.py b/dak/split_done.py index a628b2c4..c595f68e 100755 --- a/dak/split_done.py +++ b/dak/split_done.py @@ -19,30 +19,30 @@ ################################################################################ -import glob, os, stat, time; -import utils; +import glob, os, stat, time +import utils ################################################################################ def main(): Cnf = utils.get_conf() - count = 0; + count = 0 os.chdir(Cnf["Dir::Queue::Done"]) - files = glob.glob("%s/*" % (Cnf["Dir::Queue::Done"])); + files = glob.glob("%s/*" % (Cnf["Dir::Queue::Done"])) for filename in files: if os.path.isfile(filename): - mtime = time.gmtime(os.stat(filename)[stat.ST_MTIME]); - dirname = time.strftime("%Y/%m/%d", mtime); + mtime = time.gmtime(os.stat(filename)[stat.ST_MTIME]) + dirname = time.strftime("%Y/%m/%d", mtime) if not os.path.exists(dirname): - print "Creating: %s" % (dirname); - os.makedirs(dirname); - dest = dirname + '/' + os.path.basename(filename); + print "Creating: %s" % (dirname) + os.makedirs(dirname) + dest = dirname + '/' + os.path.basename(filename) if os.path.exists(dest): - utils.fubar("%s already exists." % (dest)); - print "Move: %s -> %s" % (filename, dest) ; - os.rename(filename, dest); - count = count + 1; - print "Moved %d files." % (count); + utils.fubar("%s already exists." % (dest)) + print "Move: %s -> %s" % (filename, dest) + os.rename(filename, dest) + count = count + 1 + print "Moved %d files." % (count) ############################################################ diff --git a/dak/stats.py b/dak/stats.py index a11b902e..bf22a774 100755 --- a/dak/stats.py +++ b/dak/stats.py @@ -31,14 +31,14 @@ ################################################################################ -import pg, sys; -import utils; -import apt_pkg; +import pg, sys +import utils +import apt_pkg ################################################################################ -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ @@ -63,67 +63,67 @@ def per_arch_space_use(): SELECT a.arch_string as Architecture, sum(f.size) FROM files f, binaries b, architecture a WHERE a.id=b.architecture AND f.id=b.file - GROUP BY a.arch_string"""); - print q; - q = projectB.query("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'"); - print q; + GROUP BY a.arch_string""") + print q + q = projectB.query("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'") + print q ################################################################################ def daily_install_stats(): - stats = {}; - file = utils.open_file("2001-11"); + stats = {} + file = utils.open_file("2001-11") for line in file.readlines(): - split = line.strip().split('~'); - program = split[1]; + split = line.strip().split('~') + program = split[1] if program != "katie": - continue; - action = split[2]; + continue + action = split[2] if action != "installing changes" and action != "installed": - continue; - date = split[0][:8]; + continue + date = split[0][:8] if not stats.has_key(date): - stats[date] = {}; - stats[date]["packages"] = 0; - stats[date]["size"] = 0.0; + stats[date] = {} + stats[date]["packages"] = 0 + stats[date]["size"] = 0.0 if action == "installing changes": - stats[date]["packages"] += 1; + stats[date]["packages"] += 1 elif action == "installed": - stats[date]["size"] += float(split[5]); + stats[date]["size"] += float(split[5]) - dates = stats.keys(); - dates.sort(); + dates = stats.keys() + dates.sort() for date in dates: packages = stats[date]["packages"] size = int(stats[date]["size"] / 1024.0 / 1024.0) - print "%s %s %s" % (date, packages, size); + print "%s %s %s" % (date, packages, size) ################################################################################ def longest(list): - longest = 0; + longest = 0 for i in list: - l = len(i); + l = len(i) if l > longest: - longest = l; - return longest; + longest = l + return longest def suite_sort(a, b): if Cnf.has_key("Suite::%s::Priority" % (a)): - a_priority = int(Cnf["Suite::%s::Priority" % (a)]); + a_priority = int(Cnf["Suite::%s::Priority" % (a)]) else: - a_priority = 0; + a_priority = 0 if Cnf.has_key("Suite::%s::Priority" % (b)): - b_priority = int(Cnf["Suite::%s::Priority" % (b)]); + b_priority = int(Cnf["Suite::%s::Priority" % (b)]) else: - b_priority = 0; - return cmp(a_priority, b_priority); + b_priority = 0 + return cmp(a_priority, b_priority) def output_format(suite): - output_suite = []; + output_suite = [] for word in suite.split("-"): - output_suite.append(word[0]); - return "-".join(output_suite); + output_suite.append(word[0]) + return "-".join(output_suite) # Obvious query with GROUP BY and mapped names -> 50 seconds # GROUP BY but ids instead of suite/architecture names -> 28 seconds @@ -132,119 +132,119 @@ def output_format(suite): # Simple query into large pre-created dictionary + processing -> 18 seconds def number_of_packages(): - arches = {}; - arch_ids = {}; - suites = {}; - suite_ids = {}; - d = {}; + arches = {} + arch_ids = {} + suites = {} + suite_ids = {} + d = {} # Build up suite mapping - q = projectB.query("SELECT id, suite_name FROM suite"); - suite_ql = q.getresult(); + q = projectB.query("SELECT id, suite_name FROM suite") + suite_ql = q.getresult() for i in suite_ql: - (id, name) = i; - suites[id] = name; - suite_ids[name] = id; + (id, name) = i + suites[id] = name + suite_ids[name] = id # Build up architecture mapping - q = projectB.query("SELECT id, arch_string FROM architecture"); + q = projectB.query("SELECT id, arch_string FROM architecture") for i in q.getresult(): - (id, name) = i; - arches[id] = name; - arch_ids[name] = id; + (id, name) = i + arches[id] = name + arch_ids[name] = id # Pre-create the dictionary for suite_id in suites.keys(): - d[suite_id] = {}; + d[suite_id] = {} for arch_id in arches.keys(): - d[suite_id][arch_id] = 0; + d[suite_id][arch_id] = 0 # Get the raw data for binaries q = projectB.query(""" SELECT ba.suite, b.architecture FROM binaries b, bin_associations ba - WHERE b.id = ba.bin"""); + WHERE b.id = ba.bin""") # Simultate 'GROUP by suite, architecture' with a dictionary for i in q.getresult(): - (suite_id, arch_id) = i; - d[suite_id][arch_id] = d[suite_id][arch_id] + 1; + (suite_id, arch_id) = i + d[suite_id][arch_id] = d[suite_id][arch_id] + 1 # Get the raw data for source - arch_id = arch_ids["source"]; + arch_id = arch_ids["source"] q = projectB.query(""" -SELECT suite, count(suite) FROM src_associations GROUP BY suite;"""); +SELECT suite, count(suite) FROM src_associations GROUP BY suite;""") for i in q.getresult(): - (suite_id, count) = i; - d[suite_id][arch_id] = d[suite_id][arch_id] + count; + (suite_id, count) = i + d[suite_id][arch_id] = d[suite_id][arch_id] + count ## Print the results # Setup - suite_list = suites.values(); - suite_list.sort(suite_sort); - suite_id_list = []; - suite_arches = {}; + suite_list = suites.values() + suite_list.sort(suite_sort) + suite_id_list = [] + suite_arches = {} for suite in suite_list: - suite_id = suite_ids[suite]; - suite_arches[suite_id] = {}; + suite_id = suite_ids[suite] + suite_arches[suite_id] = {} for arch in Cnf.ValueList("Suite::%s::Architectures" % (suite)): - suite_arches[suite_id][arch] = ""; - suite_id_list.append(suite_id); - output_list = map(lambda x: output_format(x), suite_list); - longest_suite = longest(output_list); - arch_list = arches.values(); - arch_list.sort(); - longest_arch = longest(arch_list); + suite_arches[suite_id][arch] = "" + suite_id_list.append(suite_id) + output_list = map(lambda x: output_format(x), suite_list) + longest_suite = longest(output_list) + arch_list = arches.values() + arch_list.sort() + longest_arch = longest(arch_list) # Header output = (" "*longest_arch) + " |" for suite in output_list: - output = output + suite.center(longest_suite)+" |"; - output = output + "\n"+(len(output)*"-")+"\n"; + output = output + suite.center(longest_suite)+" |" + output = output + "\n"+(len(output)*"-")+"\n" # per-arch data - arch_list = arches.values(); - arch_list.sort(); - longest_arch = longest(arch_list); + arch_list = arches.values() + arch_list.sort() + longest_arch = longest(arch_list) for arch in arch_list: - arch_id = arch_ids[arch]; - output = output + arch.center(longest_arch)+" |"; + arch_id = arch_ids[arch] + output = output + arch.center(longest_arch)+" |" for suite_id in suite_id_list: if suite_arches[suite_id].has_key(arch): - count = repr(d[suite_id][arch_id]); + count = repr(d[suite_id][arch_id]) else: - count = "-"; - output = output + count.rjust(longest_suite)+" |"; - output = output + "\n"; - print output; + count = "-" + output = output + count.rjust(longest_suite)+" |" + output = output + "\n" + print output ################################################################################ def main (): - global Cnf, projectB; + global Cnf, projectB - Cnf = utils.get_conf(); - Arguments = [('h',"help","Saffron::Options::Help")]; + Cnf = utils.get_conf() + Arguments = [('h',"help","Saffron::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Saffron::Options::%s" % (i)): - Cnf["Saffron::Options::%s" % (i)] = ""; + Cnf["Saffron::Options::%s" % (i)] = "" - args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Saffron::Options") if Options["Help"]: - usage(); + usage() if len(args) < 1: - utils.warn("saffron requires at least one argument"); - usage(1); + utils.warn("saffron requires at least one argument") + usage(1) elif len(args) > 1: - utils.warn("saffron accepts only one argument"); - usage(1); - mode = args[0].lower(); + utils.warn("saffron accepts only one argument") + usage(1) + mode = args[0].lower() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) if mode == "arch-space": - per_arch_space_use(); + per_arch_space_use() elif mode == "pkg-nums": - number_of_packages(); + number_of_packages() elif mode == "daily-install": - daily_install_stats(); + daily_install_stats() else: - utils.warn("unknown mode '%s'" % (mode)); - usage(1); + utils.warn("unknown mode '%s'" % (mode)) + usage(1) ################################################################################ diff --git a/dak/symlink_dists.py b/dak/symlink_dists.py index c0055a70..cb6e2f34 100755 --- a/dak/symlink_dists.py +++ b/dak/symlink_dists.py @@ -29,16 +29,16 @@ ################################################################################ -import os, pg, re, sys; -import utils, db_access; -import apt_pkg; +import os, pg, re, sys +import utils, db_access +import apt_pkg ################################################################################ -re_strip_section_prefix = re.compile(r'.*/'); +re_strip_section_prefix = re.compile(r'.*/') -Cnf = None; -projectB = None; +Cnf = None +projectB = None ################################################################################ @@ -55,24 +55,24 @@ Create compatibility symlinks from legacy locations to the pool. def fix_component_section (component, section): if component == "": - component = utils.extract_component_from_section(section)[1]; + component = utils.extract_component_from_section(section)[1] # FIXME: ugly hacks to work around override brain damage - section = re_strip_section_prefix.sub('', section); - section = section.lower().replace('non-us', ''); + section = re_strip_section_prefix.sub('', section) + section = section.lower().replace('non-us', '') if section == "main" or section == "contrib" or section == "non-free": - section = ''; + section = '' if section != '': - section += '/'; + section += '/' - return (component, section); + return (component, section) ################################################################################ def find_dislocated_stable(Cnf, projectB): dislocated_files = {} - codename = Cnf["Suite::Stable::Codename"]; + codename = Cnf["Suite::Stable::Codename"] # Source q = projectB.query(""" @@ -84,7 +84,7 @@ SELECT DISTINCT ON (f.id) c.name, sec.section, l.path, f.filename, f.id AND f.id = df.file AND f.location = l.id AND o.package = s.source AND sec.id = o.section AND NOT (f.filename ~ '^%s/') AND l.component = c.id AND o.suite = su.id -""" % (codename)); +""" % (codename)) # Only needed if you have files in legacy-mixed locations # UNION SELECT DISTINCT ON (f.id) null, sec.section, l.path, f.filename, f.id # FROM component c, override o, section sec, source s, files f, location l, @@ -93,22 +93,22 @@ SELECT DISTINCT ON (f.id) c.name, sec.section, l.path, f.filename, f.id # AND f2.id = s.file AND f2.location = l2.id AND df.source = s.id # AND f.id = df.file AND f.location = l.id AND o.package = s.source # AND sec.id = o.section AND NOT (f.filename ~ '^%s/') AND o.suite = su.id -# AND NOT EXISTS (SELECT 1 FROM location l WHERE l.component IS NOT NULL AND f.location = l.id); +# AND NOT EXISTS (SELECT 1 FROM location l WHERE l.component IS NOT NULL AND f.location = l.id) for i in q.getresult(): - (component, section) = fix_component_section(i[0], i[1]); + (component, section) = fix_component_section(i[0], i[1]) if Cnf.FindB("Dinstall::LegacyStableHasNoSections"): - section=""; - dest = "%sdists/%s/%s/source/%s%s" % (Cnf["Dir::Root"], codename, component, section, os.path.basename(i[3])); + section="" + dest = "%sdists/%s/%s/source/%s%s" % (Cnf["Dir::Root"], codename, component, section, os.path.basename(i[3])) if not os.path.exists(dest): - src = i[2]+i[3]; - src = utils.clean_symlink(src, dest, Cnf["Dir::Root"]); + src = i[2]+i[3] + src = utils.clean_symlink(src, dest, Cnf["Dir::Root"]) if Cnf.Find("Claire::Options::Verbose"): print src+' -> '+dest - os.symlink(src, dest); - dislocated_files[i[4]] = dest; + os.symlink(src, dest) + dislocated_files[i[4]] = dest # Binary - architectures = filter(utils.real_arch, Cnf.ValueList("Suite::Stable::Architectures")); + architectures = filter(utils.real_arch, Cnf.ValueList("Suite::Stable::Architectures")) q = projectB.query(""" SELECT DISTINCT ON (f.id) c.name, a.arch_string, sec.section, b.package, b.version, l.path, f.filename, f.id @@ -118,7 +118,7 @@ SELECT DISTINCT ON (f.id) c.name, a.arch_string, sec.section, b.package, AND f.id = b.file AND f.location = l.id AND o.package = b.package AND sec.id = o.section AND NOT (f.filename ~ '^%s/') AND b.architecture = a.id AND l.component = c.id AND o.suite = su.id""" % - (codename)); + (codename)) # Only needed if you have files in legacy-mixed locations # UNION SELECT DISTINCT ON (f.id) null, a.arch_string, sec.section, b.package, # b.version, l.path, f.filename, f.id @@ -128,61 +128,61 @@ SELECT DISTINCT ON (f.id) c.name, a.arch_string, sec.section, b.package, # AND f.id = b.file AND f.location = l.id AND o.package = b.package # AND sec.id = o.section AND NOT (f.filename ~ '^%s/') # AND b.architecture = a.id AND o.suite = su.id AND NOT EXISTS -# (SELECT 1 FROM location l WHERE l.component IS NOT NULL AND f.location = l.id); +# (SELECT 1 FROM location l WHERE l.component IS NOT NULL AND f.location = l.id) for i in q.getresult(): - (component, section) = fix_component_section(i[0], i[2]); + (component, section) = fix_component_section(i[0], i[2]) if Cnf.FindB("Dinstall::LegacyStableHasNoSections"): - section=""; - architecture = i[1]; - package = i[3]; - version = utils.re_no_epoch.sub('', i[4]); - src = i[5]+i[6]; - - dest = "%sdists/%s/%s/binary-%s/%s%s_%s.deb" % (Cnf["Dir::Root"], codename, component, architecture, section, package, version); - src = utils.clean_symlink(src, dest, Cnf["Dir::Root"]); + section="" + architecture = i[1] + package = i[3] + version = utils.re_no_epoch.sub('', i[4]) + src = i[5]+i[6] + + dest = "%sdists/%s/%s/binary-%s/%s%s_%s.deb" % (Cnf["Dir::Root"], codename, component, architecture, section, package, version) + src = utils.clean_symlink(src, dest, Cnf["Dir::Root"]) if not os.path.exists(dest): if Cnf.Find("Claire::Options::Verbose"): - print src+' -> '+dest; - os.symlink(src, dest); - dislocated_files[i[7]] = dest; + print src+' -> '+dest + os.symlink(src, dest) + dislocated_files[i[7]] = dest # Add per-arch symlinks for arch: all debs if architecture == "all": for arch in architectures: - dest = "%sdists/%s/%s/binary-%s/%s%s_%s.deb" % (Cnf["Dir::Root"], codename, component, arch, section, package, version); + dest = "%sdists/%s/%s/binary-%s/%s%s_%s.deb" % (Cnf["Dir::Root"], codename, component, arch, section, package, version) if not os.path.exists(dest): if Cnf.Find("Claire::Options::Verbose"): print src+' -> '+dest - os.symlink(src, dest); + os.symlink(src, dest) return dislocated_files ################################################################################ def main (): - global Cnf, projectB; + global Cnf, projectB Cnf = utils.get_conf() Arguments = [('h',"help","Claire::Options::Help"), - ('v',"verbose","Claire::Options::Verbose")]; + ('v',"verbose","Claire::Options::Verbose")] for i in ["help", "verbose" ]: if not Cnf.has_key("Claire::Options::%s" % (i)): - Cnf["Claire::Options::%s" % (i)] = ""; + Cnf["Claire::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Claire::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - db_access.init(Cnf, projectB); + db_access.init(Cnf, projectB) - find_dislocated_stable(Cnf, projectB); + find_dislocated_stable(Cnf, projectB) ####################################################################################### if __name__ == '__main__': - main(); + main() diff --git a/dak/test/001/test.py b/dak/test/001/test.py index ffd7055b..7931ac18 100755 --- a/dak/test/001/test.py +++ b/dak/test/001/test.py @@ -22,72 +22,72 @@ import os, sys -sys.path.append(os.path.abspath('../../')); +sys.path.append(os.path.abspath('../../')) import utils ################################################################################ def fail(message): - sys.stderr.write("%s\n" % (message)); - sys.exit(1); + sys.stderr.write("%s\n" % (message)) + sys.exit(1) ################################################################################ def main (): # Valid .dsc - utils.parse_changes('1.dsc',1); + utils.parse_changes('1.dsc',1) # Missing blank line before signature body try: - utils.parse_changes('2.dsc',1); + utils.parse_changes('2.dsc',1) except utils.invalid_dsc_format_exc, line: if line != 14: - fail("Incorrect line number ('%s') for test #2." % (line)); + fail("Incorrect line number ('%s') for test #2." % (line)) else: - fail("Test #2 wasn't recognised as invalid."); + fail("Test #2 wasn't recognised as invalid.") # Missing blank line after signature header try: - utils.parse_changes('3.dsc',1); + utils.parse_changes('3.dsc',1) except utils.invalid_dsc_format_exc, line: if line != 14: - fail("Incorrect line number ('%s') for test #3." % (line)); + fail("Incorrect line number ('%s') for test #3." % (line)) else: - fail("Test #3 wasn't recognised as invalid."); + fail("Test #3 wasn't recognised as invalid.") # No blank lines at all try: - utils.parse_changes('4.dsc',1); + utils.parse_changes('4.dsc',1) except utils.invalid_dsc_format_exc, line: if line != 19: - fail("Incorrect line number ('%s') for test #4." % (line)); + fail("Incorrect line number ('%s') for test #4." % (line)) else: - fail("Test #4 wasn't recognised as invalid."); + fail("Test #4 wasn't recognised as invalid.") # Extra blank line before signature body try: - utils.parse_changes('5.dsc',1); + utils.parse_changes('5.dsc',1) except utils.invalid_dsc_format_exc, line: if line != 15: - fail("Incorrect line number ('%s') for test #5." % (line)); + fail("Incorrect line number ('%s') for test #5." % (line)) else: - fail("Test #5 wasn't recognised as invalid."); + fail("Test #5 wasn't recognised as invalid.") # Extra blank line after signature header try: - utils.parse_changes('6.dsc',1); + utils.parse_changes('6.dsc',1) except utils.invalid_dsc_format_exc, line: if line != 5: - fail("Incorrect line number ('%s') for test #6." % (line)); + fail("Incorrect line number ('%s') for test #6." % (line)) else: - fail("Test #6 wasn't recognised as invalid."); + fail("Test #6 wasn't recognised as invalid.") # Valid .dsc ; ignoring errors - utils.parse_changes('1.dsc', 0); + utils.parse_changes('1.dsc', 0) # Invalid .dsc ; ignoring errors - utils.parse_changes('2.dsc', 0); + utils.parse_changes('2.dsc', 0) ################################################################################ diff --git a/dak/test/002/test.py b/dak/test/002/test.py index 9467f281..367a1de0 100755 --- a/dak/test/002/test.py +++ b/dak/test/002/test.py @@ -22,15 +22,15 @@ import os, sys -sys.path.append(os.path.abspath('../../')); +sys.path.append(os.path.abspath('../../')) import utils ################################################################################ def fail(message): - sys.stderr.write("%s\n" % (message)); - sys.exit(1); + sys.stderr.write("%s\n" % (message)) + sys.exit(1) ################################################################################ @@ -40,9 +40,9 @@ def main (): utils.parse_changes('empty.changes', 0) except utils.changes_parse_error_exc, line: if line != "[Empty changes file]": - fail("Returned exception with unexcpected error message `%s'." % (line)); + fail("Returned exception with unexcpected error message `%s'." % (line)) else: - fail("Didn't raise a 'parse error' exception for a zero-length .changes file."); + fail("Didn't raise a 'parse error' exception for a zero-length .changes file.") ################################################################################ diff --git a/dak/test/003/test.py b/dak/test/003/test.py index f8277f87..d5c15ee0 100755 --- a/dak/test/003/test.py +++ b/dak/test/003/test.py @@ -34,15 +34,15 @@ import os, sys -sys.path.append(os.path.abspath('../../')); +sys.path.append(os.path.abspath('../../')) import utils ################################################################################ def fail(message): - sys.stderr.write("%s\n" % (message)); - sys.exit(1); + sys.stderr.write("%s\n" % (message)) + sys.exit(1) ################################################################################ @@ -51,7 +51,7 @@ def main (): try: changes = utils.parse_changes('krb5_1.2.2-4_m68k.changes', 0) except utils.changes_parse_error_exc, line: - fail("parse_changes() returned an exception with error message `%s'." % (line)); + fail("parse_changes() returned an exception with error message `%s'." % (line)) o = changes.get("binary", "") if o != "": @@ -61,7 +61,7 @@ def main (): changes["binary"][j] = 1 if not changes["binary"].has_key("krb5-ftpd"): - fail("parse_changes() is broken; 'krb5-ftpd' is not in the Binary: dictionary."); + fail("parse_changes() is broken; 'krb5-ftpd' is not in the Binary: dictionary.") ################################################################################ diff --git a/dak/test/004/test.py b/dak/test/004/test.py index be272ba3..014f6c6c 100755 --- a/dak/test/004/test.py +++ b/dak/test/004/test.py @@ -20,17 +20,17 @@ ################################################################################ -import os, sys; +import os, sys -sys.path.append(os.path.abspath('../../')); +sys.path.append(os.path.abspath('../../')) import utils ################################################################################ def fail(message): - sys.stderr.write("%s\n" % (message)); - sys.exit(1); + sys.stderr.write("%s\n" % (message)) + sys.exit(1) ################################################################################ @@ -47,75 +47,75 @@ def fail(message): # [6] Everything else is case sensitive. def test(input, output): - result = utils.extract_component_from_section(input); + result = utils.extract_component_from_section(input) if result != output: - fail ("%s -> %r [should have been %r]" % (input, result, output)); + fail ("%s -> %r [should have been %r]" % (input, result, output)) def main (): # Err, whoops? should probably be "utils", "main"... - input = "main/utils"; output = ("main/utils", "main"); - test (input, output); + input = "main/utils"; output = ("main/utils", "main") + test (input, output) # Validate #3 - input = "utils"; output = ("utils", "main"); - test (input, output); + input = "utils"; output = ("utils", "main") + test (input, output) - input = "non-free/libs"; output = ("non-free/libs", "non-free"); - test (input, output); + input = "non-free/libs"; output = ("non-free/libs", "non-free") + test (input, output) - input = "contrib/net"; output = ("contrib/net", "contrib"); - test (input, output); + input = "contrib/net"; output = ("contrib/net", "contrib") + test (input, output) # Validate #3 with a prefix - input = "non-US"; output = ("non-US", "non-US/main"); - test (input, output); + input = "non-US"; output = ("non-US", "non-US/main") + test (input, output) # Validate #4 - input = "main"; output = ("main", "main"); - test (input, output); + input = "main"; output = ("main", "main") + test (input, output) - input = "contrib"; output = ("contrib", "contrib"); - test (input, output); + input = "contrib"; output = ("contrib", "contrib") + test (input, output) - input = "non-free"; output = ("non-free", "non-free"); - test (input, output); + input = "non-free"; output = ("non-free", "non-free") + test (input, output) # Validate #4 with a prefix - input = "non-US/main"; output = ("non-US/main", "non-US/main"); - test (input, output); + input = "non-US/main"; output = ("non-US/main", "non-US/main") + test (input, output) - input = "non-US/contrib"; output = ("non-US/contrib", "non-US/contrib"); - test (input, output); + input = "non-US/contrib"; output = ("non-US/contrib", "non-US/contrib") + test (input, output) - input = "non-US/non-free"; output = ("non-US/non-free", "non-US/non-free"); - test (input, output); + input = "non-US/non-free"; output = ("non-US/non-free", "non-US/non-free") + test (input, output) # Validate #5 - input = "non-us"; output = ("non-us", "non-US/main"); - test (input, output); + input = "non-us"; output = ("non-us", "non-US/main") + test (input, output) - input = "non-us/contrib"; output = ("non-us/contrib", "non-US/contrib"); - test (input, output); + input = "non-us/contrib"; output = ("non-us/contrib", "non-US/contrib") + test (input, output) # Validate #6 (section) - input = "utIls"; output = ("utIls", "main"); - test (input, output); + input = "utIls"; output = ("utIls", "main") + test (input, output) # Others.. - input = "non-US/libs"; output = ("non-US/libs", "non-US/main"); - test (input, output); - input = "non-US/main/libs"; output = ("non-US/main/libs", "non-US/main"); - test (input, output); - input = "non-US/contrib/libs"; output = ("non-US/contrib/libs", "non-US/contrib"); - test (input, output); - input = "non-US/non-free/libs"; output = ("non-US/non-free/libs", "non-US/non-free"); - test (input, output); + input = "non-US/libs"; output = ("non-US/libs", "non-US/main") + test (input, output) + input = "non-US/main/libs"; output = ("non-US/main/libs", "non-US/main") + test (input, output) + input = "non-US/contrib/libs"; output = ("non-US/contrib/libs", "non-US/contrib") + test (input, output) + input = "non-US/non-free/libs"; output = ("non-US/non-free/libs", "non-US/non-free") + test (input, output) ################################################################################ diff --git a/dak/test/005/test.py b/dak/test/005/test.py index 5f561fa9..a84db60e 100755 --- a/dak/test/005/test.py +++ b/dak/test/005/test.py @@ -22,15 +22,15 @@ import os, sys -sys.path.append(os.path.abspath('../../')); +sys.path.append(os.path.abspath('../../')) import utils ################################################################################ def fail(message): - sys.stderr.write("%s\n" % (message)); - sys.exit(1); + sys.stderr.write("%s\n" % (message)) + sys.exit(1) ################################################################################ @@ -40,10 +40,10 @@ def main (): try: changes = utils.parse_changes("%s.changes" % (file), strict_whitespace) except utils.changes_parse_error_exc, line: - fail("%s[%s]: parse_changes() returned an exception with error message `%s'." % (file, strict_whitespace, line)); - oh_dear = changes.get("you"); + fail("%s[%s]: parse_changes() returned an exception with error message `%s'." % (file, strict_whitespace, line)) + oh_dear = changes.get("you") if oh_dear: - fail("%s[%s]: parsed and accepted unsigned data!" % (file, strict_whitespace)); + fail("%s[%s]: parsed and accepted unsigned data!" % (file, strict_whitespace)) ################################################################################ diff --git a/dak/test/006/test.py b/dak/test/006/test.py index d3d90919..9293c021 100755 --- a/dak/test/006/test.py +++ b/dak/test/006/test.py @@ -23,35 +23,35 @@ import os, sys -sys.path.append(os.path.abspath('../../')); +sys.path.append(os.path.abspath('../../')) import utils ################################################################################ def fail(message): - sys.stderr.write("%s\n" % (message)); - sys.exit(1); + sys.stderr.write("%s\n" % (message)) + sys.exit(1) ################################################################################ def check_valid(s, xa, xb, xc, xd): (a, b, c, d) = utils.fix_maintainer(s) if a != xa: - fail("rfc822_maint: %s (returned) != %s (expected [From: '%s']" % (a, xa, s)); + fail("rfc822_maint: %s (returned) != %s (expected [From: '%s']" % (a, xa, s)) if b != xb: - fail("rfc2047_maint: %s (returned) != %s (expected [From: '%s']" % (b, xb, s)); + fail("rfc2047_maint: %s (returned) != %s (expected [From: '%s']" % (b, xb, s)) if c != xc: - fail("name: %s (returned) != %s (expected [From: '%s']" % (c, xc, s)); + fail("name: %s (returned) != %s (expected [From: '%s']" % (c, xc, s)) if d != xd: - fail("email: %s (returned) != %s (expected [From: '%s']" % (d, xd, s)); + fail("email: %s (returned) != %s (expected [From: '%s']" % (d, xd, s)) def check_invalid(s): try: - utils.fix_maintainer(s); - fail("%s was parsed successfully but is expected to be invalid." % (s)); + utils.fix_maintainer(s) + fail("%s was parsed successfully but is expected to be invalid." % (s)) except utils.ParseMaintError, unused: - pass; + pass def main (): # Check Valid UTF-8 maintainer field @@ -60,7 +60,7 @@ def main (): xb = "=?utf-8?b?Tm/DqGwgS8O2dGhl?= " xc = "Noèl Köthe" xd = "noel@debian.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check valid ISO-8859-1 maintainer field s = "Noèl Köthe " @@ -68,7 +68,7 @@ def main (): xb = "=?iso-8859-1?q?No=E8l_K=F6the?= " xc = "Noèl Köthe" xd = "noel@debian.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check valid ASCII maintainer field s = "James Troup " @@ -76,7 +76,7 @@ def main (): xb = "James Troup " xc = "James Troup" xd = "james@nocrew.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check "Debian vs RFC822" fixup of names with '.' or ',' in them s = "James J. Troup " @@ -84,13 +84,13 @@ def main (): xb = "james@nocrew.org (James J. Troup)" xc = "James J. Troup" xd = "james@nocrew.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) s = "James J, Troup " xa = "james@nocrew.org (James J, Troup)" xb = "james@nocrew.org (James J, Troup)" xc = "James J, Troup" xd = "james@nocrew.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check just-email form s = "james@nocrew.org" @@ -98,7 +98,7 @@ def main (): xb = " " xc = "" xd = "james@nocrew.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check bracketed just-email form s = "" @@ -106,7 +106,7 @@ def main (): xb = " " xc = "" xd = "james@nocrew.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check Krazy quoted-string local part email address s = "Cris van Pelt <\"Cris van Pelt\"@tribe.eu.org>" @@ -114,16 +114,16 @@ def main (): xb = "Cris van Pelt <\"Cris van Pelt\"@tribe.eu.org>" xc = "Cris van Pelt" xd = "\"Cris van Pelt\"@tribe.eu.org" - check_valid(s, xa, xb, xc, xd); + check_valid(s, xa, xb, xc, xd) # Check empty string - s = xa = xb = xc = xd = ""; - check_valid(s, xa, xb, xc, xd); + s = xa = xb = xc = xd = "" + check_valid(s, xa, xb, xc, xd) # Check for missing email address - check_invalid("James Troup"); + check_invalid("James Troup") # Check for invalid email address - check_invalid("James Troup