dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
cat $webdir/cruft-report-daily.txt | mail -a "X-Debian: DAK" -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
}
+
+function pg_timestamp() {
+ tsname=${1:-"unknown"}
+ log "Saving postgres transaction id for ${tsname}"
+ psql -tAc 'select txid_current();' > $base/backup/txid_${tsname}_$(date +%Y.%m.%d-%H:%M:%S)
+}
stage $GO &
GO=(
- FUNC="pgdump_pre"
+ FUNC="pg_timestamp"
TIME="pg_dump1"
- ARGS=""
+ ARGS="predinstall"
ERR=""
)
stage $GO
stage $GO &
GO=(
- FUNC="pgdump_post"
+ FUNC="pg_timestamp"
TIME="pg_dump2"
- ARGS=""
+ ARGS="postdinstall"
ERR=""
)
-stage $GO &
+stage $GO
GO=(
FUNC="expire"
rm -f "${LOCK_BRITNEY}"
-GO=(
- FUNC="pgdakdev"
- TIME="dak-dev db"
- ARGS=""
- ERR="false"
-)
-stage $GO &
-
GO=(
FUNC="merkel3"
TIME="merkel ddaccessible sync"
stage $GO &
GO=(
- FUNC="compress"
- TIME="compress"
+ FUNC="cleantransactions"
+ TIME=""
ARGS=""
ERR=""
)
-stage $GO &
+stage $GO
GO=(
FUNC="aptftpcleanup"
fi
trap cleanup 0
+
+pg_timestamp preunchecked
do_newstage
do_unchecked
fi
dak contents -l 10000 scan
+pg_timestamp postunchecked
#ln -sf $base/backup/dumpall_$POSTDUMP currentall
}
-# Load the dak-dev projectb
-function pgdakdev() {
- # Make sure to unset any possible psql variables so we don't drop the wrong
- # f****** database by accident
- local PGDATABASE
- unset PGDATABASE
- local PGHOST
- unset PGHOST
- local PGPORT
- unset PGPORT
- local PGUSER
- unset PGUSER
- cd $base/backup
- echo "drop database projectb" | psql -p 5434 template1
- #cat currentall | psql -p 5433 template1
- createdb -p 5434 -T template1 projectb
- fgrep -v '\connect' current | psql -p 5434 projectb
-}
-
# Updating various files
function updates() {
log "Updating Bugs docu, Mirror list and mailing-lists.txt"
apt-ftparchive -q clean apt.conf
}
-function compress() {
- log "Compress old psql backups"
+function cleantransactions() {
+ log "Cleanup transaction ids older than 3 months"
cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
-
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 -print0 \
- | xargs --no-run-if-empty -0 -P4 -n1 xz -9v
-
- find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 \
- | xargs --no-run-if-empty -0 -P4 -n1 xz -9v
- finddup -l -d $base/backup
+ find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
}
function logstats() {
#!/usr/bin/env python
-""" Cruft checker and hole filler for overrides """
-# Copyright (C) 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
-# Copyright (C) 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
+""" Cruft checker and hole filler for overrides
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
+@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
+@copyright: 2011 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-Options = None
-Logger = None
+Options = None #: Commandline arguments parsed into this
+Logger = None #: Our logging object
sections = {}
priorities = {}
blacklist = {}
if not Options["No-Action"]:
session.execute("""DELETE FROM override WHERE package = :package
AND suite = :suite_id AND component = :component_id
- AND type = :type_id""",
+ AND type = :type_id
+ AND created < now() - interval '14 days'""",
{'package': package, 'suite_id': osuite_id,
'component_id': component_id, 'type_id': type_id})
# create source overrides based on binary overrides, as source
if not Options["No-Action"]:
session.execute("""DELETE FROM override
WHERE package = :package AND suite = :suite_id
- AND component = :component_id AND type = :type_id""",
+ AND component = :component_id AND type = :type_id
+ AND created < now() - interval '14 days'""",
{'package': package, 'suite_id': osuite_id,
'component_id': component_id, 'type_id': type_id})
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Fix up constraints for pg 9.0
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2011 Mark Hymers <mhy@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from socket import gethostname;
+
+################################################################################
+def do_update(self):
+ """
+ Fix up constraints for pg 9.0
+ """
+ print __doc__
+ try:
+ c = self.db.cursor()
+
+ c.execute("ALTER TABLE policy_queue DROP constraint policy_queue_perms_check")
+ c.execute("ALTER TABLE policy_queue DROP constraint policy_queue_change_perms_check")
+ c.execute("ALTER TABLE policy_queue ADD CONSTRAINT policy_queue_perms_check CHECK (perms SIMILAR TO '[0-7][0-7][0-7][0-7]')")
+ c.execute("ALTER TABLE policy_queue ADD CONSTRAINT policy_queue_change_perms_check CHECK (change_perms SIMILAR TO '[0-7][0-7][0-7][0-7]')")
+
+ c.execute("UPDATE config SET value = '43' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, 'Unable to apply update 43, rollback issued. Error message : %s' % (str(msg))
import apt_pkg
from tempfile import mkstemp, mkdtemp
import commands
-from multiprocessing import Pool
+from multiprocessing import Pool, TimeoutError
from daklib import daklog
from daklib.dbconn import *
Options = None #: Commandline arguments parsed into this
Logger = None #: Our logging object
+results = [] #: Results of the subprocesses
################################################################################
(result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
sn="a-f %s,%s: " % (suite, arch)
print sn + output.replace('\n', '\n%s' % (sn))
+ return result
# Clean up any left behind files
finally:
def sname(arch):
return arch.arch_string
+def get_result(arg):
+ global results
+ if arg:
+ results.append(arg)
+
########################################################################
########################################################################
def main ():
- global Options, Logger
+ global Options, Logger, results
cnf = Config()
startdir = os.getcwd()
os.chdir(cnf["Dir::TempPath"])
+ broken=[]
# For each given suite, each architecture, run one apt-ftparchive
for s in suites:
+ results=[]
# Setup a multiprocessing Pool. As many workers as we have CPU cores.
pool = Pool()
arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
for a in arch_list:
- pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
+ pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)
# No more work will be added to our pool, close it and then wait for all to finish
pool.close()
pool.join()
+ if len(results) > 0:
+ Logger.log(['Trouble, something with a-f broke, resultcodes: %s' % (results)])
+ print "Trouble, something with a-f broke, resultcodes: %s" % (results)
+ sys.exit(1)
+
os.chdir(startdir)
# this script doesn't change the database
session.close()
table = Table(table_name, self.db_meta, autoload=True)
setattr(self, 'tbl_%s' % table_name, table)
- # bin_contents needs special attention until update #41 has been
- # applied
+ # bin_contents needs special attention until the SERIAL type is
+ # correctly detected and the workaround has been removed; see comment
+ # above
self.tbl_bin_contents = Table('bin_contents', self.db_meta, \
Column('file', Text, primary_key = True),
Column('binary_id', Integer, ForeignKey('binaries.id'), \
# module
import warnings
warnings.filterwarnings('ignore', \
- "apt_pkg-.ParseSection\(\) is deprecated. Please see apt_pkg\.TagSection\(\) for the replacement\.", \
+ "apt_pkg.ParseSection\(\) is deprecated. Please see apt_pkg\.TagSection\(\) for the replacement\.", \
DeprecationWarning)
warnings.filterwarnings('ignore', \
"Attribute 'Find' of the 'apt_pkg\.TagSection' object is deprecated, use 'find' instead\.", \
<helix> elmo: I can't believe people pay you to fix computers
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+<mhy> I wish they wouldnt leave biscuits out, thats just tempting. Damnit.
def parse_file_dates(list):
out = []
# dump_2006.05.02-11:52:01.bz2
- p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$')
+ p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.xz)?$')
for file in list:
m = p.search(file)
if m:
#!/bin/sh -e
-. vars
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
export TERM=linux
# Fetches latest copy of mailing-lists.txt
# Michael Beattie <mjb@debian.org>
-. vars
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
cd $ftpdir/doc
# Very Very hackish script... dont laugh.
# Michael Beattie <mjb@debian.org>
-. vars
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
prog=$scriptdir/mirrorlist/mirror_list.pl
masterlist=$scriptdir/mirrorlist/Mirrors.masterlist
# Fetches latest copy of pseudo-packages
# Joerg Jaspert <joerg@debian.org>
-. vars
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
cd ${scriptdir}/masterfiles
projects / dak.git / commitdiff