X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=9aadf836059925e76e56db028784318a341ce6e1;hb=705fde638084f64c52b161a7bf8703716772ca3f;hp=8d7b6d1c432692208d0d041777f5b8bfec33c6ed;hpb=e35de7a7441fcd49f49240352ccda01569d8a05b;p=dak.git diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 8d7b6d1c..9aadf836 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -19,6 +19,17 @@ function remove_all_locks() { rm -f $LOCK_DAILY $LOCK_ACCEPTED } +function remove_locks { + remove_all_locks + trap - EXIT TERM HUP INT QUIT + ts "locked part finished" +} + +function lockaccepted { + lockfile "$LOCK_ACCEPTED" + trap remove_all_locks EXIT TERM HUP INT QUIT +} + # If we error out this one is called, *FOLLOWED* by cleanup above function onerror() { ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S") @@ -45,7 +56,7 @@ function onerror() { # pushing merkels QA user, part one function qa1() { log "Telling QA user that we start dinstall" - ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 + ssh -n -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1 } # Updating various files @@ -76,7 +87,7 @@ function i18n1() { # Lets check! if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then # Yay, worked, lets copy around - for dir in stretch sid; do + for dir in ${extimportdists}; do if [ -d dists/${dir}/ ]; then cd dists/${dir}/main/i18n rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/. @@ -101,20 +112,21 @@ function i18n1() { function dep11() { log "Synchronizing AppStream metadata" # First sync their newest data - mkdir -p ${scriptdir}/dep11 - cd ${scriptdir}/dep11 + local dep11dir="${scriptdir}/dep11" + mkdir -p ${dep11dir} + cd ${dep11dir} rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true # Lets check! - if ${scriptsdir}/dep11-basic-validate.py . ${scriptdir}/dep11/; then + if ${scriptsdir}/dep11-basic-validate.py . ${dep11dir}/; then # Yay, worked, lets copy around - for dir in stretch sid; do + for dir in ${extimportdists}; do if [ -d ${dir}/ ]; then for comp in main contrib non-free; do mkdir -p ${ftpdir}/dists/${dir}/${comp}/dep11 cd ${dir}/${comp} rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/. - cd ${scriptdir}/dep11 + cd ${dep11dir} done fi done @@ -421,28 +433,29 @@ function mkchecksums() { } function mirror() { - local archiveroot mirrordir + local archiveroot targetpath TRACEFILE - log "Regenerating \"public\" mirror/ hardlink fun" - DATE_SERIAL=$(date +"%Y%m%d01") - FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} ) - if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then - SERIAL="$DATE_SERIAL" - else - SERIAL="$FILESOAPLUS1" - fi - date -u > ${TRACEFILE} - echo "Using dak v1" >> ${TRACEFILE} - echo "Running on host: $(hostname -f)" >> ${TRACEFILE} - echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} - - # Ugly "hack", but hey, it does what we want. - cp ${TRACEFILE} ${TRACEFILE_BDO} - - for archive in ftp-master backports; do + for archive in "${public_archives[@]}"; do archiveroot="$(get_archiveroot "${archive}")" - mirrordir="${archiveroot}/../mirror" - cd ${mirrordir} + targetpath="${mirrordir}/${archive}" + TRACEFILE="${archiveroot}/project/trace/ftp-master.debian.org" + mkdir -p "${archiveroot}/project/trace/" + + log "Regenerating \"public\" mirror/${archive} hardlink fun" + DATE_SERIAL=$(date +"%Y%m%d01") + FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} || echo ${DATE_SERIAL} ) + if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then + SERIAL="$DATE_SERIAL" + else + SERIAL="$FILESOAPLUS1" + fi + date -u > ${TRACEFILE} + echo "Using dak v1" >> ${TRACEFILE} + echo "Running on host: $(hostname -f)" >> ${TRACEFILE} + echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} + + mkdir -p ${targetpath} + cd ${targetpath} rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new --ignore-errors ${archiveroot}/. . done } @@ -476,60 +489,82 @@ function ddaccess() { } function mirrorpush() { - log "Checking the public archive copy" - cd ${mirrordir}/dists - - broken=0 - for release in $(find . -name "InRelease"); do - echo "Processing: ${release}" - subdir=${release%/InRelease} - while read SHASUM SIZE NAME; do - if ! [ -f "${subdir}/${NAME}" ]; then - bname=$(basename ${NAME}) - if [[ "${bname}" =~ ^(Packages|Sources|Contents-[a-zA-Z0-9-]+|Translation-[a-zA-Z_]+|Components-(amd64|i386)\.yml|icons-(128x128|64x64)\.tar)$ ]]; then - - # We don't keep unpacked files, don't check for their existance. - # We might want to go and check their unpacked shasum, but right now - # I don't care. I believe it should be enough if all the packed shasums - # match. + log "Checking the public archive copies..." + + local archiveroot targetpath + + for archive in "${public_archives[@]}"; do + log "... archive: ${archive}" + archiveroot="$(get_archiveroot "${archive}")" + targetpath="${mirrordir}/${archive}" + cd ${archiveroot}/dists + + broken=0 + for release in $(find . -name "InRelease"); do + echo "Processing: ${release}" + subdir=${release%/InRelease} + while read SHASUM SIZE NAME; do + if ! [ -f "${subdir}/${NAME}" ]; then + bname=$(basename ${NAME}) + if [[ "${bname}" =~ ^(Packages|Sources|Contents-[a-zA-Z0-9-]+|Translation-[a-zA-Z_]+|Components-(amd64|i386)\.yml|icons-(128x128|64x64)\.tar)$ ]]; then + + # We don't keep unpacked files, don't check for their existance. + # We might want to go and check their unpacked shasum, but right now + # I don't care. I believe it should be enough if all the packed shasums + # match. + continue + fi + broken=$(( broken + 1 )) + echo "File ${subdir}/${NAME} is missing" continue fi - broken=$(( broken + 1 )) - echo "File ${subdir}/${NAME} is missing" - continue - fi - - # We do have symlinks in the tree (see the contents files currently). - # So we use "readlink -f" to check the size of the target, as thats basically - # what gen-releases does - fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}")) - if [ ${fsize} -ne ${SIZE} ]; then - broken=$(( broken + 1 )) - echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}" - continue - fi - fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}")) - fshasum=${fshasum%% *} - if [ "${fshasum}" != "${SHASUM}" ]; then - broken=$(( broken + 1 )) - echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}" - continue - fi - done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d') - done + # We do have symlinks in the tree (see the contents files currently). + # So we use "readlink -f" to check the size of the target, as thats basically + # what gen-releases does + fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}")) + if [ ${fsize} -ne ${SIZE} ]; then + broken=$(( broken + 1 )) + echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}" + continue + fi - if [ $broken -gt 0 ]; then - log_error "Trouble with the public mirror, found ${broken} errors" - return 21 - fi + fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}")) + fshasum=${fshasum%% *} + if [ "${fshasum}" != "${SHASUM}" ]; then + broken=$(( broken + 1 )) + echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}" + continue + fi + done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d') + done - log "Starting the mirrorpush" - date -u > /srv/ftp.debian.org/web/mirrorstart - echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart - echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart - sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 & - sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 & + if [ $broken -gt 0 ]; then + log_error "Trouble with the public mirror for ${archive}, found ${broken} errors" + continue + else + log "Starting the mirrorpush for ${archive}" + case ${archive} in + ftp-master) + fname="mirrorstart" + pusharg="" + ;; + debian-debug) + pusharg="-a debug" + ;;& + backports) + pusharg="-a backports" + ;;& + *) + fname="mirrorstart.${archive}" + ;; + esac + date -u > /srv/ftp.debian.org/web/${fname} + echo "Using dak v1" >> /srv/ftp.debian.org/web/${fname} + echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/${fname} + sudo -H -u archvsync /home/archvsync/runmirrors ${pusharg} > ~dak/runmirrors-${archive}.log 2>&1 & + fi + done } function mirrorpush-backports() { @@ -580,42 +615,18 @@ function logstats() { # save timestamp when we start function savetimestamp() { - NOW=`date "+%Y.%m.%d-%H:%M:%S"` - echo ${NOW} > "${dbdir}/dinstallstart" + NOW=`date "+%Y.%m.%d-%H:%M:%S"` + echo ${NOW} > "${dbdir}/dinstallstart" } function maillogfile() { cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" -a "From: Debian FTP Masters " cron@ftp-master.debian.org } -function renamelogfile() { - if [ -f "${dbdir}/dinstallstart" ]; then - NOW=$(cat "${dbdir}/dinstallstart") -# maillogfile - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" - logstats "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" - else - error "Problem, I don't know when dinstall started, unable to do log statistics." - NOW=`date "+%Y.%m.%d-%H:%M:%S"` -# maillogfile - mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" - bzip2 -9 "$logdir/dinstall_${NOW}.log" - fi -} - function testingsourcelist() { dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list } -# do a last run of process-unchecked before dinstall is on. -function process_unchecked() { - log "Processing the unchecked queue" - UNCHECKED_WITHOUT_LOCK="-p" - do_unchecked - sync_debbugs -} - # Function to update a "statefile" telling people what we are doing # (more or less). # @@ -652,35 +663,21 @@ function changelogs() { fi } -function gitpdiff() { - # Might be that we want to change this to have more than one git repository. - # Advantage of one is that we do not need much space in terms of storage in git itself, - # git gc is pretty good on our input. - # But it might be faster. Well, lets test. - log "Adjusting the git tree for pdiffs" - cd ${dbdir}/git/git/ - - # The regex needs the architectures seperated with \| - garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q') - - # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we - # want to work with. - # Also, we only want contents, packages and sources. - for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do - log "${file}" - basen=${file%%.gz}; - basen=${basen##../}; - dir=${basen%/*}; - mkdir -p $dir; - zcat $file > $basen; - done +function signotherfiles() { + log "Signing extra mirror files" - # Second, add all there is into git - cd dists - git add . - # Maybe we want to make this the same for tag and commit? But well, shouldn't matter - COMD=$(date -Is) - TAGD=$(date +%Y-%m-%d-%H-%M) - git commit -m "Commit of ${COMD}" - git tag "${TAGD}" - } + local archiveroot + + for archive in "${public_archives[@]}"; do + log "... archive: ${archive}" + archiveroot="$(get_archiveroot "${archive}")" + local TMPLO=$( mktemp -p ${TMPDIR} ) + trap "rm -f ${TMPLO}" ERR EXIT TERM HUP INT QUIT + + cd ${archiveroot} + rm -f extrafiles + sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes') > ${TMPLO} + gpg --no-options --batch --no-tty --armour --personal-digest-preferences=SHA256 --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --clearsign --output ${archiveroot}/extrafiles ${TMPLO} + rm -f ${TMPLO} + done +}