X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=fcd2de356aafed0c1ad6d0fe55bb12c55695623e;hb=e65fe609400677be4ee3429e7251184b2fbc178e;hp=281d428782391cea5c8042c1acf492dc51a3ca48;hpb=99b369add85beec0a3335d5687bdf699d47da6b2;p=dak.git

diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index 281d4287..fcd2de35 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -5,6 +5,7 @@ function ts() {
 }
 
 # Remove daily lock
+# FIXME: Dead code when switched to cronscript
 function remove_daily_lock() {
     rm -f $LOCK_DAILY
 }
@@ -19,6 +20,17 @@ function remove_all_locks() {
     rm -f $LOCK_DAILY $LOCK_ACCEPTED
 }
 
+function remove_locks {
+    remove_all_locks
+    trap - EXIT TERM HUP INT QUIT
+    ts "locked part finished"
+}
+
+function lockaccepted {
+    lockfile "$LOCK_ACCEPTED"
+    trap remove_all_locks EXIT TERM HUP INT QUIT
+}
+
 # If we error out this one is called, *FOLLOWED* by cleanup above
 function onerror() {
     ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S")
@@ -39,13 +51,13 @@ function onerror() {
 }
 
 ########################################################################
-# the actual dinstall functions follow                                 #
+# the actual functions follow                                          #
 ########################################################################
 
 # pushing merkels QA user, part one
 function qa1() {
     log "Telling QA user that we start dinstall"
-    ssh -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
+    ssh -n -2 -i ~dak/.ssh/push_merkel_qa  -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@qa.debian.org sleep 1
 }
 
 # Updating various files
@@ -76,7 +88,7 @@ function i18n1() {
             # Lets check!
             if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                 # Yay, worked, lets copy around
-                for dir in stretch sid; do
+                for dir in ${extimportdists}; do
                     if [ -d dists/${dir}/ ]; then
                         cd dists/${dir}/main/i18n
                         rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
@@ -97,6 +109,34 @@ function i18n1() {
     fi
 }
 
+# Syncing AppStream/DEP-11 data
+function dep11() {
+    log "Synchronizing AppStream metadata"
+    # First sync their newest data
+    local dep11dir="${scriptdir}/dep11"
+    mkdir -p ${dep11dir}
+    cd ${dep11dir}
+    rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true
+
+    # Lets check!
+    if ${scriptsdir}/dep11-basic-validate.py . ${dep11dir}/; then
+        # Yay, worked, lets copy around
+        for dir in ${extimportdists}; do
+            if [ -d ${dir}/ ]; then
+                for comp in main contrib non-free; do
+                    mkdir -p ${ftpdir}/dists/${dir}/${comp}/dep11
+                    cd ${dir}/${comp}
+                    rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/.
+                    cd ${dep11dir}
+                done
+            fi
+        done
+    else
+        echo "ARRRR, bad guys, wrong files, ARRR"
+        echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" mak@debian.org
+    fi
+}
+
 function cruft() {
     log "Checking for cruft in overrides"
     dak check-overrides
@@ -105,18 +145,13 @@ function cruft() {
 function dominate() {
     log "Removing obsolete source and binary associations"
     dak dominate
+    dak manage-debug-suites unstable-debug experimental-debug
 }
 
 function autocruft() {
     log "Check for obsolete binary packages"
-    suites=${1:-"unstable experimental"}
-    # Initially only run with -n and output into a file, to check.
-    dstamp=$(date -u +"%Y-%m-%d_%H:%M")
-    echo "Report for ${dstamp}" > $webdir/auto-cruft-report_${dstamp}.txt
-    for suite in ${suites}; do
-        dak auto-decruft -n -s ${suite} >> $webdir/auto-cruft-report_${dstamp}.txt
-    done
-    cat $webdir/auto-cruft-report_${dstamp}.txt | mail -a "X-Debian: DAK" -e -s "Debian archive AUTOMATIC cruft report for ${dstamp}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" -c niels@thykier.net ftpmaster@ftp-master.debian.org
+    dak auto-decruft -s unstable
+    dak auto-decruft -s experimental --if-newer-version-in unstable --if-newer-version-in-rm-msg "NVIU"
 }
 
 function fingerprints() {
@@ -234,11 +269,14 @@ function mkmaintainers() {
 function copyoverrides() {
     log 'Copying override files into public view ...'
 
-    for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,stretch,sid}.{,extra.}{main,contrib,non-free}*; do
-        bname=${ofile##*/}
-        gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
-        chmod g+w ${indices}/${bname}.gz
-    done
+    (
+        shopt -s nullglob
+        for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,stretch,sid}.{,extra.}{main,contrib,non-free}*; do
+            bname=${ofile##*/}
+            gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
+            chmod g+w ${indices}/${bname}.gz
+        done
+    )
 }
 
 function mkfilesindices() {
@@ -396,28 +434,29 @@ function mkchecksums() {
 }
 
 function mirror() {
-    local archiveroot mirrordir
-
-    log "Regenerating \"public\" mirror/ hardlink fun"
-    DATE_SERIAL=$(date +"%Y%m%d01")
-    FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
-    if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
-        SERIAL="$DATE_SERIAL"
-    else
-        SERIAL="$FILESOAPLUS1"
-    fi
-    date -u > ${TRACEFILE}
-    echo "Using dak v1" >> ${TRACEFILE}
-    echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
-    echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
-
-    # Ugly "hack", but hey, it does what we want.
-    cp ${TRACEFILE} ${TRACEFILE_BDO}
+    local archiveroot targetpath TRACEFILE
 
     for archive in "${public_archives[@]}"; do
         archiveroot="$(get_archiveroot "${archive}")"
-        mirrordir="${archiveroot}/../mirror"
-        cd ${mirrordir}
+        targetpath="${mirrordir}/${archive}"
+        TRACEFILE="${archiveroot}/project/trace/ftp-master.debian.org"
+        mkdir -p "${archiveroot}/project/trace/"
+
+        log "Regenerating \"public\" mirror/${archive} hardlink fun"
+        DATE_SERIAL=$(date +"%Y%m%d01")
+        FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} || echo ${DATE_SERIAL} )
+        if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
+            SERIAL="$DATE_SERIAL"
+        else
+            SERIAL="$FILESOAPLUS1"
+        fi
+        date -u > ${TRACEFILE}
+        echo "Using dak v1" >> ${TRACEFILE}
+        echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
+        echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
+
+        mkdir -p ${targetpath}
+        cd ${targetpath}
         rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new  --ignore-errors ${archiveroot}/. .
     done
 }
@@ -444,66 +483,83 @@ function bts() {
     sudo -u dak-unpriv dak bts-categorize
 }
 
-function ddaccess() {
-    # Tell our dd accessible mirror to sync itself up. Including ftp dir.
-    log "Trigger dd accessible parts sync including ftp dir"
-    ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
-}
-
 function mirrorpush() {
-    log "Checking the public archive copy"
-    cd ${mirrordir}/dists
-
-    broken=0
-    for release in $(find . -name "InRelease"); do
-        echo "Processing: ${release}"
-        subdir=${release%/InRelease}
-        while read SHASUM SIZE NAME; do
-            if ! [ -f "${subdir}/${NAME}" ]; then
-               bname=$(basename ${NAME})
-                if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
-                    # We don't keep unpacked files, don't check for their existance.
-                    # We might want to go and check their unpacked shasum, but right now
-                    # I don't care. I believe it should be enough if all the packed shasums
-                    # match.
+    log "Checking the public archive copies..."
+
+    local archiveroot targetpath
+
+    for archive in "${public_archives[@]}"; do
+        log "... archive: ${archive}"
+        archiveroot="$(get_archiveroot "${archive}")"
+        targetpath="${mirrordir}/${archive}"
+        cd ${archiveroot}/dists
+
+        broken=0
+        for release in $(find . -name "InRelease"); do
+            echo "Processing: ${release}"
+            subdir=${release%/InRelease}
+            while read SHASUM SIZE NAME; do
+                if ! [ -f "${subdir}/${NAME}" ]; then
+                    bname=$(basename ${NAME})
+                    if [[ "${bname}" =~ ^(Packages|Sources|Contents-[a-zA-Z0-9-]+|Translation-[a-zA-Z_]+|Components-(amd64|i386)\.yml|icons-(128x128|64x64)\.tar)$ ]]; then
+
+                        # We don't keep unpacked files, don't check for their existance.
+                        # We might want to go and check their unpacked shasum, but right now
+                        # I don't care. I believe it should be enough if all the packed shasums
+                        # match.
+                        continue
+                    fi
+                    broken=$(( broken + 1 ))
+                    echo "File ${subdir}/${NAME} is missing"
                     continue
                 fi
-                broken=$(( broken + 1 ))
-                echo "File ${subdir}/${NAME} is missing"
-                continue
-            fi
-
-           # We do have symlinks in the tree (see the contents files currently).
-           # So we use "readlink -f" to check the size of the target, as thats basically
-           # what gen-releases does
-            fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
-            if [ ${fsize} -ne ${SIZE} ]; then
-                broken=$(( broken + 1 ))
-                echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
-                continue
-            fi
 
-            fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
-            fshasum=${fshasum%% *}
-            if [ "${fshasum}" != "${SHASUM}" ]; then
-                broken=$(( broken + 1 ))
-                echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
-                continue
-            fi
-        done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
-    done
+                # We do have symlinks in the tree (see the contents files currently).
+                # So we use "readlink -f" to check the size of the target, as thats basically
+                # what gen-releases does
+                fsize=$(stat -c %s $(readlink -f "${subdir}/${NAME}"))
+                if [ ${fsize} -ne ${SIZE} ]; then
+                    broken=$(( broken + 1 ))
+                    echo "File ${subdir}/${NAME} has size ${fsize}, expected is ${SIZE}"
+                    continue
+                fi
 
-    if [ $broken -gt 0 ]; then
-        log_error "Trouble with the public mirror, found ${broken} errors"
-        return 21
-    fi
+                fshasum=$(sha1sum $(readlink -f "${subdir}/${NAME}"))
+                fshasum=${fshasum%% *}
+                if [ "${fshasum}" != "${SHASUM}" ]; then
+                    broken=$(( broken + 1 ))
+                    echo "File ${subdir}/${NAME} has checksum ${fshasum}, expected is ${SHASUM}"
+                    continue
+                fi
+            done < <(sed '1,/SHA1:/d' "${release}" | sed '/SHA256:/,$d')
+        done
 
-    log "Starting the mirrorpush"
-    date -u > /srv/ftp.debian.org/web/mirrorstart
-    echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
-    echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
-    sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
-    sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 &
+        if [ $broken -gt 0 ]; then
+            log_error "Trouble with the public mirror for ${archive}, found ${broken} errors"
+            continue
+        else
+            log "Starting the mirrorpush for ${archive}"
+            case ${archive} in
+                ftp-master)
+                    fname="mirrorstart"
+                    pusharg=""
+                    ;;
+                debian-debug)
+                    pusharg="-a debug"
+                    ;;&
+                backports)
+                    pusharg="-a backports"
+                    ;;&
+                *)
+                    fname="mirrorstart.${archive}"
+                    ;;
+            esac
+            date -u > /srv/ftp.debian.org/web/${fname}
+            echo "Using dak v1" >> /srv/ftp.debian.org/web/${fname}
+            echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/${fname}
+            sudo -H -u archvsync /home/archvsync/runmirrors ${pusharg} > ~dak/runmirrors-${archive}.log 2>&1 &
+        fi
+    done
 }
 
 function mirrorpush-backports() {
@@ -545,7 +601,7 @@ function stats() {
 function cleantransactions() {
     log "Cleanup transaction ids older than 3 months"
     cd $base/backup/
-    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
+    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -delete
 }
 
 function logstats() {
@@ -554,42 +610,17 @@ function logstats() {
 
 # save timestamp when we start
 function savetimestamp() {
-	NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-	echo ${NOW} > "${dbdir}/dinstallstart"
+    echo ${NOW} > "${dbdir}/dinstallstart"
 }
 
 function maillogfile() {
     cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org
 }
 
-function renamelogfile() {
-    if [ -f "${dbdir}/dinstallstart" ]; then
-        NOW=$(cat "${dbdir}/dinstallstart")
-#        maillogfile
-        mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
-        logstats "$logdir/dinstall_${NOW}.log"
-        bzip2 -9 "$logdir/dinstall_${NOW}.log"
-    else
-        error "Problem, I don't know when dinstall started, unable to do log statistics."
-        NOW=`date "+%Y.%m.%d-%H:%M:%S"`
-#        maillogfile
-        mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
-        bzip2 -9 "$logdir/dinstall_${NOW}.log"
-    fi
-}
-
 function testingsourcelist() {
     dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list
 }
 
-# do a last run of process-unchecked before dinstall is on.
-function process_unchecked() {
-    log "Processing the unchecked queue"
-    UNCHECKED_WITHOUT_LOCK="-p"
-    do_unchecked
-    sync_debbugs
-}
-
 # Function to update a "statefile" telling people what we are doing
 # (more or less).
 #
@@ -626,35 +657,32 @@ function changelogs() {
     fi
 }
 
-function gitpdiff() {
-    # Might be that we want to change this to have more than one git repository.
-    # Advantage of one is that we do not need much space in terms of storage in git itself,
-    # git gc is pretty good on our input.
-    # But it might be faster. Well, lets test.
-    log "Adjusting the git tree for pdiffs"
-    cd ${dbdir}/git/git/
-
-    # The regex needs the architectures seperated with \|
-    garchs=$(dak admin a list|sed -e ':q;N;s/\n/\\|/g;t q')
-
-    # First, get all the files we want to work on. ../dists/ is a symlink to the real dists/ we
-    # want to work with.
-    # Also, we only want contents, packages and sources.
-    for file in $(find ../dists/ -regex ".*/\(Contents-\($archs\)\|\(Packages\|Sources\)\).gz"); do
-        log "${file}"
-        basen=${file%%.gz};
-        basen=${basen##../};
-        dir=${basen%/*};
-        mkdir -p $dir;
-        zcat $file > $basen;
+# Generate a list of extra mirror files, sha256sum em and sign that
+function signotherfiles() {
+    log "Signing extra mirror files"
+
+    local archiveroot
+
+    for archive in "${public_archives[@]}"; do
+        log "... archive: ${archive}"
+        archiveroot="$(get_archiveroot "${archive}")"
+        local TMPLO=$( mktemp -p ${TMPDIR} )
+        trap "rm -f ${TMPLO}" ERR EXIT TERM HUP INT QUIT
+
+        cd ${archiveroot}
+        rm -f extrafiles
+        sha256sum $(find * -type f | egrep -v '(pool|i18n|dep11|source)/|Contents-.*\.(gz|diff)|installer|binary-|(In)?Release(.gpg)?|\.changes' | sort) > ${TMPLO}
+        gpg --no-options --batch --no-tty --armour --personal-digest-preferences=SHA256 --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --clearsign --output ${archiveroot}/extrafiles ${TMPLO}
+        rm -f ${TMPLO}
     done
+}
 
-    # Second, add all there is into git
-    cd dists
-    git add .
-    # Maybe we want to make this the same for tag and commit? But well, shouldn't matter
-    COMD=$(date  -Is)
-    TAGD=$(date +%Y-%m-%d-%H-%M)
-    git commit -m "Commit of ${COMD}"
-    git tag "${TAGD}"
- }
+function startup() {
+    touch "${DINSTALLSTART}"
+    ts "startup"
+    DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+    lockfile -l 3600 "${LOCK_DAILY}"
+    trap onerror ERR
+    TEMPFILES="${TEMPFILES} ${LOCK_DAILY}"
+    touch "${LOCK_BRITNEY}"
+}