X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=config%2Fdebian%2Fdinstall.functions;h=bd57a3a8de757a1b193e034f891add2c3a3b8cbb;hb=33233a41d742249ceb0c816c15f2850970ba3634;hp=ebd2b69269dfd2f4f8b4f32b88f2a51a90ed00b9;hpb=c19edbf26093a7a2da8b3414f6188c5584998201;p=dak.git

diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index ebd2b692..bd57a3a8 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -9,9 +9,14 @@ function remove_daily_lock() {
     rm -f $LOCK_DAILY
 }
 
+# Remove changelog lock
+function remove_changelog_lock() {
+    rm -f $LOCK_CHANGELOG
+}
+
 # Remove all locks
 function remove_all_locks() {
-    rm -f $LOCK_DAILY $LOCK_ACCEPTED $LOCK_NEW
+    rm -f $LOCK_DAILY $LOCK_ACCEPTED
 }
 
 # If we error out this one is called, *FOLLOWED* by cleanup above
@@ -71,10 +76,10 @@ function i18n1() {
             # Lets check!
             if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                 # Yay, worked, lets copy around
-                for dir in wheezy sid; do
+                for dir in jessie sid; do
                     if [ -d dists/${dir}/ ]; then
                         cd dists/${dir}/main/i18n
-                        rsync -aq --delete --delete-after  . ${ftpdir}/dists/${dir}/main/i18n/.
+                        rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
                     fi
                     cd ${scriptdir}/i18nsync
                 done
@@ -146,16 +151,23 @@ function overrides() {
 }
 
 function mpfm() {
+    local archiveroot
+
     log "Generating package / file mapping"
-    dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2
+    for archive in "${public_archives[@]}"; do
+        archiveroot="$(get_archiveroot "${archive}")"
+        dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
+    done
 }
 
 function packages() {
     log "Generating Packages and Sources files"
-    #cd $configdir
-    #dak generate-packages-sources
-    dak generate-packages-sources2
-    dak contents generate
+    for archive in "${public_archives[@]}"; do
+        log "  Generating Packages/Sources for ${archive}"
+        dak generate-packages-sources2 -a "${archive}"
+        log "  Generating Contents for ${archive}"
+        dak contents generate -a "${archive}"
+    done
 }
 
 function pdiff() {
@@ -164,72 +176,62 @@ function pdiff() {
 }
 
 function release() {
-    # XXX: disable once we can remove i18n/Index (#649314)
-    log "Generating i18n/Index"
-    (cd "$ftpdir/dists"; for dist in testing unstable; do $scriptsdir/generate-i18n-Index $dist; done)
     log "Generating Release files"
-    dak generate-releases
+    for archive in "${public_archives[@]}"; do
+        dak generate-releases -a "${archive}"
+    done
 }
 
 function dakcleanup() {
     log "Cleanup old packages/files"
     dak clean-suites -m 10000
-    dak clean-queues
-}
-
-function buildd_dir() {
-    # Rebuilt the buildd dir to avoid long times of 403
-    log "Regenerating the buildd incoming dir"
-    STAMP=$(date "+%Y%m%d%H%M")
-    make_buildd_dir
+    dak clean-queues -i "$unchecked"
 }
 
 function mklslar() {
-    cd $ftpdir
+    local archiveroot
+    local FILENAME=ls-lR
 
-    FILENAME=ls-lR
+    for archive in "${public_archives[@]}"; do
+        archiveroot="$(get_archiveroot "${archive}")"
+        cd "${archiveroot}"
 
-    log "Removing any core files ..."
-    find -type f -name core -print0 | xargs -0r rm -v
+        log "Removing any core files ..."
+        find -type f -name core -print -delete
 
-    log "Checking permissions on files in the FTP tree ..."
-    find -type f \( \! -perm -444 -o -perm +002 \) -ls
-    find -type d \( \! -perm -555 -o -perm +002 \) -ls
+        log "Checking symlinks ..."
+        symlinks -rd .
 
-    log "Checking symlinks ..."
-    symlinks -rd .
-
-    log "Creating recursive directory listing ... "
-    rm -f .${FILENAME}.new
-    TZ=UTC ls -lR > .${FILENAME}.new
-
-    if [ -r ${FILENAME}.gz ] ; then
-        mv -f ${FILENAME}.gz ${FILENAME}.old.gz
-        mv -f .${FILENAME}.new ${FILENAME}
-        rm -f ${FILENAME}.patch.gz
-        zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz
-        rm -f ${FILENAME}.old.gz
-    else
-        mv -f .${FILENAME}.new ${FILENAME}
-    fi
-
-    gzip -9cfN ${FILENAME} >${FILENAME}.gz
-    rm -f ${FILENAME}
+        log "Creating recursive directory listing ... "
+        rm -f ${FILENAME}.gz
+        TZ=UTC ls -lR | gzip -9c --rsyncable > ${FILENAME}.gz
+    done
 }
 
 function mkmaintainers() {
+    local archiveroot
+    local indices
+
     log 'Creating Maintainers index ... '
 
-    cd $indices
-    dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers
-    gzip -9v --rsyncable <Maintainers >Maintainers.gz
-    gzip -9v --rsyncable <Uploaders >Uploaders.gz
+    for archive in "${public_archives[@]}"; do
+        archiveroot="$(get_archiveroot "${archive}")"
+	indices="${archiveroot}/indices"
+	if ! [ -d "${indices}" ]; then
+	    mkdir "${indices}"
+	fi
+        cd "${indices}"
+
+        dak make-maintainers -a "${archive}" ${scriptdir}/masterfiles/pseudo-packages.maintainers
+        gzip -9v --rsyncable <Maintainers >Maintainers.gz
+        gzip -9v --rsyncable <Uploaders >Uploaders.gz
+    done
 }
 
 function copyoverrides() {
     log 'Copying override files into public view ...'
 
-    for ofile in ${overridedir}/override.{lenny,squeeze,wheezy,sid}.{,extra.}{main,contrib,non-free}*; do
+    for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,sid}.{,extra.}{main,contrib,non-free}*; do
         bname=${ofile##*/}
         gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
         chmod g+w ${indices}/${bname}.gz
@@ -237,13 +239,27 @@ function copyoverrides() {
 }
 
 function mkfilesindices() {
+    set +o pipefail
     umask 002
     cd $base/ftp/indices/files/components
 
     ARCHLIST=$(tempfile)
 
     log "Querying postgres"
-    echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST
+    local query="
+      SELECT './pool/' || c.name || '/' || f.filename AS path, a.arch_string AS arch_string
+      FROM files f
+      JOIN files_archive_map af ON f.id = af.file_id
+      JOIN component c ON af.component_id = c.id
+      JOIN archive ON af.archive_id = archive.id
+      LEFT OUTER JOIN
+        (binaries b
+         JOIN architecture a ON b.architecture = a.id)
+        ON f.id = b.file
+      WHERE archive.name = 'ftp-master'
+      ORDER BY path, arch_string
+    "
+    psql -At -c "$query" >$ARCHLIST
 
     includedirs () {
         perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }'
@@ -276,12 +292,46 @@ function mkfilesindices() {
     log "Generating suite lists"
 
     suite_list () {
-        printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t
-
-        printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t
+	local suite_id="$(printf %d $1)"
+	local query
+	query="
+          SELECT DISTINCT './pool/' || c.name || '/' || f.filename
+          FROM
+            (SELECT sa.source AS source
+               FROM src_associations sa
+              WHERE sa.suite = $suite_id
+             UNION
+             SELECT esr.src_id
+               FROM extra_src_references esr
+               JOIN bin_associations ba ON esr.bin_id = ba.bin
+               WHERE ba.suite = $suite_id
+             UNION
+             SELECT b.source AS source
+               FROM bin_associations ba
+               JOIN binaries b ON ba.bin = b.id WHERE ba.suite = $suite_id) s
+            JOIN dsc_files df ON s.source = df.source
+            JOIN files f ON df.file = f.id
+            JOIN files_archive_map af ON f.id = af.file_id
+            JOIN component c ON af.component_id = c.id
+            JOIN archive ON af.archive_id = archive.id
+            WHERE archive.name = 'ftp-master'
+        "
+	psql -F' ' -A -t -c "$query"
+
+	query="
+          SELECT './pool/' || c.name || '/' || f.filename
+          FROM bin_associations ba
+          JOIN binaries b ON ba.bin = b.id
+          JOIN files f ON b.file = f.id
+          JOIN files_archive_map af ON f.id = af.file_id
+          JOIN component c ON af.component_id = c.id
+          JOIN archive ON af.archive_id = archive.id
+          WHERE ba.suite = $suite_id AND archive.name = 'ftp-master'
+        "
+	psql -F' ' -A -t -c "$query"
     }
 
-    printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At |
+    psql -F' ' -At -c "SELECT id, suite_name FROM suite" |
     while read id suite; do
         [ -e $base/ftp/dists/$suite ] || continue
         (
@@ -292,7 +342,7 @@ function mkfilesindices() {
                     [ "$(readlink $distdir)" != "$distname" ] || echo $distdir
                 done
             )
-            suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,'
+            suite_list $id
         ) | sort -u | gzip -9 > suite-${suite}.list.gz
     done
 
@@ -312,31 +362,39 @@ function mkfilesindices() {
     done
 
     (cd $base/ftp/
-	    for dist in sid wheezy; do
+	    for dist in sid jessie; do
 		    find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
 	    done
     )
 
-    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-wheezy.list.gz) |
+    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz) |
     sort -u | poolfirst > ../typical.files
 
     rm -f $ARCHLIST
     log "Done!"
+    set -o pipefail
 }
 
 function mkchecksums() {
-    dsynclist=$dbdir/dsync.list
-    md5list=$indices/md5sums
+    local archiveroot dsynclist md5list
 
-    log -n "Creating md5 / dsync index file ... "
+    for archive in "${public_archives[@]}"; do
+        archiveroot="$(get_archiveroot "${archive}")"
+        dsynclist=$dbdir/dsync.${archive}.list
+        md5list=${archiveroot}/indices/md5sums
 
-    cd "$ftpdir"
-    ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
-    ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
-    ${bindir}/dsync-flist -q link-dups $dsynclist || true
+        log -n "Creating md5 / dsync index file for ${archive}... "
+
+        cd "$archiveroot"
+        ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5
+        ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz
+        ${bindir}/dsync-flist -q link-dups $dsynclist || true
+    done
 }
 
 function mirror() {
+    local archiveroot
+
     log "Regenerating \"public\" mirror/ hardlink fun"
     DATE_SERIAL=$(date +"%Y%m%d01")
     FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
@@ -349,8 +407,16 @@ function mirror() {
     echo "Using dak v1" >> ${TRACEFILE}
     echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
     echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
-    cd ${mirrordir}
-    rsync -aH --link-dest ${ftpdir} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new  --ignore-errors ${ftpdir}/. .
+
+    # Ugly "hack", but hey, it does what we want.
+    cp ${TRACEFILE} ${TRACEFILE_BDO}
+
+    for archive in "${public_archives[@]}"; do
+        archiveroot="$(get_archiveroot "${archive}")"
+        mirrordir="${archiveroot}/../mirror"
+        cd ${mirrordir}
+        rsync -aH --link-dest ${archiveroot} --delete --delete-after --delete-excluded --exclude Packages.*.new --exclude Sources.*.new  --ignore-errors ${archiveroot}/. .
+    done
 }
 
 function expire() {
@@ -366,8 +432,8 @@ function transitionsclean() {
 }
 
 function dm() {
-    log "Updating DM html page"
-    $scriptsdir/dm-monitor >$webdir/dm-uploaders.html
+    log "Updating DM permissions page"
+    dak acl export-per-source dm >$exportdir/dm.txt
 }
 
 function bts() {
@@ -378,7 +444,7 @@ function bts() {
 function ddaccess() {
     # Tell our dd accessible mirror to sync itself up. Including ftp dir.
     log "Trigger dd accessible parts sync including ftp dir"
-    ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool
+    ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 pool
 }
 
 function mirrorpush() {
@@ -399,7 +465,7 @@ function mirrorpush() {
                     # match.
                     continue
                 fi
-               broken=$(( broken + 1 ))
+                broken=$(( broken + 1 ))
                 echo "File ${subdir}/${NAME} is missing"
                 continue
             fi
@@ -434,6 +500,12 @@ function mirrorpush() {
     echo "Using dak v1" >> /srv/ftp.debian.org/web/mirrorstart
     echo "Running on host $(hostname -f)" >> /srv/ftp.debian.org/web/mirrorstart
     sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 &
+    sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 &
+}
+
+function mirrorpush-backports() {
+    log "Syncing backports mirror"
+    sudo -u backports /home/backports/bin/update-archive
 }
 
 function i18n2() {
@@ -441,11 +513,14 @@ function i18n2() {
     STAMP=$(date "+%Y%m%d%H%M")
     mkdir -p ${scriptdir}/i18n/${STAMP}
     cd ${scriptdir}/i18n/${STAMP}
-    dak control-suite -l stable > squeeze
-    dak control-suite -l testing > wheezy
-    dak control-suite -l unstable > sid
+    for suite in stable testing unstable; do
+        codename=$(dak admin s show ${suite}|grep '^Codename')
+        codename=${codename##* }
+        echo "Codename is ${codename}"
+        dak control-suite -l ${suite} >${codename}
+    done
     echo "${STAMP}" > timestamp
-    gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp
+    gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 473041FA --detach-sign -o timestamp.gpg timestamp
     rm -f md5sum
     md5sum * > md5sum
     cd ${webdir}/
@@ -464,12 +539,6 @@ function stats() {
     dak stats pkg-nums > $webdir/pkg-nums
 }
 
-function aptftpcleanup() {
-    log "Clean up apt-ftparchive's databases"
-    cd $configdir
-    apt-ftparchive -q clean apt.conf
-}
-
 function cleantransactions() {
     log "Cleanup transaction ids older than 3 months"
     cd $base/backup/
@@ -518,13 +587,6 @@ function process_unchecked() {
     sync_debbugs
 }
 
-# do a run of newstage only before dinstall is on.
-function newstage() {
-    log "Processing the newstage queue"
-    UNCHECKED_WITHOUT_LOCK="-p"
-    do_newstage
-}
-
 # Function to update a "statefile" telling people what we are doing
 # (more or less).
 #
@@ -542,12 +604,23 @@ EOF
 
 # extract changelogs and stuff
 function changelogs() {
-    log "Extracting changelogs"
-    dak make-changelog -e
-    mkdir -p ${exportpublic}/changelogs
-    cd ${exportpublic}/changelogs
-    rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
-    sudo -H -u archvsync /home/archvsync/runmirrors metaftpdo > ~dak/runmirrors-metadata.log 2>&1 &
+    if lockfile -r3 $LOCK_CHANGELOG; then
+        log "Extracting changelogs"
+        dak make-changelog -e -a ftp-master
+        [ -f ${exportdir}/changelogs/filelist.yaml ] && xz -f ${exportdir}/changelogs/filelist.yaml
+        mkdir -p ${exportpublic}/changelogs
+        cd ${exportpublic}/changelogs
+        rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
+        sudo -H -u staticsync /usr/local/bin/static-update-component metadata.ftp-master.debian.org >/dev/null 2>&1 &
+
+        dak make-changelog -e -a backports
+        [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml
+        mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs
+        cd /srv/backports-master.debian.org/rsync/export/changelogs
+        rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. .
+        remove_changelog_lock
+        trap remove_changelog_lock EXIT TERM HUP INT QUIT
+    fi
 }
 
 function gitpdiff() {