dep11 people not knowing their own layout poooooooooor dep11

[dak.git] / config / debian / dinstall.functions

diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index 47fa563a59f89aee42b703b6b8b880aee6c3220f..0e5787bd8b42dbbd527c910cb6ad0a35ca824c4b 100644 (file)
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -35,7 +35,7 @@ function onerror() {
         cat "${STAGEFILE}.log"
     else
         echo "file ${STAGEFILE}.log does not exist, sorry"
         cat "${STAGEFILE}.log"
     else
         echo "file ${STAGEFILE}.log does not exist, sorry"

-    fi | mail -s "${subject}" -a "X-Debian: DAK" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
+    fi | mail -s "${subject}" -a "X-Debian: DAK" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org

 }
 
 ########################################################################
 }
 
 ########################################################################


@@ -76,7 +76,7 @@ function i18n1() {
             # Lets check!
             if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                 # Yay, worked, lets copy around
             # Lets check!
             if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then
                 # Yay, worked, lets copy around

-                for dir in jessie sid; do
+                for dir in stretch sid; do

                     if [ -d dists/${dir}/ ]; then
                         cd dists/${dir}/main/i18n
                         rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.
                     if [ -d dists/${dir}/ ]; then
                         cd dists/${dir}/main/i18n
                         rsync -aq --delete --delete-after --exclude Translation-en.bz2 --exclude Translation-*.diff/ . ${ftpdir}/dists/${dir}/main/i18n/.


@@ -85,15 +85,42 @@ function i18n1() {
                 done
             else
                 echo "ARRRR, bad guys, wrong files, ARRR"
                 done
             else
                 echo "ARRRR, bad guys, wrong files, ARRR"

-                echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
+                echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org

             fi
         else
             echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"
             fi
         else
             echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR"

-            echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
+            echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org

         fi
     else
         echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
         fi
     else
         echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."

-        echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
+        echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -a "X-Debian: DAK" -s "You can't keep blaming yourself. Just blame yourself once, and move on." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" debian-l10n-devel@lists.alioth.debian.org
+    fi
+}
+
+# Syncing AppStream/DEP-11 data
+function dep11() {
+    log "Synchronizing AppStream metadata"
+    # First sync their newest data
+    mkdir -p ${scriptdir}/dep11
+    cd ${scriptdir}/dep11
+    rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true
+
+    # Lets check!
+    if ${scriptsdir}/dep11-basic-validate.py . ${scriptdir}/dep11/; then
+        # Yay, worked, lets copy around
+        for dir in stretch sid; do
+            if [ -d ${dir}/ ]; then
+                for comp in main contrib non-free; do
+                    mkdir -p ${ftpdir}/dists/${dir}/${comp}/dep11
+                    cd ${dir}/${comp}
+                    rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/.
+                    cd ${scriptdir}/dep11
+                done
+            fi
+        done
+    else
+        echo "ARRRR, bad guys, wrong files, ARRR"
+        echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" mak@debian.org

     fi
 }
 
     fi
 }
 


@@ -107,9 +134,10 @@ function dominate() {
     dak dominate
 }
 
     dak dominate
 }
 

-function filelist() {
-    log "Generating file lists for apt-ftparchive"
-    dak generate-filelist
+function autocruft() {
+    log "Check for obsolete binary packages"
+    dak auto-decruft -s unstable
+    dak auto-decruft -s experimental --if-newer-version-in unstable --if-newer-version-in-rm-msg "NVIU"

 }
 
 function fingerprints() {
 }
 
 function fingerprints() {


@@ -144,10 +172,6 @@ function overrides() {
     log "Writing overrides into text files"
     cd $overridedir
     dak make-overrides
     log "Writing overrides into text files"
     cd $overridedir
     dak make-overrides

-
-    # FIXME
-    rm -f override.sid.all3
-    for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done

 }
 
 function mpfm() {
 }
 
 function mpfm() {


@@ -163,7 +187,9 @@ function mpfm() {
 function packages() {
     log "Generating Packages and Sources files"
     for archive in "${public_archives[@]}"; do
 function packages() {
     log "Generating Packages and Sources files"
     for archive in "${public_archives[@]}"; do

+        log "  Generating Packages/Sources for ${archive}"

         dak generate-packages-sources2 -a "${archive}"
         dak generate-packages-sources2 -a "${archive}"

+        log "  Generating Contents for ${archive}"

         dak contents generate -a "${archive}"
     done
 }
         dak contents generate -a "${archive}"
     done
 }


@@ -174,15 +200,6 @@ function pdiff() {
 }
 
 function release() {
 }
 
 function release() {

-    # XXX: disable once we can remove i18n/Index (#649314)
-    log "Generating i18n/Index"
-    (
-        cd "$ftpdir/dists";
-        for dist in testing unstable experimental proposed-updates testing-proposed-updates; do
-            $scriptsdir/generate-i18n-Index $dist;
-        done
-    )
-

     log "Generating Release files"
     for archive in "${public_archives[@]}"; do
         dak generate-releases -a "${archive}"
     log "Generating Release files"
     for archive in "${public_archives[@]}"; do
         dak generate-releases -a "${archive}"


@@ -195,13 +212,6 @@ function dakcleanup() {
     dak clean-queues -i "$unchecked"
 }
 
     dak clean-queues -i "$unchecked"
 }
 

-function buildd_dir() {
-    # Rebuilt the buildd dir to avoid long times of 403
-    log "Regenerating the buildd incoming dir"
-    STAMP=$(date "+%Y%m%d%H%M")
-    make_buildd_dir
-}
-

 function mklslar() {
     local archiveroot
     local FILENAME=ls-lR
 function mklslar() {
     local archiveroot
     local FILENAME=ls-lR


@@ -245,11 +255,14 @@ function mkmaintainers() {
 function copyoverrides() {
     log 'Copying override files into public view ...'
 
 function copyoverrides() {
     log 'Copying override files into public view ...'
 

-    for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,sid}.{,extra.}{main,contrib,non-free}*; do
-        bname=${ofile##*/}
-        gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
-        chmod g+w ${indices}/${bname}.gz
-    done
+    (
+        shopt -s nullglob
+        for ofile in ${overridedir}/override.{squeeze,wheezy,jessie,stretch,sid}.{,extra.}{main,contrib,non-free}*; do
+            bname=${ofile##*/}
+            gzip -9cv --rsyncable ${ofile} > ${indices}/${bname}.gz
+            chmod g+w ${indices}/${bname}.gz
+        done
+    )

 }
 
 function mkfilesindices() {
 }
 
 function mkfilesindices() {


@@ -376,12 +389,12 @@ function mkfilesindices() {
     done
 
     (cd $base/ftp/
     done
 
     (cd $base/ftp/

-           for dist in sid jessie; do
+           for dist in sid jessie stretch; do

                    find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
            done
     )
 
                    find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
            done
     )
 

-    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz) |
+    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz) |

     sort -u | poolfirst > ../typical.files
 
     rm -f $ARCHLIST
     sort -u | poolfirst > ../typical.files
 
     rm -f $ARCHLIST


@@ -407,7 +420,7 @@ function mkchecksums() {
 }
 
 function mirror() {
 }
 
 function mirror() {

-    local archiveroot
+    local archiveroot mirrordir

 
     log "Regenerating \"public\" mirror/ hardlink fun"
     DATE_SERIAL=$(date +"%Y%m%d01")
 
     log "Regenerating \"public\" mirror/ hardlink fun"
     DATE_SERIAL=$(date +"%Y%m%d01")


@@ -452,13 +465,13 @@ function dm() {
 
 function bts() {
     log "Categorizing uncategorized bugs filed against ftp.debian.org"
 
 function bts() {
     log "Categorizing uncategorized bugs filed against ftp.debian.org"

-    dak bts-categorize
+    sudo -u dak-unpriv dak bts-categorize

 }
 
 function ddaccess() {
     # Tell our dd accessible mirror to sync itself up. Including ftp dir.
     log "Trigger dd accessible parts sync including ftp dir"
 }
 
 function ddaccess() {
     # Tell our dd accessible mirror to sync itself up. Including ftp dir.
     log "Trigger dd accessible parts sync including ftp dir"

-    ${scriptsdir}/sync-dd ries-sync ries-sync1 ries-sync2 pool
+    ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync

 }
 
 function mirrorpush() {
 }
 
 function mirrorpush() {


@@ -472,7 +485,7 @@ function mirrorpush() {
         while read SHASUM SIZE NAME; do
             if ! [ -f "${subdir}/${NAME}" ]; then
                bname=$(basename ${NAME})
         while read SHASUM SIZE NAME; do
             if ! [ -f "${subdir}/${NAME}" ]; then
                bname=$(basename ${NAME})

-                if [[ "${bname}" =~ ^(Packages|Sources|Translation-[a-zA-Z_]+)$ ]]; then
+                if [[ "${bname}" =~ ^(Packages|Sources|Contents-[a-zA-Z0-9-]+|Translation-[a-zA-Z_]+)$ ]]; then

                     # We don't keep unpacked files, don't check for their existance.
                     # We might want to go and check their unpacked shasum, but right now
                     # I don't care. I believe it should be enough if all the packed shasums
                     # We don't keep unpacked files, don't check for their existance.
                     # We might want to go and check their unpacked shasum, but right now
                     # I don't care. I believe it should be enough if all the packed shasums


@@ -517,6 +530,11 @@ function mirrorpush() {
     sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 &
 }
 
     sudo -H -u archvsync /home/archvsync/runmirrors -a backports > ~dak/runmirrorsbpo.log 2>&1 &
 }
 

+function mirrorpush-backports() {
+    log "Syncing backports mirror"
+    sudo -u backports /home/backports/bin/update-archive
+}
+

 function i18n2() {
     log "Exporting package data foo for i18n project"
     STAMP=$(date "+%Y%m%d%H%M")
 function i18n2() {
     log "Exporting package data foo for i18n project"
     STAMP=$(date "+%Y%m%d%H%M")


@@ -551,7 +569,7 @@ function stats() {
 function cleantransactions() {
     log "Cleanup transaction ids older than 3 months"
     cd $base/backup/
 function cleantransactions() {
     log "Cleanup transaction ids older than 3 months"
     cd $base/backup/

-    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
+    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -delete

 }
 
 function logstats() {
 }
 
 function logstats() {


@@ -565,7 +583,7 @@ function savetimestamp() {
 }
 
 function maillogfile() {
 }
 
 function maillogfile() {

-    cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
+    cat "$LOGFILE" | mail -a "X-Debian: DAK" -s "Log for dinstall run of ${NOW}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" cron@ftp-master.debian.org

 }
 
 function renamelogfile() {
 }
 
 function renamelogfile() {


@@ -620,13 +638,14 @@ function changelogs() {
         mkdir -p ${exportpublic}/changelogs
         cd ${exportpublic}/changelogs
         rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .
         mkdir -p ${exportpublic}/changelogs
         cd ${exportpublic}/changelogs
         rsync -aHW --delete --delete-after --ignore-errors ${exportdir}/changelogs/. .

-        sudo -H -u staticsync /usr/local/bin/static-update-component ftp-master.metadata.debian.org >/dev/null 2>&1 &
+        sudo -H -u staticsync /usr/local/bin/static-update-component metadata.ftp-master.debian.org >/dev/null 2>&1 &

 
         dak make-changelog -e -a backports
         [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml
         mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs
         cd /srv/backports-master.debian.org/rsync/export/changelogs
         rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. .
 
         dak make-changelog -e -a backports
         [ -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml ] && xz -f /srv/backports-master.debian.org/export/changelogs/filelist.yaml
         mkdir -p /srv/backports-master.debian.org/rsync/export/changelogs
         cd /srv/backports-master.debian.org/rsync/export/changelogs
         rsync -aHW --delete --delete-after --ignore-errors /srv/backports-master.debian.org/export/changelogs/. .

+        remove_changelog_lock

         trap remove_changelog_lock EXIT TERM HUP INT QUIT
     fi
 }
         trap remove_changelog_lock EXIT TERM HUP INT QUIT
     fi
 }