wanna-build/trigger.daily

   1 #!/bin/sh
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 # Files:
   7 #     Sources-* == upstream fetched file
   8 #     Sources.* == uncompressed, concat'd version
   9 PATH="/bin:/usr/bin"
  10 #testing must be before unstable so late upld don't build for testing needlessly
  11 DISTS="oldstable-security stable-security testing-security stable testing unstable"
  12 STATS_DISTS="unstable testing stable"
  13 SECTIONS="main contrib non-free"
  14 ARCHS_oldstable="m68k arm sparc alpha powerpc i386 mips mipsel ia64 hppa s390"
  15 ARCHS_stable="$ARCHS_oldstable"
  16 ARCHS_testing="$ARCHS_stable"
  17 ARCHS_unstable="$ARCHS_testing hurd-i386 sh"
  18 TMPDIR="/org/wanna-build/tmp"
  19 WGETOPT="-q -t2 -w0 -T10"
  20 CURLOPT="-q -s -S -f -y 5 -K /org/wanna-build/trigger.curlrc"
  21 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  22
  23 DAY=`date +%w`
  24
  25 if lockfile -! -l 3600 $LOCKFILE; then
  26         echo "Cannot lock $LOCKFILE"
  27         exit 1
  28 fi
  29
  30 cleanup() {
  31         rm -f "$LOCKFILE"
  32 }
  33 trap cleanup 0
  34
  35 echo Updating wanna-build databases...
  36 umask 027
  37
  38 if [ "$DAY" = "0" ]; then
  39         savelog -c 26 -p /org/wanna-build/db/merge.log
  40 fi
  41
  42 exec >> /org/wanna-build/db/merge.log 2>&1
  43
  44 echo -------------------------------------------------------------------------
  45 echo "merge triggered `date`"
  46
  47 cd $TMPDIR
  48
  49 #
  50 # Make one big Packages and Sources file.
  51 #
  52 for d in $DISTS ; do
  53         dist=`echo $d | sed s/-.*$//`
  54         case "$dist" in
  55                 oldstable)
  56                         ARCHS="$ARCHS_oldstable"
  57                         ;;
  58                 stable)
  59                         ARCHS="$ARCHS_stable"
  60                         ;;
  61                 testing)
  62                         ARCHS="$ARCHS_testing"
  63                         ;;
  64                 *)
  65                         ARCHS="$ARCHS_unstable"
  66                         ;;
  67         esac
  68         NON_US="false" # this is changed below if sources can be found
  69         rm -f Sources.$d
  70         if [ "$d" = "unstable" ]; then
  71                 gzip -dc /org/incoming.debian.org/buildd/Sources.gz >> Sources.$d
  72         fi
  73         for a in $ARCHS ; do
  74                 rm -f Packages.$d.$a quinn-$d.$a
  75                 if [ "$d" = "unstable" ]; then
  76                         gzip -dc /org/incoming.debian.org/buildd/Packages.gz >> Packages.$d.$a
  77                 fi
  78         done
  79
  80         for s in $SECTIONS ; do
  81                 if echo $d | grep -qv -- -security; then
  82                         rm -f Sources.gz
  83                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/source/Sources.gz; then
  84                                 mv Sources.gz Sources-non-us.$d.$s.gz
  85                                 NON_US="true"
  86                         fi
  87                         gzip -dc Sources-non-us.$d.$s.gz >> Sources.$d
  88                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/source/Sources.gz >> Sources.$d
  89                         if [ "$d" = "testing" -o "$d" = "stable" ]; then
  90                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/source/Sources.gz; then
  91                                         mv Sources.gz Sources-non-us.$d-p-u.$s.gz
  92                                 fi
  93                                 gzip -dc Sources-non-us.$d-p-u.$s.gz >> Sources.$d
  94                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/source/Sources.gz >> Sources.$d
  95                         fi
  96
  97                         rm -f Packages.gz
  98                         for a in $ARCHS ; do
  99                                 if [ "$NON_US" = "true" ]; then
 100                                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/binary-$a/Packages.gz; then
 101                                                 mv Packages.gz Packages-non-us.$d.$s.$a.gz
 102                                         fi
 103                                 fi
 104                                 gzip -dc Packages-non-us.$d.$s.$a.gz >> Packages.$d.$a
 105
 106                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/binary-$a/Packages.gz >> Packages.$d.$a
 107                                 if [ "$d" = "testing" -o "$d" = "stable" ]; then
 108                                         if [ "$NON_US" = "true" ]; then
 109                                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/binary-$a/Packages.gz; then
 110                                                         mv Packages.gz Packages-non-us.$d-p-u.$s.$a.gz
 111                                                 fi
 112                                         fi
 113                                         gzip -dc Packages-non-us.$d-p-u.$s.$a.gz >> Packages.$d.$a
 114                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/binary-$a/Packages.gz >> Packages.$d.$a
 115                                 fi
 116                                 if [ "$d" = "unstable" -a "$s" = "main" ]; then
 117                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/debian-installer/binary-$a/Packages.gz >> Packages.$d.$a
 118                                 fi
 119                         done
 120                 else
 121                         rm -f Sources.gz
 122                         if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/source/Sources.gz; then
 123                                 mv Sources.gz Sources-$d.$s.gz
 124                         fi
 125                         gzip -dc Sources-$d.$s.gz >> Sources.$d
 126                         if [ "$s" = "main" ]; then
 127                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Sources.gz -o Sources.gz; then
 128                                         mv Sources.gz Sources-$d.accepted.gz
 129                                 fi
 130                                 gzip -dc Sources-$d.accepted.gz >> Sources.$d
 131                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Packages.gz -o Packages.gz; then
 132                                         mv Packages.gz Packages.$d.accepted.gz
 133                                 fi
 134                         fi
 135                         rm -f Packages.gz
 136                         for a in $ARCHS ; do
 137                                 if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/binary-$a/Packages.gz; then
 138                                         mv Packages.gz Packages.$d.$s.$a.gz
 139                                 fi
 140                                 gzip -dc Packages.$d.$s.$a.gz >> Packages.$d.$a
 141                                 if [ "$s" = "main" ]; then
 142                                         gzip -dc Packages.$d.accepted.gz >> Packages.$d.$a
 143                                 fi
 144                         done
 145                 fi
 146         done
 147
 148         for a in $ARCHS ; do
 149                 if [ "$d" = "unstable" -o ! -e "quinn-unstable.$a-old" ]; then
 150                         quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 151                 else
 152                         if echo $d | grep -qv -- -security; then
 153                                 quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a | fgrep -v -f quinn-unstable.$a-old | grep ":out-of-date\]$" >> quinn-$d.$a
 154                         else
 155                                 quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 156                         fi
 157                 fi
 158         done
 159 done
 160
 161 umask 002
 162 for a in $ARCHS_unstable ; do
 163         wanna-build --create-maintenance-lock --database=$a/build-db
 164
 165         for d in $DISTS ; do
 166                 dist=`echo $d | sed s/-.*$//`
 167                 case "$dist" in
 168                         oldstable)
 169                                 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
 170                                         continue
 171                                 fi
 172                                 ;;
 173                         stable)
 174                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 175                                         continue
 176                                 fi
 177                                 ;;
 178                         testing)
 179                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 180                                         continue
 181                                 fi
 182                                 ;;
 183                         *)
 184                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 185                                         continue
 186                                 fi
 187                                 ;;
 188                 esac
 189                 wanna-build --merge-packages --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a
 190                 perl -pi -e 's#^(non-US/)?(non-free)/.*$##msg' quinn-$d.$a
 191                 wanna-build --merge-quinn --dist=$d --database=$a/build-db quinn-$d.$a
 192                 wanna-build --merge-sources --dist=$d --database=$a/build-db Sources.$d
 193                 wanna-build --clean-db --dist=$d --database=$a/build-db
 194                 mv Packages.$d.$a Packages.$d.$a-old
 195                 mv quinn-$d.$a quinn-$d.$a-old
 196         done
 197         if [ "$DAY" = "0" ]; then
 198                 savelog -p -c 26 /org/wanna-build/db/$a/transactions.log
 199         fi
 200         wanna-build --remove-maintenance-lock --database=$a/build-db
 201 done
 202 umask 022
 203 for d in $DISTS; do
 204         mv Sources.$d Sources.$d-old
 205 done
 206
 207 echo "merge ended `date`"
 208 /org/wanna-build/bin/wb-graph >> /org/wanna-build/etc/graph-data
 209 /org/wanna-build/bin/wb-graph -p >> /org/wanna-build/etc/graph2-data
 210 rm -f "$LOCKFILE"
 211 trap -
 212 /org/buildd.debian.org/bin/makegraph
 213 for a in $ARCHS_stable; do
 214         echo Last Updated: `date -u` > /org/buildd.debian.org/web/stats/$a.txt
 215         for d in $STATS_DISTS; do
 216                 /org/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /org/buildd.debian.org/web/stats/$a.txt
 217         done
 218 done