wanna-build/trigger.daily

   1 #!/bin/sh
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 PATH="/bin:/usr/bin"
   7 DISTS="unstable oldstable-security stable stable-security testing testing-security"
   8 SECTIONS="main contrib non-free"
   9 ARCHS_oldstable="m68k arm sparc alpha powerpc i386"
  10 ARCHS_stable="mips mipsel ia64 hppa s390 $ARCHS_oldstable"
  11 ARCHS_testing="$ARCHS_stable"
  12 ARCHS_unstable="$ARCHS_testing hurd-i386 sh"
  13 TMPDIR="/org/wanna-build/tmp"
  14 WGETOPT="-q -t2 -w0 -T60"
  15 CURLOPT="-q -s -S -f -K /org/wanna-build/trigger.curlrc"
  16 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  17
  18 DAY=`date +%w`
  19
  20 if lockfile -! -l 3600 $LOCKFILE; then
  21         echo "Cannot lock $LOCKFILE"
  22         exit 1
  23 fi
  24
  25 cleanup() {
  26         rm -f "$LOCKFILE"
  27 }
  28 trap cleanup 0
  29
  30 echo Updating wanna-build databases...
  31 umask 027
  32
  33 if [ "$DAY" = "0" ]; then
  34         savelog -c 26 -p /org/wanna-build/db/merge.log
  35 fi
  36
  37 exec >> /org/wanna-build/db/merge.log 2>&1
  38
  39 echo -------------------------------------------------------------------------
  40 echo "merge triggered `date`"
  41
  42 cd $TMPDIR
  43
  44 #
  45 # Make one big Packages and Sources file.
  46 #
  47 for d in $DISTS ; do
  48         dist=`echo $d | sed s/-.*$//`
  49         case "$dist" in
  50                 oldstable)
  51                         ARCHS="$ARCHS_oldstable"
  52                         ;;
  53                 stable)
  54                         ARCHS="$ARCHS_stable"
  55                         ;;
  56                 testing)
  57                         ARCHS="$ARCHS_testing"
  58                         ;;
  59                 *)
  60                         ARCHS="$ARCHS_unstable"
  61                         ;;
  62         esac
  63         NON_US="false" # this is changed below if sources can be found
  64         rm -f Sources.$d
  65         if [ "$d" = "unstable" ]; then
  66                 gzip -dc /org/incoming.debian.org/buildd/Sources.gz >> Sources.$d
  67         fi
  68         for a in $ARCHS ; do
  69                 rm -f Packages.$d.$a quinn-$d.$a
  70                 if [ "$d" = "unstable" ]; then
  71                         gzip -dc /org/incoming.debian.org/buildd/Packages.gz >> Packages.$d.$a
  72                 fi
  73         done
  74
  75         for s in $SECTIONS ; do
  76                 if echo $d | grep -qv -- -security; then
  77                         rm -f Sources.gz
  78                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/source/Sources.gz; then
  79                                 mv Sources.gz Sources-non-us.$d.$s.gz
  80                                 NON_US="true"
  81                         fi
  82                         gzip -dc Sources-non-us.$d.$s.gz >> Sources.$d
  83                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/source/Sources.gz >> Sources.$d
  84                         if [ "$d" = "testing" -o "$d" = "stable" ]; then
  85                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/source/Sources.gz; then
  86                                         mv Sources.gz Sources-non-us.$d-p-u.$s.gz
  87                                 fi
  88                                 gzip -dc Sources-non-us.$d-p-u.$s.gz >> Sources.$d
  89                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/source/Sources.gz >> Sources.$d
  90                         fi
  91
  92                         rm -f Packages.gz
  93                         for a in $ARCHS ; do
  94                                 if [ "$NON_US" = "true" ]; then
  95                                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/binary-$a/Packages.gz; then
  96                                                 mv Packages.gz Packages-non-us.$d.$s.$a.gz
  97                                         fi
  98                                 fi
  99                                 gzip -dc Packages-non-us.$d.$s.$a.gz >> Packages.$d.$a
 100
 101                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/binary-$a/Packages.gz >> Packages.$d.$a
 102                                 if [ "$d" = "testing" -o "$d" = "stable" ]; then
 103                                         if [ "$NON_US" = "true" ]; then
 104                                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/binary-$a/Packages.gz; then
 105                                                         mv Packages.gz Packages-non-us.$d-p-u.$s.$a.gz
 106                                                 fi
 107                                         fi
 108                                         gzip -dc Packages-non-us.$d-p-u.$s.$a.gz >> Packages.$d.$a
 109                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/binary-$a/Packages.gz >> Packages.$d.$a
 110                                 fi
 111                                 if [ "$d" = "unstable" -a "$s" = "main" ]; then
 112                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/debian-installer/binary-$a/Packages.gz >> Packages.$d.$a
 113                                 fi
 114                         done
 115                 else
 116                         rm -f Sources.gz
 117                         if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/source/Sources.gz; then
 118                                 mv Sources.gz Sources-$d.$s.gz
 119                         fi
 120                         gzip -dc Sources-$d.$s.gz >> Sources.$d
 121                         if [ "$s" = "main" ]; then
 122                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Sources.gz -o Sources.gz; then
 123                                         mv Sources.gz Sources-$d.accepted.gz
 124                                 fi
 125                                 gzip -dc Sources-$d.accepted.gz >> Sources.$d
 126                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Packages.gz -o Packages.gz; then
 127                                         mv Packages.gz Packages.$d.accepted.gz
 128                                 fi
 129                         fi
 130                         rm -f Packages.gz
 131                         for a in $ARCHS ; do
 132                                 if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/binary-$a/Packages.gz; then
 133                                         mv Packages.gz Packages.$d.$s.$a.gz
 134                                 fi
 135                                 gzip -dc Packages.$d.$s.$a.gz >> Packages.$d.$a
 136                                 if [ "$s" = "main" ]; then
 137                                         gzip -dc Packages.$d.accepted.gz >> Packages.$d.$a
 138                                 fi
 139                         done
 140                 fi
 141         done
 142
 143         if [ "$d" = "oldstable" ]; then
 144                 gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Sources.gz >> Sources.$d
 145                 if "$NON_US" = "true" ]; then
 146                         rm -f Sources.gz
 147                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Sources.gz; then
 148                                 mv Sources.gz Sources-non-us.p-u.gz
 149                         fi
 150                         gzip -dc Sources-non-us.p-u.gz >> Sources.$d
 151                         rm -f Packages.gz
 152                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Packages.gz; then
 153                                 mv Packages.gz Packages-non-us.p-u.gz
 154                         fi
 155                 fi
 156
 157                 for a in $ARCHS ; do
 158                         gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Packages.gz >> Packages.$d.$a
 159                         gzip -dc Packages-non-us.p-u.gz >> Packages.$d.$a
 160                 done
 161         fi
 162         for a in $ARCHS ; do
 163                 if [ "$d" = "unstable" ]; then
 164                         quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 165                 else
 166                         quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a | fgrep -v -f quinn-unstable.$a >> quinn-$d.$a
 167                 fi
 168         done
 169 done
 170
 171 umask 002
 172 for a in $ARCHS_unstable ; do
 173         wanna-build --create-maintenance-lock --database=$a/build-db
 174
 175         for d in $DISTS ; do
 176                 dist=`echo $d | sed s/-.*$//`
 177                 case "$dist" in
 178                         oldstable)
 179                                 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
 180                                         continue
 181                                 fi
 182                                 ;;
 183                         stable)
 184                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 185                                         continue
 186                                 fi
 187                                 ;;
 188                         testing)
 189                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 190                                         continue
 191                                 fi
 192                                 ;;
 193                         *)
 194                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 195                                         continue
 196                                 fi
 197                                 ;;
 198                 esac
 199                 wanna-build --merge-packages --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a
 200                 perl -pi -e 's#^(non-US/)?(non-free)/.*$##msg' quinn-$d.$a
 201                 wanna-build --merge-quinn --dist=$d --database=$a/build-db quinn-$d.$a
 202                 wanna-build --merge-sources --dist=$d --database=$a/build-db Sources.$d
 203                 mv Packages.$d.$a Packages.$d.$a-old
 204                 mv quinn-$d.$a quinn-$d.$a-old
 205                 if [ "$DAY" = "0" -a "$d" = "unstable" ]; then
 206                         savelog -p -c 26 /org/wanna-build/db/$a/transactions.log
 207                 fi
 208         done
 209         wanna-build --remove-maintenance-lock --database=$a/build-db
 210 done
 211 umask 022
 212 for d in $DISTS; do
 213         mv Sources.$d Sources.$d-old
 214 done
 215
 216 echo "merge ended `date`"
 217 /org/wanna-build/bin/wb-graph >> /org/wanna-build/etc/graph-data
 218 /org/wanna-build/bin/wb-graph -p >> /org/wanna-build/etc/graph2-data
 219 rm -f "$LOCKFILE"
 220 trap -
 221 /org/buildd.debian.org/bin/makegraph