wanna-build/trigger.daily

   1 #!/bin/sh
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 PATH="/bin:/usr/bin"
   7 DISTS="oldstable-security stable stable-security testing testing-security unstable"
   8 SECTIONS="main contrib non-free"
   9 ARCHS_oldstable="m68k arm sparc alpha powerpc i386"
  10 ARCHS_stable="mips mipsel ia64 hppa s390 $ARCHS_oldstable"
  11 ARCHS_testing="$ARCHS_stable"
  12 ARCHS_unstable="$ARCHS_testing hurd-i386 sh"
  13 TMPDIR="/org/wanna-build/tmp"
  14 WGETOPT="-q -t2 -w0 -T60"
  15 CURLOPT="-q -s -S -f -K /org/wanna-build/trigger.curlrc"
  16 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  17
  18 DAY=`date +%w`
  19
  20 echo Updating wanna-build databases...
  21 umask 022
  22
  23 if [ "$DAY" = "0" ]; then
  24         savelog -c 26 -p /org/wanna-build/db/merge.log
  25 fi
  26
  27 exec >> /org/wanna-build/db/merge.log 2>&1
  28 lockfile -l 3600 $LOCKFILE
  29 cleanup() {
  30         rm -f "$LOCKFILE"
  31 }
  32 trap cleanup 0
  33
  34 echo -------------------------------------------------------------------------
  35 echo "merge triggered `date`"
  36
  37 cd $TMPDIR
  38
  39 #
  40 # Make one big Packages and Sources file.
  41 #
  42 for d in $DISTS ; do
  43         dist=`echo $d | sed s/-.*$//`
  44         case "$dist" in
  45                 oldstable)
  46                         ARCHS="$ARCHS_oldstable"
  47                         ;;
  48                 stable)
  49                         ARCHS="$ARCHS_stable"
  50                         ;;
  51                 testing)
  52                         ARCHS="$ARCHS_testing"
  53                         ;;
  54                 *)
  55                         ARCHS="$ARCHS_unstable"
  56                         ;;
  57         esac
  58         NON_US="false" # this is changed below if sources can be found
  59         rm -f Sources.$d
  60         if [ "$d" = "unstable" ]; then
  61                 gzip -dc /org/incoming.debian.org/buildd/Sources.gz >> Sources.$d
  62         fi
  63         for a in $ARCHS ; do
  64                 rm -f Packages.$d.$a quinn-$d.$a
  65                 if [ "$d" = "unstable" ]; then
  66                         gzip -dc /org/incoming.debian.org/buildd/Packages.gz >> Packages.$d.$a
  67                 fi
  68         done
  69
  70
  71         for s in $SECTIONS ; do
  72                 if echo $d | grep -qv -- -security; then
  73                         rm -f Sources.gz
  74                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/source/Sources.gz; then
  75                                 mv Sources.gz Sources-non-us.$d.$s.gz
  76                                 NON_US="true"
  77                         fi
  78                         gzip -dc Sources-non-us.$d.$s.gz >> Sources.$d
  79                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/source/Sources.gz >> Sources.$d
  80                         if [ "$d" = "testing" -o "$d" = "stable" ]; then
  81                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/source/Sources.gz; then
  82                                         mv Sources.gz Sources-non-us.$d-p-u.$s.gz
  83                                 fi
  84                                 gzip -dc Sources-non-us.$d-p-u.$s.gz >> Sources.$d
  85                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/source/Sources.gz >> Sources.$d
  86                         fi
  87
  88                         rm -f Packages.gz
  89                         for a in $ARCHS ; do
  90                                 if [ "$NON_US" = "true" ]; then
  91                                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/binary-$a/Packages.gz; then
  92                                                 mv Packages.gz Packages-non-us.$d.$s.$a.gz
  93                                         fi
  94                                 fi
  95                                 gzip -dc Packages-non-us.$d.$s.$a.gz >> Packages.$d.$a
  96
  97                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/binary-$a/Packages.gz >> Packages.$d.$a
  98                                 if [ "$d" = "testing" -o "$d" = "stable" ]; then
  99                                         if [ "$NON_US" = "true" ]; then
 100                                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/binary-$a/Packages.gz; then
 101                                                         mv Packages.gz Packages-non-us.$d-p-u.$s.$a.gz
 102                                                 fi
 103                                         fi
 104                                         gzip -dc Packages-non-us.$d-p-u.$s.$a.gz >> Packages.$d.$a
 105                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/binary-$a/Packages.gz >> Packages.$d.$a
 106                                 fi
 107                                 if [ "$d" = "unstable" -a "$s" = "main" ]; then
 108                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/debian-installer/binary-$a/Packages.gz >> Packages.$d.$a
 109                                 fi
 110                         done
 111                 else
 112                         rm -f Sources.gz
 113                         if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/source/Sources.gz; then
 114                                 mv Sources.gz Sources-$d.$s.gz
 115                         fi
 116                         gzip -dc Sources-$d.$s.gz >> Sources.$d
 117                         if [ "$s" = "main" ]; then
 118                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Sources.gz -o Sources.gz; then
 119                                         mv Sources.gz Sources-$d.accepted.gz
 120                                 fi
 121                                 gzip -dc Sources-$d.accepted.gz >> Sources.$d
 122                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Packages.gz -o Packages.gz; then
 123                                         mv Packages.gz Packages-$d.accepted.gz
 124                                 fi
 125                         fi
 126                         rm -f Packages.gz
 127                         for a in $ARCHS ; do
 128                                 if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/binary-$a/Packages.gz; then
 129                                         mv Packages.gz Packages.$d.$s.$a.gz
 130                                 fi
 131                                 gzip -dc Packages.$d.$s.$a.gz >> Packages.$d.$a
 132                                 if [ "$s" = "main" ]; then
 133                                         gzip -dc Packages.$d.accepted.gz >> Packages.$d.$a
 134                                 fi
 135                         done
 136                 fi
 137         done
 138
 139         if [ "$d" = "oldstable" ]; then
 140                 gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Sources.gz >> Sources.$d
 141                 if "$NON_US" = "true" ]; then
 142                         rm -f Sources.gz
 143                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Sources.gz; then
 144                                 mv Sources.gz Sources-non-us.p-u.gz
 145                         fi
 146                         gzip -dc Sources-non-us.p-u.gz >> Sources.$d
 147                         rm -f Packages.gz
 148                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Packages.gz; then
 149                                 mv Packages.gz Packages-non-us.p-u.gz
 150                         fi
 151                 fi
 152
 153                 for a in $ARCHS ; do
 154                         gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Packages.gz >> Packages.$d.$a
 155                         gzip -dc Packages-non-us.p-u.gz >> Packages.$d.$a
 156                 done
 157         fi
 158         for a in $ARCHS ; do
 159                 quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 160         done
 161 done
 162
 163 umask 002
 164 for a in $ARCHS_unstable ; do
 165         wanna-build --create-maintenance-lock --database=$a/build-db
 166
 167         for d in $DISTS ; do
 168                 dist=`echo $d | sed s/-.*$//`
 169                 case "$dist" in
 170                         stable)
 171                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 172                                         continue
 173                                 fi
 174                                 ;;
 175                         testing)
 176                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 177                                         continue
 178                                 fi
 179                                 ;;
 180                         *)
 181                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 182                                         continue
 183                                 fi
 184                                 ;;
 185                 esac
 186                 wanna-build --merge-packages --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a
 187                 perl -pi -e 's#^(non-US/)?(non-free)/.*$##msg' quinn-$d.$a
 188                 wanna-build --merge-quinn --dist=$d --database=$a/build-db quinn-$d.$a
 189                 wanna-build --merge-sources --dist=$d --database=$a/build-db Sources.$d
 190                 mv Packages.$d.$a Packages.$d.$a-old
 191                 mv quinn-$d.$a quinn-$d.$a-old
 192                 if [ "$DAY" = "0" -a "$d" = "unstable" ]; then
 193                         savelog -p -c 26 /org/wanna-build/db/$a/transactions.log
 194                 fi
 195         done
 196         wanna-build --remove-maintenance-lock --database=$a/build-db
 197 done
 198 umask 022
 199 for d in $DISTS; do
 200         mv Sources.$d Sources.$d-old
 201 done
 202
 203 echo "merge ended `date`"
 204 rm -f "$LOCKFILE"
 205 trap -
 206 /org/wanna-build/bin/wb-graph >> /org/wanna-build/etc/graph-data
 207 /org/wanna-build/bin/wb-graph -p >> /org/wanna-build/etc/graph2-data
 208 /org/buildd.debian.org/bin/makegraph