wanna-build/trigger.daily

   1 #!/bin/sh
   2 #
   3 # Updates wanna-build databases after the archive maintenance
   4 # finishes
   5 #
   6 # Files:
   7 #     Sources-* == upstream fetched file
   8 #     Sources.* == uncompressed, concat'd version
   9 PATH="/bin:/usr/bin"
  10 #testing must be before unstable so late upld don't build for testing needlessly
  11 DISTS="oldstable-security stable stable-security testing testing-security unstable"
  12 SECTIONS="main contrib non-free"
  13 ARCHS_oldstable="m68k arm sparc alpha powerpc i386"
  14 ARCHS_stable="mips mipsel ia64 hppa s390 $ARCHS_oldstable"
  15 ARCHS_testing="$ARCHS_stable"
  16 ARCHS_unstable="$ARCHS_testing hurd-i386 sh"
  17 TMPDIR="/org/wanna-build/tmp"
  18 WGETOPT="-q -t2 -w0 -T60"
  19 CURLOPT="-q -s -S -f -K /org/wanna-build/trigger.curlrc"
  20 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
  21
  22 DAY=`date +%w`
  23
  24 if lockfile -! -l 3600 $LOCKFILE; then
  25         echo "Cannot lock $LOCKFILE"
  26         exit 1
  27 fi
  28
  29 cleanup() {
  30         rm -f "$LOCKFILE"
  31 }
  32 trap cleanup 0
  33
  34 echo Updating wanna-build databases...
  35 umask 027
  36
  37 if [ "$DAY" = "0" ]; then
  38         savelog -c 26 -p /org/wanna-build/db/merge.log
  39 fi
  40
  41 exec >> /org/wanna-build/db/merge.log 2>&1
  42
  43 echo -------------------------------------------------------------------------
  44 echo "merge triggered `date`"
  45
  46 cd $TMPDIR
  47
  48 #
  49 # Make one big Packages and Sources file.
  50 #
  51 for d in $DISTS ; do
  52         dist=`echo $d | sed s/-.*$//`
  53         case "$dist" in
  54                 oldstable)
  55                         ARCHS="$ARCHS_oldstable"
  56                         ;;
  57                 stable)
  58                         ARCHS="$ARCHS_stable"
  59                         ;;
  60                 testing)
  61                         ARCHS="$ARCHS_testing"
  62                         ;;
  63                 *)
  64                         ARCHS="$ARCHS_unstable"
  65                         ;;
  66         esac
  67         NON_US="false" # this is changed below if sources can be found
  68         rm -f Sources.$d
  69         if [ "$d" = "unstable" ]; then
  70                 gzip -dc /org/incoming.debian.org/buildd/Sources.gz >> Sources.$d
  71         fi
  72         for a in $ARCHS ; do
  73                 rm -f Packages.$d.$a quinn-$d.$a
  74                 if [ "$d" = "unstable" ]; then
  75                         gzip -dc /org/incoming.debian.org/buildd/Packages.gz >> Packages.$d.$a
  76                 fi
  77         done
  78
  79         for s in $SECTIONS ; do
  80                 if echo $d | grep -qv -- -security; then
  81                         rm -f Sources.gz
  82                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/source/Sources.gz; then
  83                                 mv Sources.gz Sources-non-us.$d.$s.gz
  84                                 NON_US="true"
  85                         fi
  86                         gzip -dc Sources-non-us.$d.$s.gz >> Sources.$d
  87                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/source/Sources.gz >> Sources.$d
  88                         if [ "$d" = "testing" -o "$d" = "stable" ]; then
  89                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/source/Sources.gz; then
  90                                         mv Sources.gz Sources-non-us.$d-p-u.$s.gz
  91                                 fi
  92                                 gzip -dc Sources-non-us.$d-p-u.$s.gz >> Sources.$d
  93                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/source/Sources.gz >> Sources.$d
  94                         fi
  95
  96                         rm -f Packages.gz
  97                         for a in $ARCHS ; do
  98                                 if [ "$NON_US" = "true" ]; then
  99                                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/binary-$a/Packages.gz; then
 100                                                 mv Packages.gz Packages-non-us.$d.$s.$a.gz
 101                                         fi
 102                                 fi
 103                                 gzip -dc Packages-non-us.$d.$s.$a.gz >> Packages.$d.$a
 104
 105                                 gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/binary-$a/Packages.gz >> Packages.$d.$a
 106                                 if [ "$d" = "testing" -o "$d" = "stable" ]; then
 107                                         if [ "$NON_US" = "true" ]; then
 108                                                 if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/binary-$a/Packages.gz; then
 109                                                         mv Packages.gz Packages-non-us.$d-p-u.$s.$a.gz
 110                                                 fi
 111                                         fi
 112                                         gzip -dc Packages-non-us.$d-p-u.$s.$a.gz >> Packages.$d.$a
 113                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/binary-$a/Packages.gz >> Packages.$d.$a
 114                                 fi
 115                                 if [ "$d" = "unstable" -a "$s" = "main" ]; then
 116                                         gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/debian-installer/binary-$a/Packages.gz >> Packages.$d.$a
 117                                 fi
 118                         done
 119                 else
 120                         rm -f Sources.gz
 121                         if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/source/Sources.gz; then
 122                                 mv Sources.gz Sources-$d.$s.gz
 123                         fi
 124                         gzip -dc Sources-$d.$s.gz >> Sources.$d
 125                         if [ "$s" = "main" ]; then
 126                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Sources.gz -o Sources.gz; then
 127                                         mv Sources.gz Sources-$d.accepted.gz
 128                                 fi
 129                                 gzip -dc Sources-$d.accepted.gz >> Sources.$d
 130                                 if curl $CURLOPT http://security.debian.org/buildd/$dist/Packages.gz -o Packages.gz; then
 131                                         mv Packages.gz Packages.$d.accepted.gz
 132                                 fi
 133                         fi
 134                         rm -f Packages.gz
 135                         for a in $ARCHS ; do
 136                                 if wget $WGETOPT http://security.debian.org/debian-security/dists/$dist/updates/$s/binary-$a/Packages.gz; then
 137                                         mv Packages.gz Packages.$d.$s.$a.gz
 138                                 fi
 139                                 gzip -dc Packages.$d.$s.$a.gz >> Packages.$d.$a
 140                                 if [ "$s" = "main" ]; then
 141                                         gzip -dc Packages.$d.accepted.gz >> Packages.$d.$a
 142                                 fi
 143                         done
 144                 fi
 145         done
 146
 147         if [ "$d" = "oldstable" ]; then
 148                 gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Sources.gz >> Sources.$d
 149                 if "$NON_US" = "true" ]; then
 150                         rm -f Sources.gz
 151                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Sources.gz; then
 152                                 mv Sources.gz Sources-non-us.p-u.gz
 153                         fi
 154                         gzip -dc Sources-non-us.p-u.gz >> Sources.$d
 155                         rm -f Packages.gz
 156                         if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Packages.gz; then
 157                                 mv Packages.gz Packages-non-us.p-u.gz
 158                         fi
 159                 fi
 160
 161                 for a in $ARCHS ; do
 162                         gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Packages.gz >> Packages.$d.$a
 163                         gzip -dc Packages-non-us.p-u.gz >> Packages.$d.$a
 164                 done
 165         fi
 166         for a in $ARCHS ; do
 167                 if [ "$d" = "unstable" -o ! -e "quinn-unstable.$a-old" ]; then
 168                         quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 169                 else
 170                         quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a | fgrep -v -f quinn-unstable.$a-old >> quinn-$d.$a
 171                 fi
 172         done
 173 done
 174
 175 umask 002
 176 for a in $ARCHS_unstable ; do
 177         wanna-build --create-maintenance-lock --database=$a/build-db
 178
 179         for d in $DISTS ; do
 180                 dist=`echo $d | sed s/-.*$//`
 181                 case "$dist" in
 182                         oldstable)
 183                                 if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
 184                                         continue
 185                                 fi
 186                                 ;;
 187                         stable)
 188                                 if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 189                                         continue
 190                                 fi
 191                                 ;;
 192                         testing)
 193                                 if echo $ARCHS_testing | grep -q -v "\b$a\b"; then
 194                                         continue
 195                                 fi
 196                                 ;;
 197                         *)
 198                                 if echo $ARCHS_unstable | grep -q -v "\b$a\b"; then
 199                                         continue
 200                                 fi
 201                                 ;;
 202                 esac
 203                 wanna-build --merge-packages --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a
 204                 perl -pi -e 's#^(non-US/)?(non-free)/.*$##msg' quinn-$d.$a
 205                 wanna-build --merge-quinn --dist=$d --database=$a/build-db quinn-$d.$a
 206                 wanna-build --merge-sources --dist=$d --database=$a/build-db Sources.$d
 207                 mv Packages.$d.$a Packages.$d.$a-old
 208                 mv quinn-$d.$a quinn-$d.$a-old
 209                 if [ "$DAY" = "0" -a "$d" = "unstable" ]; then
 210                         savelog -p -c 26 /org/wanna-build/db/$a/transactions.log
 211                 fi
 212         done
 213         wanna-build --remove-maintenance-lock --database=$a/build-db
 214 done
 215 umask 022
 216 for d in $DISTS; do
 217         mv Sources.$d Sources.$d-old
 218 done
 219
 220 echo "merge ended `date`"
 221 /org/wanna-build/bin/wb-graph >> /org/wanna-build/etc/graph-data
 222 /org/wanna-build/bin/wb-graph -p >> /org/wanna-build/etc/graph2-data
 223 rm -f "$LOCKFILE"
 224 trap -
 225 /org/buildd.debian.org/bin/makegraph