X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=wanna-build%2Ftrigger.daily;h=0107564ee8065ec467358d1cafbe6b41b7c0c80a;hb=155a2a479011874c09351074f2970cce66275b82;hp=8d80dbd4797866a745c1deeffd4aa8d411030475;hpb=a28cb5af382c0a7601451f65b90def4ab69021e4;p=dak.git

diff --git a/wanna-build/trigger.daily b/wanna-build/trigger.daily
index 8d80dbd4..0107564e 100755
--- a/wanna-build/trigger.daily
+++ b/wanna-build/trigger.daily
@@ -1,35 +1,45 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Updates wanna-build databases after the archive maintenance
 # finishes
 #
+# Files:
+#     Sources-* == upstream fetched file
+#     Sources.* == uncompressed, concat'd version
 PATH="/bin:/usr/bin"
-DISTS="oldstable-security stable stable-security testing testing-security unstable"
+#testing must be before unstable so late upld don't build for testing needlessly
+DISTS="oldstable-security stable-security testing-security stable testing unstable"
+STATS_DISTS="unstable testing stable"
 SECTIONS="main contrib non-free"
-ARCHS_oldstable="m68k arm sparc alpha powerpc i386"
-ARCHS_stable="mips mipsel ia64 hppa s390 $ARCHS_oldstable"
+ARCHS_oldstable="m68k arm sparc alpha powerpc i386 mips mipsel ia64 hppa s390"
+ARCHS_stable="$ARCHS_oldstable"
 ARCHS_testing="$ARCHS_stable"
-ARCHS_unstable="$ARCHS_testing hurd-i386 sh"
+ARCHS_unstable="$ARCHS_testing hurd-i386"
 TMPDIR="/org/wanna-build/tmp"
-WGETOPT="-q -t2 -w0 -T60"
-CURLOPT="-q -s -S -f -K /org/wanna-build/trigger.curlrc"
+WGETOPT="-q -t2 -w0 -T10"
+CURLOPT="-q -s -S -f -y 5 -K /org/wanna-build/trigger.curlrc"
 LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress"
 
 DAY=`date +%w`
 
+if lockfile -! -l 3600 $LOCKFILE; then
+	echo "Cannot lock $LOCKFILE"
+	exit 1
+fi
+
+cleanup() {
+	rm -f "$LOCKFILE"
+}
+trap cleanup 0
+
 echo Updating wanna-build databases...
-umask 022
+umask 027
 
 if [ "$DAY" = "0" ]; then
 	savelog -c 26 -p /org/wanna-build/db/merge.log
 fi
 
 exec >> /org/wanna-build/db/merge.log 2>&1
-lockfile -l 3600 $LOCKFILE
-cleanup() {
-	rm -f "$LOCKFILE"
-}
-trap cleanup 0
 
 echo -------------------------------------------------------------------------
 echo "merge triggered `date`"
@@ -55,7 +65,6 @@ for d in $DISTS ; do
 			ARCHS="$ARCHS_unstable"
 			;;
 	esac
-	NON_US="false" # this is changed below if sources can be found
 	rm -f Sources.$d
 	if [ "$d" = "unstable" ]; then
 		gzip -dc /org/incoming.debian.org/buildd/Sources.gz >> Sources.$d
@@ -67,41 +76,18 @@ for d in $DISTS ; do
 		fi
 	done
 
-
 	for s in $SECTIONS ; do
 		if echo $d | grep -qv -- -security; then
 			rm -f Sources.gz
-			if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/source/Sources.gz; then
-				mv Sources.gz Sources-non-us.$d.$s.gz
-				NON_US="true"
-			fi
-			gzip -dc Sources-non-us.$d.$s.gz >> Sources.$d
 			gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/source/Sources.gz >> Sources.$d
 			if [ "$d" = "testing" -o "$d" = "stable" ]; then
-				if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/source/Sources.gz; then
-					mv Sources.gz Sources-non-us.$d-p-u.$s.gz
-				fi
-				gzip -dc Sources-non-us.$d-p-u.$s.gz >> Sources.$d
 				gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/source/Sources.gz >> Sources.$d
 			fi
 
 			rm -f Packages.gz
 			for a in $ARCHS ; do
-				if [ "$NON_US" = "true" ]; then
-					if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d/non-US/$s/binary-$a/Packages.gz; then
-						mv Packages.gz Packages-non-us.$d.$s.$a.gz
-					fi
-				fi
-				gzip -dc Packages-non-us.$d.$s.$a.gz >> Packages.$d.$a
-
 				gzip -dc /org/ftp.debian.org/ftp/dists/$d/$s/binary-$a/Packages.gz >> Packages.$d.$a
 				if [ "$d" = "testing" -o "$d" = "stable" ]; then
-					if [ "$NON_US" = "true" ]; then
-						if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/$d-proposed-updates/non-US/$s/binary-$a/Packages.gz; then
-							mv Packages.gz Packages-non-us.$d-p-u.$s.$a.gz
-						fi
-					fi
-					gzip -dc Packages-non-us.$d-p-u.$s.$a.gz >> Packages.$d.$a
 					gzip -dc /org/ftp.debian.org/ftp/dists/$d-proposed-updates/$s/binary-$a/Packages.gz >> Packages.$d.$a
 				fi
 				if [ "$d" = "unstable" -a "$s" = "main" ]; then
@@ -120,7 +106,7 @@ for d in $DISTS ; do
 				fi
 				gzip -dc Sources-$d.accepted.gz >> Sources.$d
 				if curl $CURLOPT http://security.debian.org/buildd/$dist/Packages.gz -o Packages.gz; then
-					mv Packages.gz Packages-$d.accepted.gz
+					mv Packages.gz Packages.$d.accepted.gz
 				fi
 			fi
 			rm -f Packages.gz
@@ -136,27 +122,20 @@ for d in $DISTS ; do
 		fi
 	done
 
-	if [ "$d" = "oldstable" ]; then
-		gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Sources.gz >> Sources.$d
-		if "$NON_US" = "true" ]; then
-			rm -f Sources.gz
-			if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Sources.gz; then
-				mv Sources.gz Sources-non-us.p-u.gz
-			fi
-			gzip -dc Sources-non-us.p-u.gz >> Sources.$d
-			rm -f Packages.gz
-			if wget $WGETOPT http://non-us.debian.org/debian-non-US/dists/old-proposed-updates/Packages.gz; then
-				mv Packages.gz Packages-non-us.p-u.gz
+	for a in $ARCHS ; do
+		if [ "$d" = "unstable" -o ! -e "quinn-unstable.$a-old" ]; then
+			quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
+		else
+			if echo $d | grep -qv -- -security; then
+				quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a | fgrep -v -f quinn-unstable.$a-old | grep ":out-of-date\]$" >> quinn-$d.$a
+				sed -e 's/\[\w*:\w*]$//' quinn-$d-security.$a > quinn-$d-security.$a.grep
+				grep -vf quinn-$d-security.$a quinn-$d.$a > quinn-$d.$a.grep
+				mv quinn-$d.$a.grep quinn-$d.$a
+				rm quinn-$d-security.$a.grep
+			else
+				quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 			fi
 		fi
-			
-		for a in $ARCHS ; do
-			gzip -dc /org/ftp.debian.org/ftp/dists/old-proposed-updates/Packages.gz >> Packages.$d.$a
-			gzip -dc Packages-non-us.p-u.gz >> Packages.$d.$a
-		done
-	fi
-	for a in $ARCHS ; do
-		quinn-diff -A $a -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -s Sources.$d -p Packages.$d.$a >> quinn-$d.$a
 	done
 done
 
@@ -167,6 +146,11 @@ for a in $ARCHS_unstable ; do
 	for d in $DISTS ; do
 		dist=`echo $d | sed s/-.*$//`
 		case "$dist" in
+			oldstable)
+				if echo $ARCHS_oldstable | grep -q -v "\b$a\b"; then
+					continue
+				fi
+				;;
 			stable)
 				if echo $ARCHS_stable | grep -q -v "\b$a\b"; then
 					continue
@@ -183,16 +167,14 @@ for a in $ARCHS_unstable ; do
 				fi
 				;;
 		esac
-		wanna-build --merge-packages --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a
-		perl -pi -e 's#^(non-US/)?(non-free)/.*$##msg' quinn-$d.$a
-		wanna-build --merge-quinn --dist=$d --database=$a/build-db quinn-$d.$a
-		wanna-build --merge-sources --dist=$d --database=$a/build-db Sources.$d
+		perl -pi -e 's#^(non-free)/.*$##msg' quinn-$d.$a
+		wanna-build --merge-all --arch=$a --dist=$d --database=$a/build-db Packages.$d.$a quinn-$d.$a Sources.$d
 		mv Packages.$d.$a Packages.$d.$a-old
 		mv quinn-$d.$a quinn-$d.$a-old
-		if [ "$DAY" = "0" -a "$d" = "unstable" ]; then
-			savelog -p -c 26 /org/wanna-build/db/$a/transactions.log
-		fi
 	done
+	if [ "$DAY" = "0" ]; then
+		savelog -p -c 26 /org/wanna-build/db/$a/transactions.log
+	fi
 	wanna-build --remove-maintenance-lock --database=$a/build-db
 done
 umask 022
@@ -201,8 +183,14 @@ for d in $DISTS; do
 done
 
 echo "merge ended `date`"
-rm -f "$LOCKFILE"
-trap -
 /org/wanna-build/bin/wb-graph >> /org/wanna-build/etc/graph-data
 /org/wanna-build/bin/wb-graph -p >> /org/wanna-build/etc/graph2-data
+rm -f "$LOCKFILE"
+trap -
 /org/buildd.debian.org/bin/makegraph
+for a in $ARCHS_stable; do
+	echo Last Updated: `date -u` > /org/buildd.debian.org/web/stats/$a.txt
+	for d in $STATS_DISTS; do
+		/org/wanna-build/bin/wanna-build-statistics --database=$a/build-db --dist=$d >> /org/buildd.debian.org/web/stats/$a.txt
+	done
+done