From: Torsten Werner Date: Tue, 11 Jan 2011 20:11:24 +0000 (+0100) Subject: Merge branch 'master' into dbtests X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=ec257c02a5d62fd27844c70814acd9616b24b4c8;hp=6412bf770cd0837a81a3ef1c6cc52a1dcded6ccf Merge branch 'master' into dbtests --- diff --git a/config/backports/cron.dinstall b/config/backports/cron.dinstall index c5c123ad..c2a5b322 100755 --- a/config/backports/cron.dinstall +++ b/config/backports/cron.dinstall @@ -405,6 +405,9 @@ stage $GO & # ) # stage $GO +# we need to wait for the background processes before the end of dinstall +wait + log "Daily cron scripts successful, all done" exec > "$logdir/afterdinstall.log" 2>&1 diff --git a/config/backports/cron.unchecked b/config/backports/cron.unchecked index 08fcbf3e..46f30e07 100755 --- a/config/backports/cron.unchecked +++ b/config/backports/cron.unchecked @@ -78,7 +78,6 @@ function do_buildd () { function do_dists () { cd $configdir dak generate-filelist - GZIP='--rsyncable' ; export GZIP dak generate-packages-sources } diff --git a/config/backports/dinstall.functions b/config/backports/dinstall.functions index 6e436f8b..d8ca5172 100644 --- a/config/backports/dinstall.functions +++ b/config/backports/dinstall.functions @@ -198,7 +198,6 @@ function mpfm() { function packages() { log "Generating Packages and Sources files" cd $configdir - GZIP='--rsyncable' ; export GZIP apt-ftparchive generate apt.conf #dak generate-packages-sources } @@ -249,13 +248,13 @@ function mklslar() { mv -f ${FILENAME}.gz ${FILENAME}.old.gz mv -f .${FILENAME}.new ${FILENAME} rm -f ${FILENAME}.patch.gz - zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz + zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz rm -f ${FILENAME}.old.gz else mv -f .${FILENAME}.new ${FILENAME} fi - gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz + gzip -9cfN ${FILENAME} >${FILENAME}.gz rm -f ${FILENAME} } @@ -270,7 +269,7 @@ function mkmaintainers() { if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then log "installing Maintainers ... " mv -f .new-maintainers Maintainers - gzip --rsyncable -9v .new-maintainers.gz + gzip -9v .new-maintainers.gz mv -f .new-maintainers.gz Maintainers.gz else rm -f .new-maintainers @@ -288,7 +287,7 @@ function mkuploaders() { if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then log "installing Uploaders ... " mv -f .new-uploaders Uploaders - gzip --rsyncable -9v .new-uploaders.gz + gzip -9v .new-uploaders.gz mv -f .new-uploaders.gz Uploaders.gz else rm -f .new-uploaders @@ -339,7 +338,7 @@ function mkfilesindices() { cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep "/source/" - ) | sort -u | gzip --rsyncable -9 > source.list.gz + ) | sort -u | gzip -9 > source.list.gz log "Generating arch lists" @@ -351,7 +350,7 @@ function mkfilesindices() { cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" - ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz + ) | sort -u | gzip -9 > arch-$a.list.gz done log "Generating suite lists" @@ -374,7 +373,7 @@ function mkfilesindices() { done ) suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,' - ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz + ) | sort -u | gzip -9 > suite-${suite}.list.gz done log "Finding everything on the ftp site to generate sundries" @@ -394,7 +393,7 @@ function mkfilesindices() { (cd $base/ftp/ for dist in sid squeeze; do - find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz + find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz done ) @@ -413,7 +412,7 @@ function mkchecksums() { cd "$ftpdir" ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 - ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz + ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz ${bindir}/dsync-flist -q link-dups $dsynclist || true } diff --git a/config/backports/vars b/config/backports/vars index 736d8a6f..c4e02d93 100644 --- a/config/backports/vars +++ b/config/backports/vars @@ -53,6 +53,7 @@ TMPDIR==$base/tmp PATH=$masterdir:$PATH umask 022 unset CDPATH +GZIP='--rsyncable' ; export GZIP # Set the database variables eval $(dak admin config db-shell) diff --git a/config/debian-security/apt.conf b/config/debian-security/apt.conf index fcaa6113..456bfcb1 100644 --- a/config/debian-security/apt.conf +++ b/config/debian-security/apt.conf @@ -2,37 +2,23 @@ APT::FTPArchive::Contents off; Dir { - ArchiveDir "/org/security.debian.org/ftp/"; - OverrideDir "/org/security.debian.org/override/"; - CacheDir "/org/security.debian.org/dak-database/"; + ArchiveDir "/srv/security-master.debian.org/ftp/"; + OverrideDir "/srv/security-master.debian.org/override/"; + CacheDir "/srv/security-master.debian.org/dak-database/"; }; Default { - Packages::Compress ". gzip bzip2"; + Packages::Compress "gzip bzip2"; Sources::Compress "gzip bzip2"; DeLinkLimit 0; FileMode 0664; } -tree "dists/oldstable/updates" -{ - FileList "/org/security.debian.org/dak-database/dists/oldstable_updates/$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/org/security.debian.org/dak-database/dists/oldstable_updates/$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source"; - BinOverride "override.etch.$(SECTION)"; - ExtraOverride "override.etch.extra.$(SECTION)"; - SrcOverride "override.etch.$(SECTION).src"; - Contents " "; - Packages::Compress "gzip bzip2"; - Sources::Compress "gzip bzip2"; -}; - tree "dists/stable/updates" { - FileList "/org/security.debian.org/dak-database/dists/stable_updates/$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/org/security.debian.org/dak-database/dists/stable_updates/$(SECTION)_source.list"; + FileList "/srv/security-master.debian.org/dak-database/dists/stable_updates/$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/security-master.debian.org/dak-database/dists/stable_updates/$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source"; BinOverride "override.lenny.$(SECTION)"; @@ -45,10 +31,10 @@ tree "dists/stable/updates" tree "dists/testing/updates" { - FileList "/org/security.debian.org/dak-database/dists/testing_updates/$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/org/security.debian.org/dak-database/dists/testing_updates/$(SECTION)_source.list"; + FileList "/srv/security-master.debian.org/dak-database/dists/testing_updates/$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/security-master.debian.org/dak-database/dists/testing_updates/$(SECTION)_source.list"; Sections "main contrib non-free"; - Architectures "alpha amd64 armel hppa i386 ia64 mips mipsel powerpc s390 sparc source"; + Architectures "alpha amd64 armel hppa i386 ia64 kfreebsd-i386 kfreebsd-amd64 mips mipsel powerpc s390 sparc source"; BinOverride "override.squeeze.$(SECTION)"; ExtraOverride "override.squeeze.extra.$(SECTION)"; SrcOverride "override.squeeze.$(SECTION).src"; diff --git a/config/debian-security/apt.conf.buildd b/config/debian-security/apt.conf.buildd index 85c1f3dc..b16d0c97 100644 --- a/config/debian-security/apt.conf.buildd +++ b/config/debian-security/apt.conf.buildd @@ -2,9 +2,9 @@ APT::FTPArchive::Contents off; Dir { - ArchiveDir "/srv/security.debian.org/buildd/"; - OverrideDir "/srv/security.debian.org/override/"; - CacheDir "/srv/security.debian.org/dak-database/"; + ArchiveDir "/srv/security-master.debian.org/buildd/"; + OverrideDir "/srv/security-master.debian.org/override/"; + CacheDir "/srv/security-master.debian.org/dak-database/"; }; Default @@ -15,19 +15,6 @@ Default FileMode 0664; } -bindirectory "etch" -{ - Packages "etch/Packages"; - Sources "etch/Sources"; - Contents " "; - - BinOverride "override.etch.all3"; - SrcOverride "override.etch.all3.src"; - BinCacheDB "packages-accepted-etch.db"; - PathPrefix ""; - Packages::Extensions ".deb .udeb"; -}; - bindirectory "lenny" { Packages "lenny/Packages"; @@ -53,4 +40,3 @@ bindirectory "squeeze" PathPrefix ""; Packages::Extensions ".deb .udeb"; }; - diff --git a/config/debian-security/cron.buildd b/config/debian-security/cron.buildd index 51110027..0ea2b4fe 100755 --- a/config/debian-security/cron.buildd +++ b/config/debian-security/cron.buildd @@ -2,63 +2,73 @@ # # Executed after cron.unchecked -ARCHS_oldstable="alpha amd64 arm hppa i386 ia64 mips mipsel powerpc sparc s390" -ARCHS_stable="alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc sparc s390" -ARCHS_testing="alpha amd64 armel hppa i386 ia64 mips mipsel powerpc sparc s390" -DISTS="oldstable stable testing" -SSH_SOCKET=~/.ssh/buildd.debian.org.socket - set -e -export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars +set -u + +export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars . $SCRIPTVARS +SSH_SOCKET=~/.ssh/buildd.debian.org.socket +DISTS=$(dak admin s list) if [ -e $ftpdir/Archive_Maintenance_In_Progress ]; then - exit 0 + exit 0 fi -cd $masterdir -for d in $DISTS; do - eval SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz` - eval PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz` +for dist in $DISTS; do + eval SOURCES_$dist=`stat -c "%Y" $base/buildd/$dist/Sources.gz` + eval PACKAGES_$dist=`stat -c "%Y" $base/buildd/$dist/Packages.gz` done +cd $configdir apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd + +cd ${base}/buildd +for dist in $DISTS; do + rm -f $dist/Release* + darchs=$(dak admin s-a list-arch $dist | tr '\n' ' ') + codename=$(dak admin s show ${dist} | grep ^Codename | awk '{print $2}') + apt-ftparchive -qq -o APT::FTPArchive::Release::Codename="${codename}" -o APT::FTPArchive::Release::Origin="Debian" -o APT::FTPArchive::Release::Label="Debian" -o APT::FTPArchive::Release::Description="buildd $dist security" -o APT::FTPArchive::Release::Architectures="${darchs}" release $dist > Release + gpg --secret-keyring ${base}/s3kr1t/dot-gnupg/secring.gpg --keyring ${base}/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o Release.gpg Release + mv Release* $dist/. +done + dists= -for d in $DISTS; do - eval NEW_SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz` - eval NEW_PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz` - old=SOURCES_$d - new=NEW_$old - if [ ${!new} -gt ${!old} ]; then - if [ -z "$dists" ]; then - dists="$d" - else - dists="$dists $d" - fi - continue - fi - old=PACKAGES_$d - new=NEW_$old - if [ ${!new} -gt ${!old} ]; then - if [ -z "$dists" ]; then - dists="$d" - else - dists="$dists $d" - fi - continue - fi + + +for dist in $DISTS; do + eval NEW_SOURCES_$dist=`stat -c "%Y" $base/buildd/$dist/Sources.gz` + eval NEW_PACKAGES_$dist=`stat -c "%Y" $base/buildd/$dist/Packages.gz` + old=SOURCES_$dist + new=NEW_$old + if [ ${!new} -gt ${!old} ]; then + if [ -z "$dists" ]; then + dists="$dist" + else + dists="$dists $dist" + fi + continue + fi + old=PACKAGES_$dist + new=NEW_$old + if [ ${!new} -gt ${!old} ]; then + if [ -z "$dists" ]; then + dists="$dist" + else + dists="$dists $dist" + fi + continue + fi done if [ ! -z "$dists" ]; then - # setup ssh master process - ssh wbadm@buildd -S $SSH_SOCKET -MN 2> /dev/null & - SSH_PID=$! - while [ ! -S $SSH_SOCKET ]; do - sleep 1 - done - trap 'kill -TERM $SSH_PID' 0 - for d in $dists; do - ssh wbadm@buildd -S $SSH_SOCKET trigger.security $d - done + # setup ssh master process + ssh wbadm@buildd -S $SSH_SOCKET -MN 2> /dev/null & + SSH_PID=$! + while [ ! -S $SSH_SOCKET ]; do + sleep 1 + done + trap 'kill -TERM $SSH_PID' 0 + for d in $dists; do + ssh wbadm@buildd -S $SSH_SOCKET trigger.security $d + done fi - diff --git a/config/debian-security/cron.daily b/config/debian-security/cron.daily index e482a192..da566975 100755 --- a/config/debian-security/cron.daily +++ b/config/debian-security/cron.daily @@ -1,16 +1,16 @@ -#!/bin/sh +#!/bin/bash # # Executed daily via cron, out of dak's crontab. set -e -export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars +export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars . $SCRIPTVARS ################################################################################ # Fix overrides -rsync --delete -r --include=override\* --exclude=\* --password-file /srv/non-us.debian.org/s3kr1t/rsync-password -ql security-master@ftp-master::indices/ $overridedir +rsync --delete -r --include=override\* --exclude=\* --password-file /srv/security-master.debian.org/s3kr1t/rsync-password -ql security-master@ftp-master::indices/ $overridedir cd $overridedir for file in override*.gz; do @@ -60,19 +60,21 @@ done ################################################################################ -cd $masterdir +cd $configdir +dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg dak clean-queues +dak clean-queues -i $disembargo dak clean-suites -apt-ftparchive -q clean apt.conf -apt-ftparchive -q clean apt.conf.buildd symlinks -d -r $ftpdir -pg_dump obscurity > /org/security.debian.org/dak-backup/dump_$(date +%Y.%m.%d-%H:%M:%S) +pg_dump obscurity > /org/security-master.debian.org/dak-backup/dump_$(date +%Y.%m.%d-%H:%M:%S) +find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 | +while read dumpname; do + bzip2 -9fv "$dumpname" +done -# Vacuum the database -set +e -echo "VACUUM; VACUUM ANALYZE;" | psql obscurity 2>&1 | egrep -v "^NOTICE: Skipping \"pg_.*only table or database owner can VACUUM it$|^VACUUM$" -set -e +apt-ftparchive -q clean apt.conf +apt-ftparchive -q clean apt.conf.buildd ################################################################################ diff --git a/config/debian-security/cron.unchecked b/config/debian-security/cron.unchecked index 641f8bfb..d519b209 100755 --- a/config/debian-security/cron.unchecked +++ b/config/debian-security/cron.unchecked @@ -1,31 +1,53 @@ -#! /bin/sh +#! /bin/bash set -e -export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars +set -u + +export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars . $SCRIPTVARS +# And use one locale, no matter what the caller has set +export LANG=C +export LC_ALL=C + report=$queuedir/REPORT reportdis=$queuedir/REPORT.disembargo timestamp=$(date "+%Y-%m-%d %H:%M") doanything=false +dopolicy=false + +# So first we should go and see if any process-policy action is done +dak process-policy embargo | mail -a "X-Debian: DAK" -e -s "Automatically accepted from embargoed" team@security.debian.org -- -F "Debian FTP Masters" -f ftonaster@ftp-master.debian.org +dak process-policy disembargo | mail -a "X-Debian: DAK" -e -s "Automatically accepted from unembargoed" team@security.debian.org -- -F "Debian FTP Masters" -f ftonaster@ftp-master.debian.org + +# Now, if this really did anything, we can then sync it over. Files +# in newstage mean they are (late) accepts of security stuff, need +# to sync to ftp-master + +cd $newstage +changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) +if [ -n "$changes" ]; then + dopolicy=true + echo "$timestamp": ${changes:-"Nothing to do in newstage"} >> $report + rsync -a -q $newstage/. /srv/queued/ftpmaster/. + dak process-upload -a -d "$newstage" >> $report +fi cd $unchecked changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) if [ -n "$changes" ]; then - doanything=true - echo "$timestamp": "$changes" >> $report - dak process-unchecked -a $changes >> $report - echo "--" >> $report + doanything=true + echo "$timestamp": ${changes:-"Nothing to do in unchecked"} >> $report + dak process-upload -a -d "$unchecked" >> $report fi cd $disembargo changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) if [ -n "$changes" ]; then - doanything=true - echo "$timestamp": "$changes" >> $reportdis - dak process-unchecked -a $changes >> $reportdis - echo "--" >> $reportdis + doanything=true + echo "$timestamp": ${changes:-"Nothing to do in disembargo"} >> $reportdis + dak process-upload -a -d "$disembargo" >> $reportdis fi if ! $doanything; then @@ -33,4 +55,16 @@ if ! $doanything; then exit 0 fi -sh $masterdir/cron.buildd +if [ "x${dopolicy}x" = "xtruex" ]; then + # We had something approved from a policy queue, push out new archive + dak dominate + dak generate-filelist + cd $configdir + $configdir/map.sh + apt-ftparchive generate apt.conf + dak generate-releases + /srv/security-master.debian.org/dak/config/debian-security/make-mirror.sh + sudo -u archvsync -H /home/archvsync/signal_security +fi + +$configdir/cron.buildd diff --git a/config/debian-security/cron.weekly b/config/debian-security/cron.weekly index fc813ecf..80a83538 100755 --- a/config/debian-security/cron.weekly +++ b/config/debian-security/cron.weekly @@ -1,9 +1,9 @@ -#!/bin/sh +#!/bin/bash # # Executed weekly via cron, out of dak's crontab. set -e -export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars +export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars . $SCRIPTVARS ################################################################################ @@ -11,8 +11,9 @@ export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars # Weekly generation of release files, then pushing mirrors. # Used as we have a "Valid-until" field in our release files of 10 days. In case # we dont have a security update in that time... -cd $masterdir +cd $configdir dak generate-releases +/srv/security-master.debian.org/dak/config/debian-security/make-mirror.sh sudo -u archvsync -H /home/archvsync/signal_security diff --git a/config/debian-security/dak.conf b/config/debian-security/dak.conf index 0f77a7f4..dfd12bf9 100644 --- a/config/debian-security/dak.conf +++ b/config/debian-security/dak.conf @@ -1,10 +1,11 @@ Dinstall { GPGKeyring { - "/org/keyring.debian.org/keyrings/debian-keyring.gpg"; + "/srv/keyring.debian.org/keyrings/debian-keyring.gpg"; }; - SigningKeyring "/org/non-us.debian.org/s3kr1t/dot-gnupg/secring.gpg"; - SigningPubKeyring "/org/non-us.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; + // was non-us.d.o path before + SigningKeyring "/srv/security-master.debian.org/s3kr1t/dot-gnupg/secring.gpg"; + SigningPubKeyring "/srv/security-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; SigningKeyIds "55BE302B"; SendmailCommand "/usr/sbin/sendmail -odq -oi -t"; MyEmailAddress "Debian Installer "; @@ -13,7 +14,7 @@ Dinstall MyDistribution "Debian"; // Used in emails BugServer "bugs.debian.org"; PackagesServer "packages.debian.org"; - LockFile "/org/security.debian.org/dak/lock"; + LockFile "/org/security-master.debian.org/dak/lock"; Bcc "archive@ftp-master.debian.org"; // GroupOverrideFilename "override.group-maint"; FutureTimeTravelGrace 28800; // 8 hours @@ -24,7 +25,6 @@ Dinstall BXANotify "false"; QueueBuildSuites { - oldstable; stable; testing; }; @@ -38,7 +38,7 @@ Dinstall Process-New { - AcceptedLockFile "/org/security.debian.org/lock/unchecked.lock"; + DinstallLockFile "/srv/security-master.debian.org/lock/processnew.lock"; LockDir "/srv/security-master.debian.org/lock/new/"; }; @@ -53,9 +53,48 @@ Queue-Report { Directories { - // byhand; - // new; + byhand; + new; unembargoed; + embargoed; + }; +}; + +Import-Keyring +{ + /srv/keyring.debian.org/keyrings/debian-maintainers.gpg + { + Debian-Maintainer "true"; + }; +}; + +Import-LDAP-Fingerprints +{ + LDAPDn "ou=users,dc=debian,dc=org"; + LDAPServer "db.debian.org"; + ExtraKeyrings + { + "/srv/keyring.debian.org/keyrings/removed-keys.pgp"; + "/srv/keyring.debian.org/keyrings/removed-keys.gpg"; + "/srv/keyring.debian.org/keyrings/extra-keys.pgp"; + }; + KeyServer "wwwkeys.eu.pgp.net"; +}; + +Check-Overrides +{ + OverrideSuites + { + Stable + { + Process "0"; + }; + + Testing + { + Process "0"; + }; + }; }; @@ -76,12 +115,12 @@ Rm }; MyEmailAddress "Debian Archive Maintenance "; - LogFile "/org/security.debian.org/dak-log/removals.txt"; + LogFile "/srv/security-master.debian.org/dak-log/removals.txt"; }; Init-Archive { - ExportDir "/org/security.debian.org/dak/import-archive-files/"; + ExportDir "/srv/security-master.debian.org/dak/import-archive-files/"; }; Clean-Suites @@ -111,25 +150,6 @@ Suite // Priority determines which suite is used for the Maintainers file // as generated by 'dak make-maintainers' (highest wins). - OldStable - { - Components - { - updates/main; - updates/contrib; - updates/non-free; - }; - Announce "dak@security.debian.org"; - Version ""; - Origin "Debian"; - Label "Debian-Security"; - Description "Debian 4.0 Security Updates"; - ValidTime 864000; // 10 days - CodeName "etch"; - OverrideCodeName "etch"; - CopyDotDak "/org/security.debian.org/queue/done/"; - }; - Stable { Components @@ -146,7 +166,7 @@ Suite ValidTime 864000; // 10 days CodeName "lenny"; OverrideCodeName "lenny"; - CopyDotDak "/org/security.debian.org/queue/done/"; + CopyDotDak "/srv/security-master.debian.org/queue/done/"; }; Testing @@ -165,7 +185,7 @@ Suite ValidTime 864000; // 10 days CodeName "squeeze"; OverrideCodeName "squeeze"; - CopyDotDak "/org/security.debian.org/queue/done/"; + CopyDotDak "/srv/security-master.debian.org/queue/done/"; }; }; @@ -182,35 +202,37 @@ SuiteMappings Dir { - Root "/org/security.debian.org/ftp/"; - Pool "/org/security.debian.org/ftp/pool/"; - Dak "/org/security.debian.org/dak/"; - Templates "/org/security.debian.org/dak/templates/"; + Root "/srv/security-master.debian.org/ftp/"; + Pool "/srv/security-master.debian.org/ftp/pool/"; + Dak "/srv/security-master.debian.org/dak/"; + Templates "/srv/security-master.debian.org/dak/templates/"; PoolRoot "pool/"; - Override "/org/security.debian.org/override/"; - Lock "/org/security.debian.org/lock/"; - Lists "/org/security.debian.org/dak-database/dists/"; - Log "/org/security.debian.org/dak-log/"; - Morgue "/org/security.debian.org/morgue/"; + Override "/srv/security-master.debian.org/override/"; + Lock "/srv/security-master.debian.org/lock/"; + Cache "/srv/security-master.debian.org/database/"; + Lists "/srv/security-master.debian.org/dak-database/dists/"; + Log "/srv/security-master.debian.org/dak-log/"; + Morgue "/srv/security-master.debian.org/morgue/"; MorgueReject "reject"; - Override "/org/security.debian.org/scripts/override/"; - QueueBuild "/org/security.debian.org/buildd/"; - Upload "/srv/queued/UploadQueue/"; + Override "/srv/security-master.debian.org/scripts/override/"; + QueueBuild "/srv/security-master.debian.org/buildd/"; + Upload "/srv/queued/ftpmaster/"; Queue { - Accepted "/org/security.debian.org/queue/accepted/"; - Byhand "/org/security.debian.org/queue/byhand/"; - Done "/org/security.debian.org/queue/done/"; - Holding "/org/security.debian.org/queue/holding/"; - New "/org/security.debian.org/queue/new/"; - Reject "/org/security.debian.org/queue/reject/"; - Unchecked "/org/security.debian.org/queue/unchecked/"; + Byhand "/srv/security-master.debian.org/queue/byhand/"; + Done "/srv/security-master.debian.org/queue/done/"; + Holding "/srv/security-master.debian.org/queue/holding/"; + New "/srv/security-master.debian.org/queue/new/"; + Reject "/srv/security-master.debian.org/queue/reject/"; + Unchecked "/srv/security-master.debian.org/queue/unchecked/"; + Newstage "/srv/security-master.debian.org/queue/newstage/"; + ProposedUpdates "/does/not/exist/"; // XXX fixme OldProposedUpdates "/does/not/exist/"; // XXX fixme - Embargoed "/org/security.debian.org/queue/embargoed/"; - Unembargoed "/org/security.debian.org/queue/unembargoed/"; - Disembargo "/org/security.debian.org/queue/unchecked-disembargo/"; + Embargoed "/srv/security-master.debian.org/queue/embargoed/"; + Unembargoed "/srv/security-master.debian.org/queue/unembargoed/"; + Disembargo "/srv/security-master.debian.org/queue/unchecked-disembargo/"; }; }; @@ -239,6 +261,8 @@ Architectures s390 "IBM S/390"; sparc "Sun SPARC/UltraSPARC"; amd64 "AMD x86_64 (AMD64)"; + kfreebsd-i386 "GNU/kFreeBSD i386"; + kfreebsd-amd64 "GNU/kFreeBSD amd64"; }; @@ -362,7 +386,7 @@ OverrideType Location { - /org/security.debian.org/ftp/pool/ + /srv/security-master.debian.org/ftp/pool/ { Archive "security"; Suites diff --git a/config/debian-security/dak.conf-etc b/config/debian-security/dak.conf-etc new file mode 100644 index 00000000..e8af8d98 --- /dev/null +++ b/config/debian-security/dak.conf-etc @@ -0,0 +1,9 @@ +Config +{ + chopin.debian.org + { + DatabaseHostname "security"; + DakConfig "/org/security-master.debian.org/dak/config/debian-security/dak.conf"; + AptConfig "/org/security-master.debian.org/dak/config/debian-security/apt.conf"; + } +} \ No newline at end of file diff --git a/config/debian-security/make-mirror.sh b/config/debian-security/make-mirror.sh new file mode 100755 index 00000000..1b803258 --- /dev/null +++ b/config/debian-security/make-mirror.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +LANG=C +LC_ALL=C + +echo "Regenerating \"public\" mirror/ hardlink fun" +date -u > /srv/security-master.debian.org/ftp/project/trace/security-master.debian.org +echo "Using dak v1" >> /srv/security-master.debian.org/ftp/project/trace/security-master.debian.org +echo "Running on host: $(hostname -f)" >> /srv/security-master.debian.org/ftp/project/trace/security-master.debian.org +cd /srv/security.debian.org/archive/debian-security/ +rsync -aH --link-dest /srv/security-master.debian.org/ftp/ --exclude Archive_Maintenance_In_Progress --delete --delete-after --ignore-errors /srv/security-master.debian.org/ftp/. . diff --git a/config/debian-security/map.sh b/config/debian-security/map.sh index d0cbaf44..06cd5384 100755 --- a/config/debian-security/map.sh +++ b/config/debian-security/map.sh @@ -1,3 +1,3 @@ #!/bin/bash -dak make-pkg-file-mapping | bzip2 -9 > /org/security.debian.org/ftp/indices/package-file.map.bz2 +dak make-pkg-file-mapping | bzip2 -9 > /srv/security-master.debian.org/ftp/indices/package-file.map.bz2 diff --git a/config/debian-security/vars b/config/debian-security/vars index 2add99ea..2d040a91 100644 --- a/config/debian-security/vars +++ b/config/debian-security/vars @@ -1,22 +1,51 @@ # locations used by many scripts -base=/org/security.debian.org +base=/srv/security-master.debian.org +public=/srv/security.debian.org +bindir=$base/bin ftpdir=$base/ftp/ -masterdir=$base/dak/config/debian-security/ +masterdir=$base/dak/ +configdir=$masterdir/config/debian-security/ +webdir=$masterdir/web +indices=$ftpdir/indices +archs=$(dak admin a list | tr '\n' ' ') + +scriptdir=$base/scripts +scriptsdir=$masterdir/scripts/debian/ +dbdir=$base/dak-database/ +lockdir=$base/lock +stagedir=$lockdir/stages overridedir=$base/override +extoverridedir=$scriptdir/external-overrides +logdir=$base/log/cron/ + queuedir=$base/queue/ unchecked=$queuedir/unchecked/ +newstage=$queuedir/newstage/ disembargo=$queuedir/unchecked-disembargo/ -accepted=$queuedir/accepted/ done=$queuedir/done/ +mirrordir=$base/mirror/ +exportdir=$base/export/ +exportpublic=$public/rsync/export/ + uploadhost=ftp-master.debian.org uploaddir=/pub/UploadQueue/ +ftpgroup=debadmin + components="main non-free contrib" -suites="oldstable stable testing" +suites=$(dak admin s list) override_types="deb dsc udeb" +TMPDIR=${base}/tmp + PATH=$masterdir:$PATH umask 022 +unset CDPATH + +# Set the database variables +eval $(dak admin config db-shell) +PATH=$masterdir:$PATH +umask 022 diff --git a/config/debian/cron.daily b/config/debian/cron.daily index 5b433568..0e7f7da4 100755 --- a/config/debian/cron.daily +++ b/config/debian/cron.daily @@ -35,4 +35,7 @@ reports clean_debbugs +# Generate list of override disparities +dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz + ################################################################################ diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked index 83372f98..c7d7dfdd 100755 --- a/config/debian/cron.unchecked +++ b/config/debian/cron.unchecked @@ -77,7 +77,6 @@ function do_buildd () { function do_dists () { cd $configdir dak generate-filelist - GZIP='--rsyncable' ; export GZIP dak generate-packages-sources >/dev/null } diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 605dd2b9..f3ab3816 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -207,7 +207,6 @@ function mpfm() { function packages() { log "Generating Packages and Sources files" cd $configdir - GZIP='--rsyncable' ; export GZIP #apt-ftparchive generate apt.conf dak generate-packages-sources } @@ -258,13 +257,13 @@ function mklslar() { mv -f ${FILENAME}.gz ${FILENAME}.old.gz mv -f .${FILENAME}.new ${FILENAME} rm -f ${FILENAME}.patch.gz - zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz + zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip -9cfn - >${FILENAME}.patch.gz rm -f ${FILENAME}.old.gz else mv -f .${FILENAME}.new ${FILENAME} fi - gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz + gzip -9cfN ${FILENAME} >${FILENAME}.gz rm -f ${FILENAME} } @@ -279,7 +278,7 @@ function mkmaintainers() { if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then log "installing Maintainers ... " mv -f .new-maintainers Maintainers - gzip --rsyncable -9v .new-maintainers.gz + gzip -9v .new-maintainers.gz mv -f .new-maintainers.gz Maintainers.gz else rm -f .new-maintainers @@ -297,7 +296,7 @@ function mkuploaders() { if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then log "installing Uploaders ... " mv -f .new-uploaders Uploaders - gzip --rsyncable -9v .new-uploaders.gz + gzip -9v .new-uploaders.gz mv -f .new-uploaders.gz Uploaders.gz else rm -f .new-uploaders @@ -348,7 +347,7 @@ function mkfilesindices() { cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep "/source/" - ) | sort -u | gzip --rsyncable -9 > source.list.gz + ) | sort -u | gzip -9 > source.list.gz log "Generating arch lists" @@ -360,7 +359,7 @@ function mkfilesindices() { cd $base/ftp find ./dists -maxdepth 1 \! -type d find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" - ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz + ) | sort -u | gzip -9 > arch-$a.list.gz done log "Generating suite lists" @@ -383,7 +382,7 @@ function mkfilesindices() { done ) suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,' - ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz + ) | sort -u | gzip -9 > suite-${suite}.list.gz done log "Finding everything on the ftp site to generate sundries" @@ -403,7 +402,7 @@ function mkfilesindices() { (cd $base/ftp/ for dist in sid squeeze; do - find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz + find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz done ) @@ -422,7 +421,7 @@ function mkchecksums() { cd "$ftpdir" ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 - ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz + ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz ${bindir}/dsync-flist -q link-dups $dsynclist || true } diff --git a/config/debian/vars b/config/debian/vars index 4677d7c1..224843da 100644 --- a/config/debian/vars +++ b/config/debian/vars @@ -37,6 +37,7 @@ TMPDIR=${base}/tmp PATH=$masterdir:$PATH umask 022 unset CDPATH +GZIP='--rsyncable' ; export GZIP # Set the database variables eval $(dak admin config db-shell) diff --git a/dak/dak.py b/dak/dak.py index cdb0331f..fb002102 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -146,6 +146,8 @@ def init(): "Generate changelog between two suites"), ("copy-installer", "Copies the installer from one suite to another"), + ("override-disparity", + "Generate a list of override disparities"), ] return functionality diff --git a/dak/dakdb/update22.py b/dak/dakdb/update22.py index b6fbbb44..dbd7ced6 100755 --- a/dak/dakdb/update22.py +++ b/dak/dakdb/update22.py @@ -69,7 +69,7 @@ def do_update(self): for q in c.fetchall(): queues[q[0]] = q[1] - if q[1] in ['accepted', 'buildd']: + if q[1] in ['accepted', 'buildd', 'embargoed', 'unembargoed']: # Move to build_queue_table c.execute("""INSERT INTO build_queue (queue_name, path, copy_files) VALUES ('%s', '%s', '%s')""" % (q[1], q[2], q[3])) diff --git a/dak/ls.py b/dak/ls.py index 85ad4905..66e8d427 100755 --- a/dak/ls.py +++ b/dak/ls.py @@ -150,7 +150,10 @@ SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name AND s.file = f.id AND f.location = l.id AND l.component = c.id AND s.maintainer = m.id %s """ % (comparison_operator, con_suites), {'package': package}) - ql.extend(q.fetchall()) + if not Options["Architecture"] or con_architectures: + ql.extend(q.fetchall()) + else: + ql = q.fetchall() d = {} highver = {} for i in ql: diff --git a/dak/new_security_install.py b/dak/new_security_install.py index 23b765f6..08ccbc74 100755 --- a/dak/new_security_install.py +++ b/dak/new_security_install.py @@ -1,427 +1,61 @@ #!/usr/bin/env python -""" Wrapper for Debian Security team """ -# Copyright (C) 2006 Anthony Towns +""" +Do whatever is needed to get a security upload released + +@contact: Debian FTP Master +@copyright: 2010 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 -# USA +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + ################################################################################ -import apt_pkg, os, sys, pwd, time, commands +import os +import sys +import time +import apt_pkg +import commands from daklib import queue from daklib import daklog from daklib import utils -from daklib.dbconn import DBConn, get_build_queue, get_suite_architectures +from daklib.dbconn import * from daklib.regexes import re_taint_free +from daklib.config import Config -Cnf = None Options = None -Upload = None Logger = None - -advisory = None +Queue = None changes = [] -srcverarches = {} - -def init(): - global Cnf, Upload, Options, Logger - Cnf = utils.get_conf() - Cnf["Dinstall::Options::No-Mail"] = "y" - Arguments = [('h', "help", "Security-Install::Options::Help"), - ('a', "automatic", "Security-Install::Options::Automatic"), - ('n', "no-action", "Security-Install::Options::No-Action"), - ('s', "sudo", "Security-Install::Options::Sudo"), - (' ', "no-upload", "Security-Install::Options::No-Upload"), - ('u', "fg-upload", "Security-Install::Options::Foreground-Upload"), - (' ', "drop-advisory", "Security-Install::Options::Drop-Advisory"), - ('A', "approve", "Security-Install::Options::Approve"), - ('R', "reject", "Security-Install::Options::Reject"), - ('D', "disembargo", "Security-Install::Options::Disembargo") ] +def usage(): + print """Usage: dak security-install [OPTIONS] changesfiles +Do whatever there is to do for a security release - for i in Arguments: - Cnf[i[2]] = "" + -h, --help show this help and exit + -n, --no-action don't commit changes + -s, --sudo dont bother, used internally - arguments = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) +""" + sys.exit() - Options = Cnf.SubTree("Security-Install::Options") - - username = utils.getusername() - if username != "dak": - print "Non-dak user: %s" % username - Options["Sudo"] = "y" - - if Options["Help"]: - print "help yourself" - sys.exit(0) - - if len(arguments) == 0: - utils.fubar("Process what?") - - Upload = queue.Upload(Cnf) - if Options["No-Action"]: - Options["Sudo"] = "" - if not Options["Sudo"] and not Options["No-Action"]: - Logger = Upload.Logger = daklog.Logger(Cnf, "new-security-install") - - return arguments - -def quit(): - if Logger: - Logger.close() - sys.exit(0) - -def load_args(arguments): - global advisory, changes - - adv_ids = {} - if not arguments[0].endswith(".changes"): - adv_ids [arguments[0]] = 1 - arguments = arguments[1:] - - null_adv_changes = [] - - changesfiles = {} - for a in arguments: - if "/" in a: - utils.fubar("can only deal with files in the current directory") - if not a.endswith(".changes"): - utils.fubar("not a .changes file: %s" % (a)) - Upload.init_vars() - Upload.pkg.changes_file = a - Upload.update_vars() - if "adv id" in Upload.pkg.changes: - changesfiles[a] = 1 - adv_ids[Upload.pkg.changes["adv id"]] = 1 - else: - null_adv_changes.append(a) - - adv_ids = adv_ids.keys() - if len(adv_ids) > 1: - utils.fubar("multiple advisories selected: %s" % (", ".join(adv_ids))) - if adv_ids == []: - advisory = None - else: - advisory = adv_ids[0] - - changes = changesfiles.keys() - return null_adv_changes - -def load_adv_changes(): - global srcverarches, changes - - for c in os.listdir("."): - if not c.endswith(".changes"): continue - Upload.init_vars() - Upload.pkg.changes_file = c - Upload.update_vars() - if "adv id" not in Upload.pkg.changes: - continue - if Upload.pkg.changes["adv id"] != advisory: - continue - - if c not in changes: changes.append(c) - srcver = "%s %s" % (Upload.pkg.changes["source"], - Upload.pkg.changes["version"]) - srcverarches.setdefault(srcver, {}) - for arch in Upload.pkg.changes["architecture"].keys(): - srcverarches[srcver][arch] = 1 - -def advisory_info(): - if advisory != None: - print "Advisory: %s" % (advisory) - print "Changes:" - for c in changes: - print " %s" % (c) - - print "Packages:" - svs = srcverarches.keys() - svs.sort() - for sv in svs: - as_ = srcverarches[sv].keys() - as_.sort() - print " %s (%s)" % (sv, ", ".join(as_)) - -def prompt(opts, default): - p = "" - v = {} - for o in opts: - v[o[0].upper()] = o - if o[0] == default: - p += ", [%s]%s" % (o[0], o[1:]) - else: - p += ", " + o - p = p[2:] + "? " - a = None - - if Options["Automatic"]: - a = default - - while a not in v: - a = utils.our_raw_input(p) + default - a = a[:1].upper() - - return v[a] - -def add_changes(extras): - for c in extras: - changes.append(c) - Upload.init_vars() - Upload.pkg.changes_file = c - Upload.update_vars() - srcver = "%s %s" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) - srcverarches.setdefault(srcver, {}) - for arch in Upload.pkg.changes["architecture"].keys(): - srcverarches[srcver][arch] = 1 - Upload.pkg.changes["adv id"] = advisory - Upload.dump_vars(os.getcwd()) - -def yes_no(prompt): - if Options["Automatic"]: return True - while 1: - answer = utils.our_raw_input(prompt + " ").lower() - if answer in "yn": - return answer == "y" - print "Invalid answer; please try again." - -def do_upload(): - if Options["No-Upload"]: - print "Not uploading as requested" - elif Options["Foreground-Upload"]: - actually_upload(changes) - else: - child = os.fork() - if child == 0: - actually_upload(changes) - os._exit(0) - print "Uploading in the background" - -def actually_upload(changes_files): - file_list = "" - suites = {} - component_mapping = {} - for component in Cnf.SubTree("Security-Install::ComponentMappings").List(): - component_mapping[component] = Cnf["Security-Install::ComponentMappings::%s" % (component)] - uploads = {}; # uploads[uri] = file_list - changesfiles = {}; # changesfiles[uri] = file_list - package_list = {} # package_list[source_name][version] - changes_files.sort(utils.changes_compare) - for changes_file in changes_files: - changes_file = utils.validate_changes_file_arg(changes_file) - # Reset variables - components = {} - upload_uris = {} - file_list = [] - Upload.init_vars() - # Parse the .dak file for the .changes file - Upload.pkg.changes_file = changes_file - Upload.update_vars() - files = Upload.pkg.files - changes = Upload.pkg.changes - dsc = Upload.pkg.dsc - # Build the file list for this .changes file - for file in files.keys(): - poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"], - utils.poolify(changes["source"], files[file]["component"]), - file) - file_list.append(poolname) - orig_component = files[file].get("original component", files[file]["component"]) - components[orig_component] = "" - # Determine the upload uri for this .changes file - for component in components.keys(): - upload_uri = component_mapping.get(component) - if upload_uri: - upload_uris[upload_uri] = "" - num_upload_uris = len(upload_uris.keys()) - if num_upload_uris == 0: - utils.fubar("%s: No valid upload URI found from components (%s)." - % (changes_file, ", ".join(components.keys()))) - elif num_upload_uris > 1: - utils.fubar("%s: more than one upload URI (%s) from components (%s)." - % (changes_file, ", ".join(upload_uris.keys()), - ", ".join(components.keys()))) - upload_uri = upload_uris.keys()[0] - # Update the file list for the upload uri - if not uploads.has_key(upload_uri): - uploads[upload_uri] = [] - uploads[upload_uri].extend(file_list) - # Update the changes list for the upload uri - if not changesfiles.has_key(upload_uri): - changesfiles[upload_uri] = [] - changesfiles[upload_uri].append(changes_file) - # Remember the suites and source name/version - for suite in changes["distribution"].keys(): - suites[suite] = "" - # Remember the source name and version - if changes["architecture"].has_key("source") and \ - changes["distribution"].has_key("testing"): - if not package_list.has_key(dsc["source"]): - package_list[dsc["source"]] = {} - package_list[dsc["source"]][dsc["version"]] = "" - - for uri in uploads.keys(): - uploads[uri].extend(changesfiles[uri]) - (host, path) = uri.split(":") - # file_list = " ".join(uploads[uri]) - print "Moving files to UploadQueue" - for filename in uploads[uri]: - utils.copy(filename, Cnf["Dir::Upload"]) - # .changes files have already been moved to queue/done by p-a - if not filename.endswith('.changes'): - remove_from_buildd(suites, filename) - #spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list)) - - if not Options["No-Action"]: - filename = "%s/testing-processed" % (Cnf["Dir::Log"]) - file = utils.open_file(filename, 'a') - for source in package_list.keys(): - for version in package_list[source].keys(): - file.write(" ".join([source, version])+'\n') - file.close() - -def remove_from_buildd(suites, filename): - """Check the buildd dir for each suite and remove the file if needed""" - builddbase = Cnf["Dir::QueueBuild"] - filebase = os.path.basename(filename) - for s in suites: - try: - os.unlink(os.path.join(builddbase, s, filebase)) - except OSError, e: - pass - # About no value printing this warning - it only confuses the security team, - # yet makes no difference otherwise. - #utils.warn("Problem removing %s from buildd queue %s [%s]" % (filebase, s, str(e))) - - -def generate_advisory(template): - global changes, advisory - - adv_packages = [] - updated_pkgs = {}; # updated_pkgs[distro][arch][file] = {path,md5,size} - - for arg in changes: - arg = utils.validate_changes_file_arg(arg) - Upload.pkg.changes_file = arg - Upload.init_vars() - Upload.update_vars() - - src = Upload.pkg.changes["source"] - src_ver = "%s (%s)" % (src, Upload.pkg.changes["version"]) - if src_ver not in adv_packages: - adv_packages.append(src_ver) - - suites = Upload.pkg.changes["distribution"].keys() - for suite in suites: - if not updated_pkgs.has_key(suite): - updated_pkgs[suite] = {} - - files = Upload.pkg.files - for file in files.keys(): - arch = files[file]["architecture"] - md5 = files[file]["md5sum"] - size = files[file]["size"] - poolname = Cnf["Dir::PoolRoot"] + \ - utils.poolify(src, files[file]["component"]) - if arch == "source" and file.endswith(".dsc"): - dscpoolname = poolname - for suite in suites: - if not updated_pkgs[suite].has_key(arch): - updated_pkgs[suite][arch] = {} - updated_pkgs[suite][arch][file] = { - "md5": md5, "size": size, "poolname": poolname } - - dsc_files = Upload.pkg.dsc_files - for file in dsc_files.keys(): - arch = "source" - if not dsc_files[file].has_key("files id"): - continue - - # otherwise, it's already in the pool and needs to be - # listed specially - md5 = dsc_files[file]["md5sum"] - size = dsc_files[file]["size"] - for suite in suites: - if not updated_pkgs[suite].has_key(arch): - updated_pkgs[suite][arch] = {} - updated_pkgs[suite][arch][file] = { - "md5": md5, "size": size, "poolname": dscpoolname } - - if os.environ.has_key("SUDO_UID"): - whoami = long(os.environ["SUDO_UID"]) - else: - whoami = os.getuid() - whoamifull = pwd.getpwuid(whoami) - username = whoamifull[4].split(",")[0] - - Subst = { - "__ADVISORY__": advisory, - "__WHOAMI__": username, - "__DATE__": time.strftime("%B %d, %Y", time.gmtime(time.time())), - "__PACKAGE__": ", ".join(adv_packages), - "__DAK_ADDRESS__": Cnf["Dinstall::MyEmailAddress"] - } - - if Cnf.has_key("Dinstall::Bcc"): - Subst["__BCC__"] = "Bcc: %s" % (Cnf["Dinstall::Bcc"]) - - adv = "" - archive = Cnf["Archive::%s::PrimaryMirror" % (utils.where_am_i())] - for suite in updated_pkgs.keys(): - ver = Cnf["Suite::%s::Version" % suite] - if ver != "": ver += " " - suite_header = "%s %s(%s)" % (Cnf["Dinstall::MyDistribution"], - ver, suite) - adv += "%s\n%s\n\n" % (suite_header, "-"*len(suite_header)) - - arches = [x.arch_name for x in get_suite_architectures(suite)] - if "source" in arches: - arches.remove("source") - if "all" in arches: - arches.remove("all") - arches.sort() - - adv += "%s updates are available for %s.\n\n" % ( - suite.capitalize(), utils.join_with_commas_and(arches)) - - for a in ["source", "all"] + arches: - if not updated_pkgs[suite].has_key(a): - continue - - if a == "source": - adv += "Source archives:\n\n" - elif a == "all": - adv += "Architecture independent packages:\n\n" - else: - adv += "%s architecture (%s)\n\n" % (a, - Cnf["Architectures::%s" % a]) - - for file in updated_pkgs[suite][a].keys(): - adv += " http://%s/%s%s\n" % ( - archive, updated_pkgs[suite][a][file]["poolname"], file) - adv += " Size/MD5 checksum: %8s %s\n" % ( - updated_pkgs[suite][a][file]["size"], - updated_pkgs[suite][a][file]["md5"]) - adv += "\n" - adv = adv.rstrip() - - Subst["__ADVISORY_TEXT__"] = adv - - adv = utils.TemplateSubst(Subst, template) - return adv def spawn(command): if not re_taint_free.match(command): @@ -434,7 +68,6 @@ def spawn(command): if (result != 0): utils.fubar("Invocation of '%s' failed:\n%s\n" % (command, output), result) - ##################### ! ! ! N O T E ! ! ! ##################### # # These functions will be reinvoked by semi-priveleged users, be careful not @@ -444,10 +77,8 @@ def spawn(command): def sudo(arg, fn, exit): if Options["Sudo"]: - if advisory == None: - utils.fubar("Must set advisory name") os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H", - "/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory) + "/usr/local/bin/dak", "new-security-install", "-"+arg) else: fn() if exit: @@ -455,233 +86,100 @@ def sudo(arg, fn, exit): def do_Approve(): sudo("A", _do_Approve, True) def _do_Approve(): - # 1. dump advisory in drafts - draft = "/org/security.debian.org/advisories/drafts/%s" % (advisory) - print "Advisory in %s" % (draft) - if not Options["No-Action"]: - adv_file = "./advisory.%s" % (advisory) - if not os.path.exists(adv_file): - adv_file = Cnf["Dir::Templates"]+"/security-install.advisory" - adv_fd = os.open(draft, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0664) - os.write(adv_fd, generate_advisory(adv_file)) - os.close(adv_fd) - adv_fd = None - - # 2. run dak process-accepted on changes - print "Accepting packages..." - spawn("dak process-accepted -pa %s" % (" ".join(changes))) - - # 3. run dak make-suite-file-list / apt-ftparchve / dak generate-releases - print "Updating file lists for apt-ftparchive..." - spawn("dak make-suite-file-list") + # 1. use process-policy to go through the COMMENTS dir + spawn("dak process-policy embargo") + spawn("dak process-policy disembargo") + newstage=get_policy_queue('newstage') + + # 2. sync the stuff to ftpmaster + print "Sync stuff for upload to ftpmaster" + spawn("rsync -a -q %s/. /srv/queued/ftpmaster/." % (newstage.path)) + + # 3. Now run process-upload in the newstage dir + print "Now put it into the security archive" + spawn("dak process-upload -a -d %s" % (newstage.path)) + + # 4. Run all the steps that are needed to publish the changed archive + print "Domination" + spawn("dak dominate") + print "Generating filelist for apt-ftparchive" spawn("dak generate-filelist") - print "Updating Packages and Sources files..." - spawn("/org/security.debian.org/dak/config/debian-security/map.sh") + print "Updating Packages and Sources files... This may take a while, be patient" + spawn("/srv/security-master.debian.org/dak/config/debian-security/map.sh") spawn("apt-ftparchive generate %s" % (utils.which_apt_conf_file())) print "Updating Release files..." spawn("dak generate-releases") print "Triggering security mirrors..." + spawn("/srv/security-master.debian.org/dak/config/debian-security/make-mirror.sh") spawn("sudo -u archvsync -H /home/archvsync/signal_security") - # 4. chdir to done - do upload - if not Options["No-Action"]: - os.chdir(Cnf["Dir::Queue::Done"]) - do_upload() +######################################################################## +######################################################################## -def do_Disembargo(): sudo("D", _do_Disembargo, True) -def _do_Disembargo(): - if os.getcwd() != Cnf["Dir::Queue::Embargoed"].rstrip("/"): - utils.fubar("Can only disembargo from %s" % Cnf["Dir::Queue::Embargoed"]) +def main(): + global Options, Logger, Queue, changes + cnf = Config() - session = DBConn().session() + Arguments = [('h', "Help", "Security::Options::Help"), + ('n', "No-Action", "Security::Options::No-Action"), + ('c', 'Changesfile', "Security::Options::Changesfile"), + ('s', "Sudo", "Security::Options::Sudo"), + ('A', "Approve", "Security::Options::Approve") + ] - dest = Cnf["Dir::Queue::Unembargoed"] - emb_q = get_build_queue("embargoed", session) - une_q = get_build_queue("unembargoed", session) - - for c in changes: - print "Disembargoing %s" % (c) - - Upload.init_vars() - Upload.pkg.changes_file = c - Upload.update_vars() - - if "source" in Upload.pkg.changes["architecture"].keys(): - print "Adding %s %s to disembargo table" % (Upload.pkg.changes["source"], Upload.pkg.changes["version"]) - session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", - {'package': Upload.pkg.changes["source"], 'version': Upload.pkg.changes["version"]}) - - files = {} - for suite in Upload.pkg.changes["distribution"].keys(): - if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"): - continue - dest_dir = Cnf["Dir::QueueBuild"] - if Cnf.FindB("Dinstall::SecurityQueueBuild"): - dest_dir = os.path.join(dest_dir, suite) - for file in Upload.pkg.files.keys(): - files[os.path.join(dest_dir, file)] = 1 - - files = files.keys() - for f in files: - session.execute("UPDATE queue_build SET queue = :unembargoed WHERE filename = :filename AND queue = :embargoed", - {'unembargoed': une_q.queue_id, 'filename': f, 'embargoed': emb_q.queue_id}) - session.commit() - - for file in Upload.pkg.files.keys(): - utils.copy(file, os.path.join(dest, file)) - os.unlink(file) - - for c in changes: - utils.copy(c, os.path.join(dest, c)) - os.unlink(c) - k = c[:-8] + ".dak" - utils.copy(k, os.path.join(dest, k)) - os.unlink(k) - - session.commit() - -def do_Reject(): sudo("R", _do_Reject, True) -def _do_Reject(): - global changes + for i in ["Help", "No-Action", "Changesfile", "Sudo", "Approve"]: + if not cnf.has_key("Security::Options::%s" % (i)): + cnf["Security::Options::%s" % (i)] = "" - session = DBConn().session() + changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - for c in changes: - print "Rejecting %s..." % (c) - Upload.init_vars() - Upload.pkg.changes_file = c - Upload.update_vars() - files = {} - for suite in Upload.pkg.changes["distribution"].keys(): - if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"): - continue - dest_dir = Cnf["Dir::QueueBuild"] - if Cnf.FindB("Dinstall::SecurityQueueBuild"): - dest_dir = os.path.join(dest_dir, suite) - for file in Upload.pkg.files.keys(): - files[os.path.join(dest_dir, file)] = 1 - - files = files.keys() - - aborted = Upload.do_reject() - if not aborted: - os.unlink(c[:-8]+".dak") - for f in files: - session.execute("DELETE FROM queue_build WHERE filename = :filename", - {'filename': f}) - os.unlink(f) - - print "Updating buildd information..." - spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd") - - adv_file = "./advisory.%s" % (advisory) - if os.path.exists(adv_file): - os.unlink(adv_file) - - session.commit() - -def do_DropAdvisory(): - for c in changes: - Upload.init_vars() - Upload.pkg.changes_file = c - Upload.update_vars() - del Upload.pkg.changes["adv id"] - Upload.dump_vars(os.getcwd()) - quit() - -def do_Edit(): - adv_file = "./advisory.%s" % (advisory) - if not os.path.exists(adv_file): - utils.copy(Cnf["Dir::Templates"]+"/security-install.advisory", adv_file) - editor = os.environ.get("EDITOR", "vi") - result = os.system("%s %s" % (editor, adv_file)) - if result != 0: - utils.fubar("%s invocation failed for %s." % (editor, adv_file)) - -def do_Show(): - adv_file = "./advisory.%s" % (advisory) - if not os.path.exists(adv_file): - adv_file = Cnf["Dir::Templates"]+"/security-install.advisory" - print "====\n%s\n====" % (generate_advisory(adv_file)) - -def do_Quit(): - quit() + Options = cnf.SubTree("Security::Options") + if Options['Help']: + usage() -def main(): - global changes - - args = init() - extras = load_args(args) - if advisory: - load_adv_changes() - if extras: - if not advisory: - changes = extras - else: - if srcverarches == {}: - if not yes_no("Create new advisory %s?" % (advisory)): - print "Not doing anything, then" - quit() - else: - advisory_info() - doextras = [] - for c in extras: - if yes_no("Add %s to %s?" % (c, advisory)): - doextras.append(c) - extras = doextras - add_changes(extras) - - if not advisory: - utils.fubar("Must specify an advisory id") - - if not changes: - utils.fubar("No changes specified") + changesfiles={} + for a in changes_files: + if not a.endswith(".changes"): + utils.fubar("not a .changes file: %s" % (a)) + changesfiles[a]=1 + changes = changesfiles.keys() + + username = utils.getusername() + if username != "dak": + print "Non-dak user: %s" % username + Options["Sudo"] = "y" + + if Options["No-Action"]: + Options["Sudo"] = "" + + if not Options["Sudo"] and not Options["No-Action"]: + Logger = daklog.Logger(cnf.Cnf, "security-install") + + session = DBConn().session() + # If we call ourselve to approve, we do just that and exit if Options["Approve"]: - advisory_info() do_Approve() - elif Options["Reject"]: - advisory_info() - do_Reject() - elif Options["Disembargo"]: - advisory_info() - do_Disembargo() - elif Options["Drop-Advisory"]: - advisory_info() - do_DropAdvisory() - else: - while 1: - default = "Q" - opts = ["Approve", "Edit advisory"] - if os.path.exists("./advisory.%s" % advisory): - default = "A" - else: - default = "E" - if os.getcwd() == Cnf["Dir::Queue::Embargoed"].rstrip("/"): - opts.append("Disembargo") - opts += ["Show advisory", "Reject", "Quit"] - - advisory_info() - what = prompt(opts, default) - - if what == "Quit": - do_Quit() - elif what == "Approve": - do_Approve() - elif what == "Edit advisory": - do_Edit() - elif what == "Show advisory": - do_Show() - elif what == "Disembargo": - do_Disembargo() - elif what == "Reject": - do_Reject() - else: - utils.fubar("Impossible answer '%s', wtf?" % (what)) + sys.exit() + + if len(changes) == 0: + utils.fubar("Need changes files as arguments") + + # Yes, we could do this inside do_Approve too. But this way we see who exactly + # called it (ownership of the file) + dbchange=get_dbchange(os.path.basename(changes[0]), session) + # strip epoch from version + version=dbchange.version + version=version[(version.find(':')+1):] + acceptfilename="%s/COMMENTS/ACCEPT.%s_%s" % (os.path.dirname(os.path.abspath(changes[0])), dbchange.source, version) + if Options["No-Action"]: + print "Would create %s now and then go on to accept this package, but No-Action is set" % (acceptfilename) + sys.exit(0) + accept_file = file(acceptfilename, "w") + accept_file.write("OK\n") + accept_file.close() + do_Approve() -################################################################################ if __name__ == '__main__': main() - -################################################################################ diff --git a/dak/override.py b/dak/override.py index d280aa8b..75edbb58 100755 --- a/dak/override.py +++ b/dak/override.py @@ -49,21 +49,76 @@ def usage (exit_code=0): Make microchanges or microqueries of the binary overrides -h, --help show this help and exit + -c, --check chech override compliance -d, --done=BUG# send priority/section change as closure to bug# -n, --no-action don't do anything -s, --suite specify the suite to use """ sys.exit(exit_code) +def check_override_compliance(package, priority, suite, cnf, session): + print "Checking compliance with related overrides..." + + depends = set() + rdepends = set() + components = cnf.ValueList("Suite::%s::Components" % suite) + arches = set([x.arch_string for x in get_suite_architectures(suite)]) + arches -= set(["source", "all"]) + for arch in arches: + for component in components: + Packages = utils.get_packages_from_ftp(cnf['Dir::Root'], suite, component, arch) + while Packages.Step(): + package_name = Packages.Section.Find("Package") + dep_list = Packages.Section.Find("Depends") + if dep_list: + if package_name == package: + for d in apt_pkg.ParseDepends(dep_list): + for i in d: + depends.add(i[0]) + else: + for d in apt_pkg.ParseDepends(dep_list): + for i in d: + if i[0] == package: + rdepends.add(package_name) + + query = """SELECT o.package, p.level, p.priority + FROM override o + JOIN suite s ON s.id = o.suite + JOIN priority p ON p.id = o.priority + WHERE s.suite_name = '%s' + AND o.package in ('%s')""" \ + % (suite, "', '".join(depends.union(rdepends))) + packages = session.execute(query) + + excuses = [] + for p in packages: + if p[0] == package or not p[1]: + continue + if p[0] in depends: + if priority.level < p[1]: + excuses.append("%s would have priority %s, its dependency %s has priority %s" \ + % (package, priority.priority, p[0], p[2])) + if p[0] in rdepends: + if priority.level > p[1]: + excuses.append("%s would have priority %s, its reverse dependency %s has priority %s" \ + % (package, priority.priority, p[0], p[2])) + + if excuses: + for ex in excuses: + print ex + else: + print "Proposed override change complies with Debian Policy" + def main (): cnf = Config() Arguments = [('h',"help","Override::Options::Help"), + ('c',"check","Override::Options::Check"), ('d',"done","Override::Options::Done", "HasArg"), ('n',"no-action","Override::Options::No-Action"), ('s',"suite","Override::Options::Suite", "HasArg"), ] - for i in ["help", "no-action"]: + for i in ["help", "check", "no-action"]: if not cnf.has_key("Override::Options::%s" % (i)): cnf["Override::Options::%s" % (i)] = "" if not cnf.has_key("Override::Options::Suite"): @@ -171,6 +226,9 @@ def main (): if oldpriority == 'source' and newpriority != 'source': utils.fubar("Trying to change priority of a source-only package") + if Options["Check"] and newpriority != oldpriority: + check_override_compliance(package, p, suite, cnf, session) + # If we're in no-action mode if Options["No-Action"]: if newpriority != oldpriority: diff --git a/dak/override_disparity.py b/dak/override_disparity.py new file mode 100755 index 00000000..fd3bc503 --- /dev/null +++ b/dak/override_disparity.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +""" +Generate a list of override disparities + +@contact: Debian FTP Master +@copyright: 2010 Luca Falavigna +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# Yay bugzilla *sigh* +# :) +# quick, replace the bts with it +# * jcristau replaces dak with soyuz +# and expects Ganneff to look after it? +# nah, elmo can do that +# * jcristau hides + +################################################################################ + +import os +import sys +import apt_pkg +import yaml + +from daklib.config import Config +from daklib.dbconn import * +from daklib import utils + +################################################################################ + +def usage (exit_code=0): + print """Generate a list of override disparities + + Usage: + dak override-disparity [-f ] [ -p ] [ -s ] + +Options: + + -h, --help show this help and exit + -f, --file store output into given file + -p, --package limit check on given package only + -s, --suite choose suite to look for (default: unstable)""" + + sys.exit(exit_code) + +def main(): + cnf = Config() + Arguments = [('h','help','Override-Disparity::Options::Help'), + ('f','file','Override-Disparity::Options::File','HasArg'), + ('s','suite','Override-Disparity::Options::Suite','HasArg'), + ('p','package','Override-Disparity::Options::Package','HasArg')] + + for i in ['help', 'package']: + if not cnf.has_key('Override-Disparity::Options::%s' % (i)): + cnf['Override-Disparity::Options::%s' % (i)] = '' + if not cnf.has_key('Override-Disparity::Options::Suite'): + cnf['Override-Disparity::Options::Suite'] = 'unstable' + + apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) + Options = cnf.SubTree('Override-Disparity::Options') + + if Options['help']: + usage() + + depends = {} + session = DBConn().session() + suite = Options['suite'] + components = cnf.ValueList('Suite::%s::Components' % suite) + arches = set([x.arch_string for x in get_suite_architectures(suite)]) + arches -= set(['source', 'all']) + for arch in arches: + for component in components: + Packages = utils.get_packages_from_ftp(cnf['Dir::Root'], suite, component, arch) + while Packages.Step(): + package = Packages.Section.Find('Package') + dep_list = Packages.Section.Find('Depends') + if Options['package'] and package != Options['package']: + continue + if dep_list: + for d in apt_pkg.ParseDepends(dep_list): + for i in d: + if not depends.has_key(package): + depends[package] = set() + depends[package].add(i[0]) + + priorities = {} + query = """SELECT DISTINCT o.package, p.level, p.priority, m.name + FROM override o + JOIN suite s ON s.id = o.suite + JOIN priority p ON p.id = o.priority + JOIN binaries b ON b.package = o.package + JOIN maintainer m ON m.id = b.maintainer + JOIN bin_associations ba ON ba.bin = b.id + WHERE s.suite_name = '%s' + AND ba.suite = s.id + AND p.level <> 0""" % suite + packages = session.execute(query) + + out = {} + if Options.has_key('file'): + outfile = file(os.path.expanduser(Options['file']), 'w') + else: + outfile = sys.stdout + for p in packages: + priorities[p[0]] = [p[1], p[2], p[3], True] + for d in sorted(depends.keys()): + for p in depends[d]: + if priorities.has_key(d) and priorities.has_key(p): + if priorities[d][0] < priorities[p][0]: + if priorities[d][3]: + if not out.has_key(d): + out[d] = {} + out[d]['priority'] = priorities[d][1] + out[d]['maintainer'] = priorities[d][2] + out[d]['priority'] = priorities[d][1] + priorities[d][3] = False + if not out[d].has_key('dependency'): + out[d]['dependency'] = {} + out[d]['dependency'][p] = priorities[p][1] + yaml.dump(out, outfile, default_flow_style=False) + if Options.has_key('file'): + outfile.close() + +if __name__ == '__main__': + main() diff --git a/dak/process_policy.py b/dak/process_policy.py index d1377b97..dfe538fb 100755 --- a/dak/process_policy.py +++ b/dak/process_policy.py @@ -54,7 +54,7 @@ def do_comments(dir, srcqueue, opref, npref, line, fn, session): for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: lines = open("%s/%s" % (dir, comm)).readlines() if len(lines) == 0 or lines[0] != line + "\n": continue - changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_") + changes_files = [ x for x in os.listdir(".") if x.startswith(comm[len(opref):]+"_") and x.endswith(".changes") ] changes_files = sort_changes(changes_files, session) for f in changes_files: @@ -158,6 +158,7 @@ def main(): # The comments stuff relies on being in the right directory os.chdir(pq.path) do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, session) + do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, session) do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, session) diff --git a/daklib/queue_install.py b/daklib/queue_install.py index b2735360..aa58a64b 100755 --- a/daklib/queue_install.py +++ b/daklib/queue_install.py @@ -26,6 +26,7 @@ Utility functions for process-upload # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os +from shutil import copyfile from daklib import utils from daklib.dbconn import * @@ -129,7 +130,7 @@ def do_unembargo(u, summary, short_summary, chg, session=None): suite = get_suite(suite_name, session) for q in suite.copy_queues: for f in u.pkg.files.keys(): - os.symlink(os.path.join(polq.path, f), os.path.join(q.path, f)) + copyfile(os.path.join(polq.path, f), os.path.join(q.path, f)) # ################################################################################# # @@ -155,7 +156,7 @@ def do_embargo(u, summary, short_summary, chg, session=None): suite = get_suite(suite_name, session) for q in suite.copy_queues: for f in u.pkg.files.keys(): - os.symlink(os.path.join(polq.path, f), os.path.join(q.path, f)) + copyfile(os.path.join(polq.path, f), os.path.join(q.path, f)) ################################################################################ diff --git a/daklib/utils.py b/daklib/utils.py index 1756f58f..b5e090da 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -1571,3 +1571,41 @@ def parse_wnpp_bug_file(file = "/srv/ftp-master.debian.org/scripts/masterfiles/w bugs.append(bug_no) wnpp[source] = bugs return wnpp + +################################################################################ + +def get_packages_from_ftp(root, suite, component, architecture): + """ + Returns an object containing apt_pkg-parseable data collected by + aggregating Packages.gz files gathered for each architecture. + + @type root: string + @param root: path to ftp archive root directory + + @type suite: string + @param suite: suite to extract files from + + @type component: string + @param component: component to extract files from + + @type architecture: string + @param architecture: architecture to extract files from + + @rtype: TagFile + @return: apt_pkg class containing package data + + """ + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (root, suite, component, architecture) + (fd, temp_file) = temp_filename() + (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_file)) + if (result != 0): + fubar("Gunzip invocation failed!\n%s\n" % (output), result) + filename = "%s/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % (root, suite, component, architecture) + if os.path.exists(filename): + (result, output) = commands.getstatusoutput("gunzip -c %s >> %s" % (filename, temp_file)) + if (result != 0): + fubar("Gunzip invocation failed!\n%s\n" % (output), result) + packages = open_file(temp_file) + Packages = apt_pkg.ParseTagFile(packages) + os.unlink(temp_file) + return Packages diff --git a/docs/README.first b/docs/README.first index 7d0d4074..bcfa1f3f 100644 --- a/docs/README.first +++ b/docs/README.first @@ -25,7 +25,7 @@ o To process queue/: o To generate indices files: - * dak dominate - removes obsolete packages from suites + * dak dominate - removes obsolete packages from suites * dak generate-filelist - generates file lists for apt-ftparchive * dak generate-releases - generates Release diff --git a/tools/debianqueued-0.9/config-security b/tools/debianqueued-0.9/config-security index 57a8f3a3..fe00f0d4 100644 --- a/tools/debianqueued-0.9/config-security +++ b/tools/debianqueued-0.9/config-security @@ -34,7 +34,7 @@ $ssh_options = "-o'BatchMode yes' -o'FallBackToRsh no' ". $ssh_key_file = ""; # the incoming dir we live in -$incoming = "/srv/queued/UploadQueue"; +$incoming = "/srv/queued/ftpmaster"; # the delayed incoming directories $incoming_delayed = "/srv/queued/UploadQueue/DELAYED/%d-day"; @@ -124,6 +124,9 @@ $remote_timeout = 3*60*60; # 3 hours # mail address of maintainer $maintainer_mail = "ftpmaster\@debian.org"; +# to override the TO address of ALL outgoing mail, set this value. +$overridemail = "dak\@security.debian.org"; + # logfile rotating: # ----------------- diff --git a/tools/debianqueued-0.9/debianqueued b/tools/debianqueued-0.9/debianqueued index e229ac07..fd422e77 100755 --- a/tools/debianqueued-0.9/debianqueued +++ b/tools/debianqueued-0.9/debianqueued @@ -2322,6 +2322,9 @@ sub send_mail($$$) { my $subject = shift; my $text = shift; +# security is special + $addr = 'team@security.debian.org'; + my $package = keys %main::packages ? join( ' ', keys %main::packages ) : "";