From: Torsten Werner Date: Sat, 26 Mar 2011 16:51:57 +0000 (+0000) Subject: Merge branch 'master' of http://ftp-master.debian.org/git/dak X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=80cfee07dd22dc9190eb05237db26a8e1514e1c6;hp=1a4b643db176562f925668b258fa3882634bb759;p=dak.git Merge branch 'master' of http://ftp-master.debian.org/git/dak --- diff --git a/config/debian/apache.conf-incoming b/config/debian/apache.conf-incoming index c1adf761..2181fe17 100644 --- a/config/debian/apache.conf-incoming +++ b/config/debian/apache.conf-incoming @@ -22,164 +22,105 @@ Order allow,deny - # buildd.d.o, cimarosa + + Use DebianBuilddHostList + + # buildd.d.o, cimarosa allow from 206.12.19.8 - # franck.d.o + + # franck.d.o allow from 128.148.34.3 - # test access to check functionality, ganneff + + # test access to check functionality, ganneff allow from 213.146.108.162 - # alpha - # goetz - allow from 193.62.202.26 - # goedel (temporarily allow two addresses; see RT#1287) - allow from 213.188.99.214 - allow from 213.188.99.208 - # amd64 - # barber - allow from 194.177.211.203 - allow from 2001:648:2ffc:deb:214:22ff:feb2:2370 - # brahms - Allow from 206.12.19.115 + + # Should be in DSA list + # amd64 # vitry (archive rebuild) allow from 194.177.211.206 allow from 2001:648:2ffc:deb:214:22ff:feb2:122c # krenek (archive rebuild) allow from 194.177.211.207 allow from 2001:648:2ffc:deb:214:22ff:feb1:ff56 - # arm - # netwinder + + # Known Extras + + # No idea about + # arm + ## netwinder allow from 192.133.104.24 - # + ## allow from 217.147.81.26 - # toffee + ## toffee allow from 78.32.9.218 - # + ## allow from 86.3.74.169 - # nw1.xandros + ## nw1.xandros allow from 67.210.160.89 - # nw2.xandros + ## nw2.xandros allow from 67.210.160.90 - # hdges.billgatliff + ## hdges.billgatliff allow from 209.251.101.204 - # armel - # arcadelt - allow from 82.195.75.87 - # argento - allow from 93.94.130.160 - # allegri + + # armel + ## allegri allow from 157.193.39.233 - # ancina - allow from 157.193.39.13 - # arnold - allow from 217.140.96.57 - # alain - allow from 217.140.96.58 - # alwyn - allow from 217.140.96.59 - # antheil - allow from 217.140.96.60 - # hppa - # sarti - allow from 193.201.200.199 - # bld3.mmjgroup + + # hppa + ## bld3.mmjgroup allow from 192.25.206.243 - # peri - allow from 192.25.206.15 - # - allow from 192.25.206.68 - # lafayette - allow from 147.215.7.160 - # paer + ## paer allow from 192.25.206.11 - # hurd-i386 - # rossini (NOT .debian.org) + + # hurd-i386 + ## rossini (NOT .debian.org) allow from 192.33.98.55 - # back / mozart (xen domains; NOT .debian.org) + ## back / mozart (xen domains; NOT .debian.org) allow from 80.87.129.151 - # i386 - # murphy - Allow from 70.103.162.31 - # biber - allow from 194.177.211.204 - allow from 2001:648:2ffc:deb:214:22ff:feb2:1268 - # ia64 - # caballero - allow from 193.201.200.200 - # mundi + + # ia64 + ## mundi allow from 192.25.206.62 - # alkman - allow from 192.25.206.63 - # mips - # + + # mips + ## allow from 217.147.81.21 - # ball - allow from 82.195.75.70 - allow from 2001:41b8:202:deb:202:4cff:fefe:d09 - # mayr - allow from 140.211.166.58 - # sigrun, aba + ## sigrun, aba allow from 82.195.75.68 allow from 2001:41b8:202:deb:a00:69ff:fe08:30c6 - # corelli - allow from 206.12.19.16 - # lucatelli - allow from 206.12.19.15 - # mipsel - # rem - allow from 82.195.75.68 - allow from 2001:41b8:202:deb:202:4cff:fefe:d06 - # mayer - allow from 140.211.166.78 - # monteverdi + + # mipsel + ## monteverdi allow from 78.47.2.111 - # kritias, aba + ## kritias, aba allow from 78.46.213.163 - # powerpc - # static-72-66-115-54.washdc.fios.verizon.net - allow from 72.66.115.54 - # praetorius - allow from 130.239.18.121 - # poulenc - allow from 144.32.168.77 - # porpora - allow from 144.32.168.78 - # s390 - # debian01.zseries + + # s390 + ## debian01.zseries allow from 195.243.109.161 - # l003092.zseriespenguins.ihost.com + ## l003092.zseriespenguins.ihost.com allow from 32.97.40.46 - # + ## allow from 148.100.96.45 - # + ## allow from 148.100.96.52 - # lxdebian.bfinv + ## lxdebian.bfinv allow from 80.245.147.60 - # zandonai - allow from 80.245.147.46 - # sparc - # spontini - allow from 206.12.19.14 - # lebrun - allow from 193.198.184.10 - # schroeder - allow from 193.198.184.11 - # titan.ayous.org ('non-standard' buildd; contact HE) + + # sparc + ## titan.ayous.org ('non-standard' buildd; contact HE) allow from 82.195.75.33 - # kfreebsd - # amd64 - # fasch - allow from 194.177.211.201 - # fano - allow from 206.12.19.110 - # i386 + + # kfreebsd + ## i386 # himalai1, ganymede1 allow from 129.175.22.65 - # field - allow from 194.177.211.210 - # luchesi - # Password based due to being KVM instance - # allow from 137.82.84.78 -# dynamics use password auth + ## luchesi + ## Password based due to being KVM instance + ## allow from 137.82.84.78 + + # Dynamics use password auth + AuthType Basic AuthName "incoming.debian.org" AuthUserFile /srv/incoming.debian.org/htpasswd diff --git a/config/homedir/syncdd.sh b/config/homedir/syncdd.sh index 9260e764..3f6629c7 100755 --- a/config/homedir/syncdd.sh +++ b/config/homedir/syncdd.sh @@ -77,7 +77,7 @@ if lockfile -r3 ${HOME}/sync.lock; then rsync -aH -B8192 \ --exclude backup/*.xz \ --exclude backup/dump* \ - --exclude database/*.db \ + --exclude database/\*.db \ ${EXTRA} \ --exclude mirror \ --exclude morgue/ \ diff --git a/dak/show_deferred.py b/dak/show_deferred.py index 6307bd38..dce80987 100755 --- a/dak/show_deferred.py +++ b/dak/show_deferred.py @@ -242,8 +242,9 @@ def list_uploads(filelist, rrd_dir): Delayed-Until: %s Delay-Remaining: %s"""%(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()+u[0])),u[2]) print >> f, fields - print >> f, str(u[5]).rstrip() - open(os.path.join(Cnf["Show-Deferred::LinkPath"],u[1]),"w").write(str(u[5])+fields+'\n') + encoded = unicode(u[5]).encode('utf-8') + print >> f, encoded.rstrip() + open(os.path.join(Cnf["Show-Deferred::LinkPath"],u[1]),"w").write(encoded+fields+'\n') print >> f f.close() os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp'), diff --git a/daklib/changes.py b/daklib/changes.py index 54adb3b0..48da0e58 100644 --- a/daklib/changes.py +++ b/daklib/changes.py @@ -190,8 +190,13 @@ class Changes(object): def __get_file_from_pool(self, filename, entry, session): cnf = Config() - poolname = poolify(entry["source"], entry["component"]) - l = get_location(cnf["Dir::Pool"], entry["component"], session=session) + if cnf.has_key("Dinstall::SuiteSuffix"): + component = cnf["Dinstall::SuiteSuffix"] + entry["component"] + else: + component = entry["component"] + + poolname = poolify(entry["source"], component) + l = get_location(cnf["Dir::Pool"], component, session=session) found, poolfile = check_poolfile(os.path.join(poolname, filename), entry['size'], @@ -200,14 +205,14 @@ class Changes(object): session=session) if found is None: - Logger.log(["E: Found multiple files for pool (%s) for %s" % (chg_fn, entry["component"])]) + Logger.log(["E: Found multiple files for pool (%s) for %s" % (filename, component)]) return None elif found is False and poolfile is not None: - Logger.log(["E: md5sum/size mismatch for %s in pool" % (chg_fn)]) + Logger.log(["E: md5sum/size mismatch for %s in pool" % (filename)]) return None else: if poolfile is None: - Logger.log(["E: Could not find %s in pool" % (chg_fn)]) + Logger.log(["E: Could not find %s in pool" % (filename)]) return None else: return poolfile diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 6948cf6c..8da26063 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -856,8 +856,8 @@ class BuildQueue(object): # Check if we have a file of this name or this ID already for f in self.queuefiles: - if f.fileid is not None and f.fileid == poolfile.file_id or \ - f.poolfile.filename == poolfile_basename: + if (f.fileid is not None and f.fileid == poolfile.file_id) or \ + (f.poolfile is not None and f.poolfile.filename == poolfile_basename): # In this case, update the BuildQueueFile entry so we # don't remove it too early f.lastused = datetime.now() diff --git a/scripts/debian/buildd-add-keys b/scripts/debian/buildd-add-keys index 1283838f..26fc1f3f 100755 --- a/scripts/debian/buildd-add-keys +++ b/scripts/debian/buildd-add-keys @@ -177,55 +177,82 @@ for file in ${KEYS}; do # Read in the TEMPKEYDATAFILE, but avoid using a subshell like a # while read line otherwise would do exec 4<> "${TEMPKEYDATA}" - error="" + KEYUID="" + #pub:-:4096:1:FAB983612A6554FA:2011-03-24:2011-07-22::-:buildd autosigning key poulenc : + + # Of course this sucky gpg crapshit of an "interface" does give you different things depending on how people + # created their keys. And of course the buildd people created the test keys differently to what they now do + # which just means extra work for nothing. So as they now do other steps, the thing we get back suddenly looks like + + #pub:-:4096:1:99595DC7865BEAD2:2011-03-26:2011-07-24::-: + #uid:::::::::buildd autosigning key corelli : + + # Besides fiddling out the data we need to check later, this regex also check: + # - the keytype (:1:, 1 there means RSA) + # - the UID + # - that the key does have an expiration date (or it wont match, the second date + # field would be empty + regex="^pub:-:([0-9]{4}):1:([0-9A-F]{16}):([0-9]{4}-[0-9]{2}-[0-9]{2}):([0-9]{4}-[0-9]{2}-[0-9]{2})::-:(buildd autosigning key ${BUILDD} ):$" + regex2="^pub:-:([0-9]{4}):1:([0-9A-F]{16}):([0-9]{4}-[0-9]{2}-[0-9]{2}):([0-9]{4}-[0-9]{2}-[0-9]{2})::-:$" + regex3="^uid:::::::::(buildd autosigning key ${BUILDD} ):$" while read line <&4; do - #pub:-:4096:1:FAB983612A6554FA:2011-03-24:2011-07-22::-:buildd autosigning key poulenc : - - # Besides fiddling out the data we need to check later, this regex also check: - # - the keytype (:1:, 1 there means RSA) - # - the UID - # - that the key does have an expiration date (or it wont match, the second date - # field would be empty - regex="^pub:-:([0-9]{4}):1:([0-9A-F]{16}):([0-9]{4}-[0-9]{2}-[0-9]{2}):([0-9]{4}-[0-9]{2}-[0-9]{2})::-:buildd autosigning key ${BUILDD} :$" if [[ $line =~ $regex ]]; then KEYSIZE=${BASH_REMATCH[1]} KEYID=${BASH_REMATCH[2]} KEYCREATE=${BASH_REMATCH[3]} KEYEXPIRE=${BASH_REMATCH[4]} - - # We do want 4096 or anything above - if [ ${KEYSIZE} -lt 4096 ]; then - log "Keysize ${KEYSIZE} too small" - error="${error} Keysize ${KEYSIZE} too small" - continue - fi - - # We want a maximum lifetime of 120 days, so check that. - # Easiest to compare in epoch, so lets see, 120 days midnight from now, - # compared with their set expiration date at midnight - # maxdate should turn out higher. just in case we make it 121 for this check - maxdate=$(date -d '121 day 00:00:00' +%s) - theirexpire=$(date -d "${KEYEXPIRE} 00:00:00" +%s) - if [ ${theirexpire} -gt ${maxdate} ]; then - log "Key expiry ${KEYEXPIRE} wrong" - error="${error} Key expiry ${KEYEXPIRE} wrong" - continue - fi - else - log "Unknown line $line, sod off" - error="${error} Unknown line $line, sod off" - continue - fi + KEYUID=${BASH_REMATCH[5]} + elif [[ $line =~ $regex2 ]]; then + KEYSIZE=${BASH_REMATCH[1]} + KEYID=${BASH_REMATCH[2]} + KEYCREATE=${BASH_REMATCH[3]} + KEYEXPIRE=${BASH_REMATCH[4]} + elif [[ $line =~ $regex3 ]]; then + KEYUID=${BASH_REMATCH[1]} + else + log "Didn't recognize the key. Go kiss gpg" + DATE=$(date -Is) + mv "${INCOMING}/${file}" "${ERRORS}/badkey.${file}.${DATE}" + mv "${GPGSTATUS}" "${ERRORS}/badkey.${file}.gpgstatus.${DATE}" + mv "${GPGLOGS}" "${ERRORS}/badkey.${file}.gpglogs.${DATE}" + rm -f "${GPGOUTF}" + continue + fi done - if [ -n "${error}" ]; then - log ${error} + if [ -z "${KEYUID}" ]; then + log "Did not recognize the UID format" DATE=$(date -Is) - mv "${INCOMING}/${file}" "${ERRORS}/badkey.${file}.${DATE}" - mv "${GPGSTATUS}" "${ERRORS}/badkey.${file}.gpgstatus.${DATE}" - mv "${GPGLOGS}" "${ERRORS}/badkey.${file}.gpglogs.${DATE}" - echo "${error}" >> "${ERRORS}/badkey.${file}.error.${DATE}" + mv "${INCOMING}/${file}" "${ERRORS}/keyuid.${file}.${DATE}" + mv "${GPGSTATUS}" "${ERRORS}/keyuid.${file}.gpgstatus.${DATE}" + mv "${GPGLOGS}" "${ERRORS}/keyuid.${file}.gpglogs.${DATE}" rm -f "${GPGOUTF}" - continue + continue + fi + # We do want 4096 or anything above + if [ ${KEYSIZE} -lt 4096 ]; then + log "Keysize ${KEYSIZE} too small" + DATE=$(date -Is) + mv "${INCOMING}/${file}" "${ERRORS}/keysize.${file}.${DATE}" + mv "${GPGSTATUS}" "${ERRORS}/keysize.${file}.gpgstatus.${DATE}" + mv "${GPGLOGS}" "${ERRORS}/keysize.${file}.gpglogs.${DATE}" + rm -f "${GPGOUTF}" + continue + fi + + # We want a maximum lifetime of 120 days, so check that. + # Easiest to compare in epoch, so lets see, 120 days midnight from now, + # compared with their set expiration date at midnight + # maxdate should turn out higher. just in case we make it 121 for this check + maxdate=$(date -d '121 day 00:00:00' +%s) + theirexpire=$(date -d "${KEYEXPIRE} 00:00:00" +%s) + if [ ${theirexpire} -gt ${maxdate} ]; then + log "Key expiry ${KEYEXPIRE} wrong" + DATE=$(date -Is) + mv "${INCOMING}/${file}" "${ERRORS}/keyexpire.${file}.${DATE}" + mv "${GPGSTATUS}" "${ERRORS}/keyexpire.${file}.gpgstatus.${DATE}" + mv "${GPGLOGS}" "${ERRORS}/keyexpire.${file}.gpglogs.${DATE}" + rm -f "${GPGOUTF}" + continue fi # And now lets check how many keys this buildd already has. 2 is the maximum, so key @@ -245,7 +272,9 @@ for file in ${KEYS}; do # Right. At this point everything should be in order, which means we should put the key into # the keyring - log "Accepting key ${KEYID} for ${ARCH} buildd ${BUILDD}, expire ${KEYEXPIRE}" + KEYSUBMITTER=$(cat "${GPGSTATUS}"|grep GOODSIG) + KEYSUBMITTER=${KEYSUBMITTER##*GOODSIG} + log "${KEYSUBMITTER} added key ${KEYID} for ${ARCH} buildd ${BUILDD}, expire ${KEYEXPIRE}" gpg ${DEFGPGOPT} --status-fd 4 --logger-fd 5 --keyring "${ARCHKEYRING}" --import "${GPGOUTF}" 2>/dev/null mv "${INCOMING}/${file}" "${base}/${ARCH}" diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py index 0f23053c..89b4bb71 100755 --- a/tests/dbtest_contents.py +++ b/tests/dbtest_contents.py @@ -142,19 +142,13 @@ class ContentsTestCase(DBDakTestCase): self.setup_overrides() self.binary['hello_2.2-1_i386'].contents.append(BinContents(file = '/usr/bin/hello')) self.session.commit() - cw = BinaryContentsWriter(self.suite['squeeze'], self.arch['i386'], self.otype['deb']) + cw = BinaryContentsWriter(self.suite['squeeze'], self.arch['i386'], \ + self.otype['deb'], self.comp['main']) self.assertEqual(['/usr/bin/hello python/hello\n'], \ cw.get_list()) # test formatline and sort order self.assertEqual('/usr/bin/hello python/hello\n', \ cw.formatline('/usr/bin/hello', 'python/hello')) - # test output_filename - self.assertEqual('tests/fixtures/ftp/dists/squeeze/Contents-i386.gz', \ - normpath(cw.output_filename())) - cw = BinaryContentsWriter(self.suite['squeeze'], self.arch['i386'], \ - self.otype['udeb'], self.comp['main']) - self.assertEqual('tests/fixtures/ftp/dists/squeeze/main/Contents-i386.gz', \ - normpath(cw.output_filename())) # test unicode support self.binary['hello_2.2-1_i386'].contents.append(BinContents(file = '\xc3\xb6')) self.session.commit()