X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;ds=inline;f=dak%2Fprocess_upload.py;h=1908ecb5ed2e2310c257555be45a6cf1059d8cc5;hb=49c85ab8a723a425cc6539dd138dcd9e0913ab9f;hp=e3a19ac31609a3b866e09694e3d44660a6b05b7d;hpb=a16d283c9d00f1ed9cedcd2035a21bd9857a6476;p=dak.git
diff --git a/dak/process_upload.py b/dak/process_upload.py
index e3a19ac3..1908ecb5 100755
--- a/dak/process_upload.py
+++ b/dak/process_upload.py
@@ -125,25 +125,58 @@ Checks Debian packages from Incoming
## pu: create files for BTS
## pu: create entry in queue_build
## pu: check overrides
-import errno
+
+# Integrity checks
+## GPG
+## Parsing changes (check for duplicates)
+## Parse dsc
+## file list checks
+
+# New check layout (TODO: Implement)
+## Permission checks
+### suite mappings
+### ACLs
+### version checks (suite)
+### override checks
+
+## Source checks
+### copy orig
+### unpack
+### BTS changelog
+### src contents
+### lintian
+### urgency log
+
+## Binary checks
+### timestamps
+### control checks
+### src relation check
+### contents
+
+## Database insertion (? copy from stuff)
+### BYHAND / NEW / Policy queues
+### Pool
+
+## Queue builds
+
+from errno import EACCES, EAGAIN
import fcntl
import os
import sys
-#from datetime import datetime
import traceback
import apt_pkg
+from sqlalchemy.orm.exc import NoResultFound
from daklib import daklog
from daklib.queue import *
from daklib.queue_install import *
from daklib import utils
from daklib.dbconn import *
-#from daklib.dak_exceptions import *
-#from daklib.regexes import re_default_answer, re_issource, re_fdnic
from daklib.urgencylog import UrgencyLog
from daklib.summarystats import SummaryStats
from daklib.holding import Holding
from daklib.config import Config
+from daklib.regexes import re_match_expired
###############################################################################
@@ -155,6 +188,7 @@ Logger = None
def usage (exit_code=0):
print """Usage: dak process-upload [OPTION]... [CHANGES]...
-a, --automatic automatic run
+ -d, --directory
process uploads in
-h, --help show this help and exit.
-n, --no-action don't do anything
-p, --no-lock don't check lockfile !! for cron.daily only !!
@@ -164,12 +198,24 @@ def usage (exit_code=0):
###############################################################################
-def action(u):
+def byebye():
+ if not Options["No-Action"]:
+ # Clean out the queue files
+ session = DBConn().session()
+ session.execute("DELETE FROM changes_pending_files WHERE id NOT IN (SELECT file_id FROM changes_pending_files_map )")
+ session.commit()
+
+
+
+def action(u, session):
+ global Logger
+
cnf = Config()
+ holding = Holding()
# changes["distribution"] may not exist in corner cases
# (e.g. unreadable changes files)
- if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType):
+ if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], dict):
u.pkg.changes["distribution"] = {}
(summary, short_summary) = u.build_summaries()
@@ -182,6 +228,11 @@ def action(u):
pi = u.package_info()
+ try:
+ chg = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one()
+ except NoResultFound as e:
+ chg = None
+
if len(u.rejects) > 0:
if u.upload_too_new():
print "SKIP (too new)\n" + pi,
@@ -192,6 +243,8 @@ def action(u):
if Options["Automatic"]:
answer = 'R'
else:
+ # Are we headed for NEW / BYHAND / AUTOBYHAND?
+ # Note that policy queues are no longer handled here
qu = determine_target(u)
if qu:
print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary)
@@ -204,10 +257,32 @@ def action(u):
if Options["Automatic"]:
answer = queuekey
else:
- print "ACCEPT\n" + pi + summary,
- prompt = "[A]ccept, Skip, Quit ?"
- if Options["Automatic"]:
- answer = 'A'
+ # Does suite have a policy_queue configured
+ divert = False
+ for s in u.pkg.changes["distribution"].keys():
+ suite = get_suite(s, session)
+ if suite.policy_queue:
+ if not chg or chg.approved_for_id != suite.policy_queue.policy_queue_id:
+ # This routine will check whether the upload is a binary
+ # upload when the source is already in the target suite. If
+ # so, we skip the policy queue, otherwise we go there.
+ divert = package_to_suite(u, suite.suite_name, session=session)
+ if divert:
+ print "%s for %s\n%s%s" % ( suite.policy_queue.queue_name.upper(),
+ ", ".join(u.pkg.changes["distribution"].keys()),
+ pi, summary)
+ queuekey = "P"
+ prompt = "[P]olicy, Skip, Quit ?"
+ policyqueue = suite.policy_queue
+ if Options["Automatic"]:
+ answer = 'P'
+ break
+
+ if not divert:
+ print "ACCEPT\n" + pi + summary,
+ prompt = "[A]ccept, Skip, Quit ?"
+ if Options["Automatic"]:
+ answer = 'A'
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
@@ -220,20 +295,40 @@ def action(u):
os.chdir(u.pkg.directory)
u.do_reject(0, pi)
elif answer == 'A':
- u.pkg.add_known_changes( "Accepted" )
- u.accept(summary, short_summary)
+ if not chg:
+ chg = u.pkg.add_known_changes(holding.holding_dir, session=session, logger=Logger)
+ session.commit()
+ u.accept(summary, short_summary, session)
u.check_override()
+ chg.clean_from_queue()
+ session.commit()
+ u.remove()
+ elif answer == 'P':
+ if not chg:
+ chg = u.pkg.add_known_changes(holding.holding_dir, session=session, logger=Logger)
+ package_to_queue(u, summary, short_summary, policyqueue, chg, session)
+ session.commit()
u.remove()
elif answer == queuekey:
- u.pkg.add_known_changes( qu )
- QueueInfo[qu]["process"](u, summary, short_summary)
+ if not chg:
+ chg = u.pkg.add_known_changes(holding.holding_dir, session=session, logger=Logger)
+ QueueInfo[qu]["process"](u, summary, short_summary, chg, session)
+ session.commit()
u.remove()
elif answer == 'Q':
+ byebye()
sys.exit(0)
+ session.commit()
+
###############################################################################
-def process_it(changes_file):
+def cleanup():
+ h = Holding()
+ if not Options["No-Action"]:
+ h.clean()
+
+def process_it(changes_file, session):
global Logger
Logger.log(["Processing changes file", changes_file])
@@ -242,6 +337,9 @@ def process_it(changes_file):
holding = Holding()
+ # TODO: Actually implement using pending* tables so that we don't lose track
+ # of what is where
+
u = Upload()
u.pkg.changes_file = changes_file
u.pkg.directory = os.getcwd()
@@ -267,7 +365,7 @@ def process_it(changes_file):
# If this is the Real Thing(tm), copy things into a private
# holding directory first to avoid replacable file races.
if not Options["No-Action"]:
- os.chdir(cnf["Dir::Queue::Holding"])
+ holding.chdir_to_holding()
# Absolutize the filename to avoid the requirement of being in the
# same directory as the .changes file.
@@ -284,6 +382,12 @@ def process_it(changes_file):
if u.pkg.changes["fingerprint"]:
valid_changes_p = u.load_changes(changespath)
else:
+ for reason in rejects:
+ if re_match_expired.match(reason):
+ # Hrm, key expired. Lets see if we can still parse the .changes before
+ # we reject. Then we would be able to mail the maintainer, instead of
+ # just silently dropping the upload.
+ u.load_changes(changespath)
valid_changes_p = False
u.rejects.extend(rejects)
@@ -293,21 +397,24 @@ def process_it(changes_file):
valid_dsc_p = u.check_dsc(not Options["No-Action"])
if valid_dsc_p and not Options["No-Action"]:
u.check_source()
- u.check_lintian()
u.check_hashes()
+ if valid_dsc_p and not Options["No-Action"] and not len(u.rejects):
+ u.check_lintian()
u.check_urgency()
u.check_timestamps()
u.check_signed_by_key()
- action(u)
+ action(u, session)
except (SystemExit, KeyboardInterrupt):
+ cleanup()
raise
except:
print "ERROR"
traceback.print_exc(file=sys.stderr)
+ cleanup()
# Restore previous WD
os.chdir(u.prevdir)
@@ -318,7 +425,6 @@ def main():
cnf = Config()
summarystats = SummaryStats()
- log_urgency = False
DBConn()
@@ -334,8 +440,8 @@ def main():
if not cnf.has_key("Dinstall::Options::%s" % (i)):
cnf["Dinstall::Options::%s" % (i)] = ""
- changes_files = apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
- Options = cnf.SubTree("Dinstall::Options")
+ changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.subtree("Dinstall::Options")
if Options["Help"]:
usage()
@@ -350,20 +456,20 @@ def main():
# Obtain lock if not in no-action mode and initialize the log
if not Options["No-Action"]:
- lock_fd = os.open(cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+ lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT)
try:
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError, e:
+ except IOError as e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.")
else:
raise
- if cnf.get("Dir::UrgencyLog"):
- # Initialise UrgencyLog()
- log_urgency = True
- UrgencyLog()
- Logger = daklog.Logger(cnf, "process-upload", Options["No-Action"])
+ # Initialise UrgencyLog() - it will deal with the case where we don't
+ # want to log urgencies
+ urgencylog = UrgencyLog()
+
+ Logger = daklog.Logger("process-upload", Options["No-Action"])
# If we have a directory flag, use it to find our files
if cnf["Dinstall::Options::Directory"] != "":
@@ -386,20 +492,29 @@ def main():
for changes_file in changes_files:
print "\n" + changes_file
session = DBConn().session()
- process_it(changes_file)
+ process_it(changes_file, session)
session.close()
if summarystats.accept_count:
sets = "set"
if summarystats.accept_count > 1:
sets = "sets"
- sys.stderr.write("Installed %d package %s, %s.\n" % (summarystats.accept_count, sets,
- utils.size_type(int(summarystats.accept_bytes))))
+ print "Installed %d package %s, %s." % (summarystats.accept_count, sets,
+ utils.size_type(int(summarystats.accept_bytes)))
Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes])
+ if summarystats.reject_count:
+ sets = "set"
+ if summarystats.reject_count > 1:
+ sets = "sets"
+ print "Rejected %d package %s." % (summarystats.reject_count, sets)
+ Logger.log(["rejected", summarystats.reject_count])
+
+ byebye()
+
if not Options["No-Action"]:
- if log_urgency:
- UrgencyLog().close()
+ urgencylog.close()
+
Logger.close()
###############################################################################