$scriptsdir/expire_dumps -d . -p -f "dump_*"
}
+function transitionsclean() {
+ log "Removing out of date transitions..."
+ cd $base
+ dak transitions -c -a
+}
+
function reports() {
# Send a report on NEW/BYHAND packages
log "Nagging ftpteam about NEW/BYHAND packages"
)
stage $GO
+GO=(
+ FUNC="transitionsclean"
+ TIME="transitionsclean"
+ ARGS=""
+ ERR=""
+)
+stage $GO
+
GO=(
FUNC="reports"
TIME="reports"
FUNC="aptftpcleanup"
TIME="apt-ftparchive cleanup"
ARGS=""
- ERR=""
+ ERR="false"
)
stage $GO
Transitions
{
+ Notifications "team@release.debian.org";
TempPath "/srv/ftp.debian.org/tmp/";
};
Process-New
{
AcceptedLockFile "/srv/ftp.debian.org/lock/unchecked.lock";
+ LockDir "/srv/ftp.debian.org/lock/new/";
};
Check-Overrides
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Adding process-new comments to the DB
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+
+################################################################################
+
+def do_update(self):
+ print "Adding process-new comments to the DB"
+
+ try:
+ c = self.db.cursor()
+ c.execute("""CREATE TABLE new_comments (
+ id SERIAL PRIMARY KEY NOT NULL,
+ package TEXT NOT NULL,
+ version TEXT NOT NULL,
+ comment TEXT NOT NULL,
+ author TEXT NOT NULL
+ )""")
+
+ c.execute("GRANT SELECT ON new_comments TO ftptrainee;")
+ c.execute("GRANT INSERT ON new_comments TO ftptrainee;")
+ c.execute("GRANT UPDATE ON new_comments TO ftptrainee;")
+ c.execute("GRANT SELECT ON new_comments TO ftpteam;")
+ c.execute("GRANT INSERT ON new_comments TO ftpteam;")
+ c.execute("GRANT UPDATE ON new_comments TO ftpteam;")
+ c.execute("GRANT ALL ON new_comments TO ftpmaster;")
+
+ c.execute("UPDATE config SET value = '11' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply process-new comments update, rollback issued. Error message : %s" % (str(msg))
--- /dev/null
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Adding a date field to the process-new notes
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+
+################################################################################
+
+def do_update(self):
+ print "Adding a date field to the process-new notes"
+
+ try:
+ c = self.db.cursor()
+ c.execute("ALTER TABLE new_comments ADD COLUMN notedate TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()")
+
+ c.execute("UPDATE config SET value = '12' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply process-new update 12, rollback issued. Error message : %s" % (str(msg))
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
@license: GNU General Public License version 2 or later
"""
# This program is free software; you can redistribute it and/or modify
################################################################################
+from __future__ import with_statement
+
import copy
import errno
import os
import stat
import sys
import time
+import contextlib
+import pwd
import apt_pkg, apt_inst
import examine_package
from daklib import database
from daklib import queue
from daklib import utils
from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
+from daklib.dak_exceptions import CantOpenError, AlreadyLockedError
# Globals
Cnf = None #: Configuration, apt_pkg.Configuration
if reject_message.find("Rejected") != -1:
answer = "XXX"
- if Options["No-Action"] or Options["Automatic"]:
+ if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]:
answer = 'S'
print "REJECT\n" + reject_message,
mtime = os.stat(d["filename"])[stat.ST_MTIME]
if mtime < oldest:
oldest = mtime
- have_note += (d.has_key("process-new note"))
+ have_note += (database.has_new_comment(d["source"], d["version"]))
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
line = "%-20s %-20s %-20s" % (pkg, priority, section)
line = line.strip()+'\n'
file.write(line)
- note = Upload.pkg.changes.get("process-new note")
- if note:
- print "*"*75
- print note
- print "*"*75
+ note = database.get_new_comments(Upload.pkg.changes.get("source"))
+ if len(note) > 0:
+ for line in note:
+ print line
return broken, note
################################################################################
def edit_note(note):
# Write the current data to a temporary file
(fd, temp_filename) = utils.temp_filename()
- temp_file = os.fdopen(fd, 'w')
- temp_file.write(note)
- temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
temp_file = utils.open_file(temp_filename)
- note = temp_file.read().rstrip()
+ newnote = temp_file.read().rstrip()
temp_file.close()
- print "Note:"
- print utils.prefix_multi_line_string(note," ")
+ print "New Note:"
+ print utils.prefix_multi_line_string(newnote," ")
prompt = "[D]one, Edit, Abandon, Quit ?"
answer = "XXX"
while prompt.find(answer) == -1:
elif answer == 'Q':
end()
sys.exit(0)
- Upload.pkg.changes["process-new note"] = note
- Upload.dump_vars(Cnf["Dir::Queue::New"])
+ database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami())
################################################################################
################################################################################
-def prod_maintainer ():
+def prod_maintainer (note):
# Here we prepare an editor and get them ready to prod...
(fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ if len(note) > 0:
+ for line in note:
+ temp_file.write(line)
+ temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
os.system("%s %s" % (editor, temp_filename))
- f = os.fdopen(fd)
- prod_message = "".join(f.readlines())
- f.close()
+ temp_fh = utils.open_file(temp_filename)
+ prod_message = "".join(temp_fh.readlines())
+ temp_fh.close()
print "Prod message:"
print utils.prefix_multi_line_string(prod_message," ",include_blank_lines=1)
prompt = "[P]rod, Edit, Abandon, Quit ?"
if answer == "":
answer = m.group(1)
answer = answer[:1].upper()
- os.unlink(temp_filename)
- if answer == 'A':
- return
- elif answer == 'Q':
- end()
- sys.exit(0)
+ os.unlink(temp_filename)
+ if answer == 'A':
+ return
+ elif answer == 'Q':
+ end()
+ sys.exit(0)
# Otherwise, do the proding...
user_email_address = utils.whoami() + " <%s>" % (
Cnf["Dinstall::MyAdminAddress"])
answer = m.group(1)
answer = answer[:1].upper()
- if answer == 'A':
+ if answer == 'A' and not Options["Trainee"]:
done = add_overrides (new)
elif answer == 'C':
check_pkg()
- elif answer == 'E':
+ elif answer == 'E' and not Options["Trainee"]:
new = edit_overrides (new)
- elif answer == 'M':
- aborted = Upload.do_reject(1, Options["Manual-Reject"])
+ elif answer == 'M' and not Options["Trainee"]:
+ aborted = Upload.do_reject(manual=1,
+ reject_message=Options["Manual-Reject"],
+ note=database.get_new_comments(changes.get("source", "")))
if not aborted:
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
done = 1
elif answer == 'N':
- edit_note(changes.get("process-new note", ""))
- elif answer == 'P':
- prod_maintainer()
- elif answer == 'R':
+ edit_note(database.get_new_comments(changes.get("source", "")))
+ elif answer == 'P' and not Options["Trainee"]:
+ prod_maintainer(database.get_new_comments(changes.get("source", "")))
+ elif answer == 'R' and not Options["Trainee"]:
confirm = utils.our_raw_input("Really clear note (y/N)? ").lower()
if confirm == "y":
- del changes["process-new note"]
+ database.delete_new_comments(changes.get("source"), changes.get("version"))
elif answer == 'S':
done = 1
elif answer == 'Q':
-C, --comments-dir=DIR use DIR as comments-dir, for [o-]p-u-new
-m, --manual-reject=MSG manual reject with `msg'
-n, --no-action don't do anything
+ -t, --trainee FTP Trainee mode
-V, --version display the version number and exit"""
sys.exit(exit_code)
('h',"help","Process-New::Options::Help"),
('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
+ ('t',"trainee","Process-New::Options::Trainee"),
('n',"no-action","Process-New::Options::No-Action")]
- for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]:
if not Cnf.has_key("Process-New::Options::%s" % (i)):
Cnf["Process-New::Options::%s" % (i)] = ""
Upload = queue.Upload(Cnf)
if not Options["No-Action"]:
- Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
+ try:
+ Logger = Upload.Logger = logging.Logger(Cnf, "process-new")
+ except CantOpenError, e:
+ Options["Trainee"] = "Oh yes"
projectB = Upload.projectB
else:
raise
+
+@contextlib.contextmanager
+def lock_package(package):
+ """
+ Lock C{package} so that noone else jumps in processing it.
+
+ @type package: string
+ @param package: source package name to lock
+ """
+
+ path = os.path.join(Cnf["Process-New::LockDir"], package)
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
+ raise AlreadyLockedError, user
+
+ try:
+ yield fd
+ finally:
+ os.unlink(path)
+
def move_to_dir (dest, perms=0660, changesperms=0664):
utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
file_keys = Upload.pkg.files.keys()
Upload.update_subst()
files = Upload.pkg.files
- if not recheck():
- return
-
- (new, byhand) = check_status(files)
- if new or byhand:
- if new:
- do_new()
- if byhand:
- do_byhand()
- (new, byhand) = check_status(files)
-
- if not new and not byhand:
- do_accept()
+ try:
+ with lock_package(Upload.pkg.changes["source"]):
+ if not recheck():
+ return
+
+ (new, byhand) = check_status(files)
+ if new or byhand:
+ if new:
+ do_new()
+ if byhand:
+ do_byhand()
+ (new, byhand) = check_status(files)
+
+ if not new and not byhand:
+ do_accept()
+ except AlreadyLockedError, e:
+ print "Seems to be locked by %s already, skipping..." % (e)
################################################################################
sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes))))
Logger.log(["total",accept_count,accept_bytes])
- if not Options["No-Action"]:
+ if not Options["No-Action"] and not Options["Trainee"]:
Logger.close()
################################################################################
import apt_pkg
import cgi
from daklib import queue
+from daklib import database
from daklib import utils
from daklib.dak_exceptions import *
Upload = None
direction = []
row_number = 0
+projectB = None
################################################################################
else:
if mtime < oldest:
oldest = mtime
- have_note += (d.has_key("process-new note"))
+ have_note += (database.has_new_comment(d["source"], d["version"]))
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0; # none
usage()
Upload = queue.Upload(Cnf)
+ projectB = Upload.projectB
if Cnf.has_key("Queue-Report::Options::New"):
header()
Cnf = utils.get_conf()
- Arguments = [('h',"help","Edit-Transitions::Options::Help"),
+ Arguments = [('a',"automatic","Edit-Transitions::Options::Automatic"),
+ ('h',"help","Edit-Transitions::Options::Help"),
('e',"edit","Edit-Transitions::Options::Edit"),
('i',"import","Edit-Transitions::Options::Import", "HasArg"),
('c',"check","Edit-Transitions::Options::Check"),
('s',"sudo","Edit-Transitions::Options::Sudo"),
('n',"no-action","Edit-Transitions::Options::No-Action")]
- for i in ["help", "no-action", "edit", "import", "check", "sudo"]:
+ for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]:
if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)):
Cnf["Edit-Transitions::Options::%s" % (i)] = ""
-i, --import <file> check and import transitions from file
-c, --check check the transitions file, remove outdated entries
-S, --sudo use sudo to update transitions file
+ -a, --automatic don't prompt (only affects check).
-n, --no-action don't do anything (only affects check)"""
sys.exit(exit_code)
def check_transitions(transitions):
"""
Check if the defined transitions still apply and remove those that no longer do.
- @note: Asks the user for confirmation first.
+ @note: Asks the user for confirmation first unless -a has been set.
"""
+ global Cnf
+
to_dump = 0
to_remove = []
+ info = {}
# Now look through all defined transitions
for trans in transitions:
t = transitions[trans]
# Will be None if nothing is in testing.
current = database.get_suite_version(source, "testing")
- print_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
+ info[trans] = get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
+ print info[trans]
if current == None:
# No package in testing
if Options["no-action"]:
answer="n"
+ elif Options["automatic"]:
+ answer="y"
else:
answer = utils.our_raw_input(prompt).lower()
sys.exit(0)
elif answer == 'y':
print "Committing"
+ subst = {}
+ subst['__TRANSITION_MESSAGE__'] = "The following transitions were removed:\n"
for remove in to_remove:
+ subst['__TRANSITION_MESSAGE__'] += info[remove] + '\n'
del transitions[remove]
+ # If we have a mail address configured for transitions,
+ # send a notification
+ subst['__TRANSITION_EMAIL__'] = Cnf.get("Transitions::Notifications", "")
+ if subst['__TRANSITION_EMAIL__'] != "":
+ print "Sending notification to %s" % subst['__TRANSITION_EMAIL__']
+ subst['__DAK_ADDRESS__'] = Cnf["Dinstall::MyEmailAddress"]
+ subst['__BCC__'] = 'X-DAK: dak transitions'
+ if Cnf.has_key("Dinstall::Bcc"):
+ subst["__BCC__"] += '\nBcc: %s' % Cnf["Dinstall::Bcc"]
+ message = utils.TemplateSubst(subst,
+ os.path.join(Cnf["Dir::Templates"], 'transition.removed'))
+ utils.send_mail(message)
+
edit_file = temp_transitions_file(transitions)
write_transitions_from_file(edit_file)
################################################################################
-def print_info(trans, source, expected, rm, reason, packages):
+def get_info(trans, source, expected, rm, reason, packages):
"""
Print information about a single transition.
@param packages: list of blocked packages
"""
- print """Looking at transition: %s
+ return """Looking at transition: %s
Source: %s
New Version: %s
Responsible: %s
Description: %s
Blocked Packages (total: %d): %s
""" % (trans, source, expected, rm, reason, len(packages), ", ".join(packages))
- return
################################################################################
def transition_info(transitions):
"""
Print information about all defined transitions.
- Calls L{print_info} for every transition and then tells user if the transition is
+ Calls L{get_info} for every transition and then tells user if the transition is
still ongoing or if the expected version already hit testing.
@type transitions: dict
# Will be None if nothing is in testing.
current = database.get_suite_version(source, "testing")
- print_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
+ print get_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
if current == None:
# No package in testing
Cnf = None
projectB = None
-required_database_schema = 10
+required_database_schema = 12
################################################################################
"NoSourceFieldError": """Exception raised - we cant find the source - wtf?""",
"MissingContents": """Exception raised - we could not determine contents for this deb""",
"DBUpdateError": """Exception raised - could not update the database""",
- "ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded"""
+ "ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded""",
+ "AlreadyLockedError": """Exception raised - package already locked by someone else"""
} #: All dak exceptions
def construct_dak_exception(name, description):
@group readonly: get_suite_id, get_section_id, get_priority_id, get_override_type_id,
get_architecture_id, get_archive_id, get_component_id, get_location_id,
get_source_id, get_suite_version, get_files_id, get_maintainer, get_suites,
- get_suite_architectures
+ get_suite_architectures, get_new_comments, has_new_comment
@group read/write: get_or_set*, set_files_id
+@group writeonly: add_new_comment, delete_new_comments
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
################################################################################
+def get_new_comments(package):
+ """
+ Returns all the possible comments attached to C{package} in NEW. All versions.
+
+ @type package: string
+ @param package: name of the package
+
+ @rtype: list
+ @return: list of strings containing comments for all versions from all authors for package
+ """
+
+ comments = []
+ query = projectB.query(""" SELECT version, comment, author, notedate
+ FROM new_comments
+ WHERE package = '%s'
+ ORDER BY notedate
+ """ % (package))
+
+ for row in query.getresult():
+ comments.append("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s\n" % (row[2], row[0], row[3], row[1]))
+ comments.append("-"*72)
+
+ return comments
+
+def has_new_comment(package, version):
+ """
+ Returns true if the given combination of C{package}, C{version} has a comment.
+
+ @type package: string
+ @param package: name of the package
+
+ @type version: string
+ @param version: package version
+
+ @rtype: boolean
+ @return: true/false
+ """
+
+ exists = projectB.query("""SELECT 1 FROM new_comments
+ WHERE package='%s'
+ AND version='%s'
+ LIMIT 1"""
+ % (package, version) ).getresult()
+
+ if not exists:
+ return False
+ else:
+ return True
+
+def add_new_comment(package, version, comment, author):
+ """
+ Add a new comment for C{package}, C{version} written by C{author}
+
+ @type package: string
+ @param package: name of the package
+
+ @type version: string
+ @param version: package version
+
+ @type comment: string
+ @param comment: the comment
+
+ @type author: string
+ @param author: the authorname
+ """
+
+ projectB.query(""" INSERT INTO new_comments (package, version, comment, author)
+ VALUES ('%s', '%s', '%s', '%s')
+ """ % (package, version, comment, author) )
+
+ return
+
+def delete_new_comments(package, version):
+ """
+ Delete a comment for C{package}, C{version}, if one exists
+ """
+
+ projectB.query(""" DELETE FROM new_comments
+ WHERE package = '%s' AND version = '%s'
+ """ % (package, version))
+ return
+
+################################################################################
def copy_temporary_contents(package, version, arch, deb, reject):
"""
copy the previously stored contents from the temp table to the permanant one
###########################################################################
- def do_reject (self, manual = 0, reject_message = ""):
+ def do_reject (self, manual = 0, reject_message = "", note = ""):
"""
Reject an upload. If called without a reject message or C{manual} is
true, spawn an editor so the user can write one.
# editor so the user can add one in...
if manual and not reject_message:
(fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ if len(note) > 0:
+ for line in note:
+ temp_file.write(line)
+ temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
--- /dev/null
+From: __DAK_ADDRESS__
+To: __TRANSITION_EMAIL__
+__BCC__
+X-Debian: DAK
+Precedence: bulk
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Transitions Completed
+
+The following transitions are complete and have been removed
+from the transitions list:
+
+__TRANSITION_MESSAGE__
+
projects / dak.git / commitdiff