From: Mark Hymers Date: Wed, 27 Jul 2011 11:45:11 +0000 (+0100) Subject: Merge remote branch 'ftpmaster/master' X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=7701b2df5bca50d8535d393e9e84ce624af7cc55;hp=ca2ce4c3ffba031f5ddb84c225d474c910504ac0 Merge remote branch 'ftpmaster/master' --- diff --git a/config/backports/dak.conf b/config/backports/dak.conf index 020a9963..a9f4886c 100644 --- a/config/backports/dak.conf +++ b/config/backports/dak.conf @@ -219,7 +219,6 @@ Dir Log "/srv/backports-master.debian.org/log/"; Lock "/srv/backports-master.debian.org/lock"; Morgue "/srv/backports-master.debian.org/morgue/"; - MorgueReject "reject"; Override "/srv/backports-master.debian.org/scripts/override/"; QueueBuild "/srv/backports-master.debian.org/buildd/"; UrgencyLog "/srv/backports-master.debian.org/testing/urgencies/"; @@ -294,28 +293,6 @@ Archive }; }; -Component -{ - main - { - Description "Main"; - MeetsDFSG "true"; - }; - - contrib - { - Description "Contrib"; - MeetsDFSG "true"; - }; - - non-free - { - Description "Software that fails to meet the DFSG"; - MeetsDFSG "false"; - }; - -}; - Section { admin; @@ -382,24 +359,6 @@ Priority source 0; // i.e. unused }; -OverrideType -{ - deb; - udeb; - dsc; -}; - -Location -{ - // Pool locations on backports.debian.org - /srv/backports-master.debian.org/ftp/pool/ - { - Archive "backports"; - Type "pool"; - }; - -}; - Urgency { Default "low"; diff --git a/config/debian-security/dak.conf b/config/debian-security/dak.conf index fc4017d7..fb23848e 100644 --- a/config/debian-security/dak.conf +++ b/config/debian-security/dak.conf @@ -236,7 +236,6 @@ Dir Lists "/srv/security-master.debian.org/dak-database/dists/"; Log "/srv/security-master.debian.org/dak-log/"; Morgue "/srv/security-master.debian.org/morgue/"; - MorgueReject "reject"; Override "/srv/security-master.debian.org/scripts/override/"; QueueBuild "/srv/security-master.debian.org/buildd/"; Upload "/srv/queued/ftpmaster/"; @@ -305,29 +304,6 @@ Archive }; -Component -{ - - updates/main - { - Description "Main (updates)"; - MeetsDFSG "true"; - }; - - updates/contrib - { - Description "Contrib (updates)"; - MeetsDFSG "true"; - }; - - updates/non-free - { - Description "Software that fails to meet the DFSG"; - MeetsDFSG "false"; - }; - -}; - ComponentMappings { "main updates/main"; @@ -404,28 +380,6 @@ Priority source 0; // i.e. unused }; -OverrideType -{ - deb; - udeb; - dsc; -}; - -Location -{ - /srv/security-master.debian.org/ftp/pool/ - { - Archive "security"; - Suites - { - OldStable; - Stable; - Testing; - }; - Type "pool"; - }; -}; - Urgency { Default "low"; diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 42659d96..293df65d 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -332,7 +332,6 @@ Dir Log "/srv/ftp-master.debian.org/log/"; Lock "/srv/ftp-master.debian.org/lock"; Morgue "/srv/ftp-master.debian.org/morgue/"; - MorgueReject "reject"; Override "/srv/ftp-master.debian.org/scripts/override/"; UrgencyLog "/srv/release.debian.org/britney/input/urgencies/"; TempPath "/srv/ftp-master.debian.org/tmp/"; @@ -384,34 +383,6 @@ Archive }; }; -Component -{ - main - { - Description "Main"; - MeetsDFSG "true"; - }; - - contrib - { - Description "Contrib"; - MeetsDFSG "true"; - }; - - non-free - { - Description "Software that fails to meet the DFSG"; - MeetsDFSG "false"; - }; -}; - -OverrideType -{ - deb; - udeb; - dsc; -}; - Urgency { Default "low"; diff --git a/dak/check_archive.py b/dak/check_archive.py index a88f8eba..0d94bbcb 100755 --- a/dak/check_archive.py +++ b/dak/check_archive.py @@ -41,6 +41,7 @@ import apt_inst from daklib.dbconn import * from daklib import utils from daklib.config import Config +from daklib.dak_exceptions import InvalidDscError, ChangesUnicodeError, CantOpenError ################################################################################ @@ -148,26 +149,24 @@ def check_dscs(): Parse every .dsc file in the archive and check for it's validity. """ - cnf = Config() - count = 0 - suite = 'unstable' - - for component in cnf.SubTree("Component").List(): - component = component.lower() - list_filename = '%s%s_%s_source.list' % (cnf["Dir::Lists"], suite, component) - list_file = utils.open_file(list_filename) - - for line in list_file.readlines(): - f = line[:-1] - try: - utils.parse_changes(f, signing_rules=1, dsc_file=1) - except InvalidDscError, line: - utils.warn("syntax error in .dsc file '%s', line %s." % (f, line)) - count += 1 - except ChangesUnicodeError: - utils.warn("found invalid changes file, not properly utf-8 encoded") - count += 1 + + for src in DBConn().session().query(DBSource).order_by(DBSource.source, DBSource.version): + f = src.poolfile.fullpath + try: + utils.parse_changes(f, signing_rules=1, dsc_file=1) + except InvalidDscError: + utils.warn("syntax error in .dsc file %s" % f) + count += 1 + except ChangesUnicodeError: + utils.warn("found invalid dsc file (%s), not properly utf-8 encoded" % f) + count += 1 + except CantOpenError: + utils.warn("missing dsc file (%s)" % f) + count += 1 + except Exception, e: + utils.warn("miscellaneous error parsing dsc file (%s): %s" % (f, str(e))) + count += 1 if count: utils.warn("Found %s invalid .dsc files." % (count)) diff --git a/dak/check_overrides.py b/dak/check_overrides.py index a8e0f1d9..523cdefe 100755 --- a/dak/check_overrides.py +++ b/dak/check_overrides.py @@ -372,9 +372,12 @@ def main (): for component in cnf.SubTree("Component").List(): # It is crucial for the dsc override creation based on binary # overrides that 'dsc' goes first - otypes = cnf.ValueList("OverrideType") - otypes.remove("dsc") - otypes = ["dsc"] + otypes + otypes = ['dsc'] + for ot in session.query(OverrideType): + if ot.overridetype == 'dsc': + continue + otypes.append(ot.overridetype) + for otype in otypes: print "Processing %s [%s - %s]" \ % (osuite, component, otype) diff --git a/dak/make_overrides.py b/dak/make_overrides.py index b925d927..716b085b 100755 --- a/dak/make_overrides.py +++ b/dak/make_overrides.py @@ -122,26 +122,24 @@ def main (): sys.stderr.write("Processing %s...\n" % (suite.suite_name)) override_suite = suite.overridecodename - for component_name in cnf.SubTree("Component").List(): - component = get_component(component_name, session) - if not component: - utils.fubar('Component %s not found' % component_name) - - for otype_name in cnf.ValueList("OverrideType"): - otype = get_override_type(otype_name, session) - if not otype: - utils.fubar('OverrideType %s not found' % otype_name) + for component in session.query(Component).all(): + for otype in session.query(OverrideType).all(): + otype_name = otype.overridetype + cname = component.component_name + # TODO: Stick suffix info in database (or get rid of it) if otype_name == "deb": suffix = "" elif otype_name == "udeb": - if component == "contrib": + if cname == "contrib": continue # Ick2 suffix = ".debian-installer" elif otype_name == "dsc": suffix = ".src" + else: + utils.fubar("Don't understand OverrideType %s" % otype.overridetype) - cname = component.component_name.replace('/', '_') + cname = cname.replace('/', '_') filename = os.path.join(cnf["Dir::Override"], "override.%s.%s%s" % (override_suite, cname, suffix)) output_file = utils.open_file(filename, 'w') diff --git a/dak/process_upload.py b/dak/process_upload.py index f74993ce..4e734da4 100755 --- a/dak/process_upload.py +++ b/dak/process_upload.py @@ -424,7 +424,6 @@ def main(): cnf = Config() summarystats = SummaryStats() - log_urgency = False DBConn() @@ -464,10 +463,10 @@ def main(): utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise - if cnf.get("Dir::UrgencyLog"): - # Initialise UrgencyLog() - log_urgency = True - UrgencyLog() + + # Initialise UrgencyLog() - it will deal with the case where we don't + # want to log urgencies + urgencylog = UrgencyLog() Logger = daklog.Logger("process-upload", Options["No-Action"]) @@ -513,8 +512,7 @@ def main(): byebye() if not Options["No-Action"]: - if log_urgency: - UrgencyLog().close() + urgencylog.close() Logger.close() diff --git a/daklib/queue.py b/daklib/queue.py index 5055ea57..63375f2d 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -2181,7 +2181,7 @@ distribution.""" utils.move(self.pkg.changes_file, os.path.join(donedir, os.path.basename(self.pkg.changes_file))) - if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"): + if self.pkg.changes["architecture"].has_key("source"): UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"]) self.update_subst() diff --git a/daklib/urgencylog.py b/daklib/urgencylog.py index 7d679058..a47974dd 100644 --- a/daklib/urgencylog.py +++ b/daklib/urgencylog.py @@ -47,26 +47,43 @@ class UrgencyLog(object): self.timestamp = time.strftime("%Y%m%d%H%M%S") - # Create the log directory if it doesn't exist - self.log_dir = Config()["Dir::UrgencyLog"] + cnf = Config() + if cnf.has_key("Dir::UrgencyLog"): + # Create the log directory if it doesn't exist + self.log_dir = cnf["Dir::UrgencyLog"] - if not os.path.exists(self.log_dir) or not os.access(self.log_dir, os.W_OK): - warn("UrgencyLog directory %s does not exist or is not writeable, using /srv/ftp.debian.org/tmp/ instead" % (self.log_dir)) - self.log_dir = '/srv/ftp.debian.org/tmp/' + if not os.path.exists(self.log_dir) or not os.access(self.log_dir, os.W_OK): + warn("UrgencyLog directory %s does not exist or is not writeable, using /srv/ftp.debian.org/tmp/ instead" % (self.log_dir)) + self.log_dir = '/srv/ftp.debian.org/tmp/' + + # Open the logfile + self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp) + self.log_file = open_file(self.log_filename, 'w') + + else: + self.log_dir = None + self.log_filename = None + self.log_file = None - # Open the logfile - self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp) - self.log_file = open_file(self.log_filename, 'w') self.writes = 0 def log(self, source, version, urgency): "Log an event" + + # Don't try and log if Dir::UrgencyLog is not configured + if self.log_file is None: + return + self.log_file.write(" ".join([source, version, urgency])+'\n') self.log_file.flush() self.writes += 1 def close(self): "Close a Logger object" + # Don't try and log if Dir::UrgencyLog is not configured + if self.log_file is None: + return + self.log_file.flush() self.log_file.close() diff --git a/daklib/utils.py b/daklib/utils.py index fd4d7bbf..413bcb6d 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -143,10 +143,11 @@ def extract_component_from_section(section): # Expand default component if component == "": - if Cnf.has_key("Component::%s" % section): - component = section - else: + comp = get_component(section) + if comp is None: component = "main" + else: + component = comp.componant_name return (section, component) diff --git a/docs/README.config b/docs/README.config index a20cfc9f..ee0dc0d6 100644 --- a/docs/README.config +++ b/docs/README.config @@ -39,7 +39,6 @@ Mandatory. List of directory locations, e.g. | Lists "/org/ftp.debian.org/database/dists/"; | Log "/org/ftp.debian.org/log/"; | Morgue "/org/ftp.debian.org/morgue/"; -| MorgueReject "reject"; | QueueBuild "/org/incoming.debian.org/buildd/"; | UrgencyLog "/org/ftp.debian.org/testing/urgencies/"; | Queue @@ -79,14 +78,6 @@ Morgue (required): Removed files are moved there. The morgue has various sub-directories, including (optionally) those defined by Clean-Queues::MorgueSubDir and Clean-Suites::MorgueSubDir. -MorgueReject (required): if dak cannot move a rejected package to -Dir::Queue::Reject, it will try to move it to the Dir::MorgueReject -directory located under Dir::Morgue. - -QueueBuild (optional): This variable is only relevant if any suites -are to be auto built, i.e. if Dinstall::QueueBuildSuites has any -values. - UrgencyLog (optional): If this directory is specified, 'dak process-accepted' will store the urgency value of each upload. This is mainly used for britney (the testing script). @@ -218,10 +209,6 @@ Mandatory. List of dinstall options, e.g.: | FutureTimeTravelGrace 28800; // 8 hours | PastCutoffYear "1984"; | BXANotify "false"; -| QueueBuildSuites -| { -| unstable; -| }; | }; GPGKeyring (required): filenames of the PGP and GnuPG @@ -274,9 +261,6 @@ at the maintainer if they differ. CloseBugs (optional): a boolean (default: no); if true the automated bug closing feature of dinstall is activated. -QueueBuildSuites (optional): a list of suites which should be auto -build. - QueueBuild is a boolean; if true it activates support for auto-building from accepted. @@ -337,25 +321,6 @@ The description is currently unused. ================================================================================ -Component ---------- - -Mandatory. List of all components, e.g. - -| Component -| { -| main -| { -| Description "Main"; -| MeetsDFSG "true"; -| }; -| }; - -All three values go into the SQL database's 'component' table. -MeetsDFSG is currently unused. - -================================================================================ - Section ------- @@ -390,22 +355,6 @@ go into the SQL database's 'priority' table. ================================================================================ -OverrideType ------------- - -Mandatory. List of al valid override types, e.g. - -| OverrideType -| { -| deb; -| dsc; -| udeb; -| }; - -The type goes into the 'override_type' table in the SQL database. - -================================================================================ - Location --------