4 Generate changelog entry between two suites
6 @contact: Debian FTP Master <ftpmaster@debian.org>
7 @copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
8 @license: GNU General Public License version 2 or later
11 # This program is free software; you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation; either version 2 of the License, or
14 # (at your option) any later version.
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
21 # You should have received a copy of the GNU General Public License
22 # along with this program; if not, write to the Free Software
23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 ################################################################################
27 # <bdefreese> !dinstall
28 # <dak> bdefreese: I guess the next dinstall will be in 0hr 1min 35sec
29 # <bdefreese> Wow I have great timing
30 # <DktrKranz> dating with dinstall, part II
32 # <Ganneff> dating with that monster? do you have good combat armor?
33 # <bdefreese> +5 Plate :)
34 # <Ganneff> not a good one then
35 # <Ganneff> so you wont even manage to bypass the lesser monster in front, unchecked
36 # <DktrKranz> asbesto belt
37 # <Ganneff> helps only a step
38 # <DktrKranz> the Ultimate Weapon: cron_turned_off
40 # <Ganneff> thats debadmin limited
41 # <Ganneff> no option for you
42 # <DktrKranz> bdefreese: it seems ftp-masters want dinstall to sexual harass us, are you good in running?
43 # <Ganneff> you can run but you can not hide
44 # <bdefreese> No, I'm old and fat :)
45 # <Ganneff> you can roll but you can not hide
48 # <DktrKranz> damn dinstall, you racist bastard
50 ################################################################################
56 from shutil import rmtree
57 from yaml import safe_dump
58 from daklib.dbconn import *
59 from daklib import utils
60 from daklib.config import Config
61 from daklib.contents import UnpackedSource
62 from daklib.regexes import re_no_epoch
64 ################################################################################
66 filelist = 'filelist.yaml'
68 def usage (exit_code=0):
69 print """Generate changelog between two suites
72 make-changelog -s <suite> -b <base_suite> [OPTION]...
73 make-changelog -e -a <archive>
77 -h, --help show this help and exit
78 -s, --suite suite providing packages to compare
79 -b, --base-suite suite to be taken as reference for comparison
80 -n, --binnmu display binNMUs uploads instead of source ones
82 -e, --export export interesting files from source packages
83 -a, --archive archive to fetch data from
84 -p, --progress display progress status"""
88 def get_source_uploads(suite, base_suite, session):
90 Returns changelogs for source uploads where version is newer than base.
93 query = """WITH base AS (
94 SELECT source, max(version) AS version
96 WHERE suite_name = :base_suite
98 UNION (SELECT source, CAST(0 AS debversion) AS version
100 WHERE suite_name = :suite
101 EXCEPT SELECT source, CAST(0 AS debversion) AS version
103 WHERE suite_name = :base_suite
106 SELECT source, max(version) AS version
108 WHERE suite_name = :suite
110 SELECT DISTINCT c.source, c.version, c.changelog
112 JOIN base b ON b.source = c.source
113 JOIN cur_suite cs ON cs.source = c.source
114 WHERE c.version > b.version
115 AND c.version <= cs.version
116 AND c.architecture LIKE '%source%'
117 ORDER BY c.source, c.version DESC"""
119 return session.execute(query, {'suite': suite, 'base_suite': base_suite})
121 def get_binary_uploads(suite, base_suite, session):
123 Returns changelogs for binary uploads where version is newer than base.
126 query = """WITH base as (
127 SELECT s.source, max(b.version) AS version, a.arch_string
129 JOIN binaries b ON b.source = s.id
130 JOIN bin_associations ba ON ba.bin = b.id
131 JOIN architecture a ON a.id = b.architecture
135 WHERE suite_name = :base_suite)
136 GROUP BY s.source, a.arch_string),
138 SELECT s.source, max(b.version) AS version, a.arch_string
140 JOIN binaries b ON b.source = s.id
141 JOIN bin_associations ba ON ba.bin = b.id
142 JOIN architecture a ON a.id = b.architecture
146 WHERE suite_name = :suite)
147 GROUP BY s.source, a.arch_string)
148 SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
150 JOIN base b on b.source = c.source
151 JOIN cur_suite cs ON cs.source = c.source
152 WHERE c.version > b.version
153 AND c.version <= cs.version
154 AND c.architecture = b.arch_string
155 AND c.architecture = cs.arch_string
156 ORDER BY c.source, c.version DESC, c.architecture"""
158 return session.execute(query, {'suite': suite, 'base_suite': base_suite})
160 def display_changes(uploads, index):
162 for upload in uploads:
163 if prev_upload and prev_upload != upload[0]:
166 prev_upload = upload[0]
168 def export_files(session, archive, clpool, progress=False):
170 Export interesting files from source packages.
172 pool = os.path.join(archive.path, 'pool')
176 files = ('changelog', 'copyright', 'NEWS.Debian', 'README.Debian')
177 stats = {'unpack': 0, 'created': 0, 'removed': 0, 'errors': 0, 'files': 0}
178 query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
180 JOIN newest_source n ON n.source = s.source AND n.version = s.version
181 JOIN src_associations sa ON sa.source = s.id
182 JOIN suite su ON su.id = sa.suite
183 JOIN files f ON f.id = s.file
184 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
185 JOIN component c ON fam.component_id = c.id
186 WHERE su.archive_id = :archive_id
187 ORDER BY s.source, suite"""
189 for p in session.execute(query, {'archive_id': archive.archive_id}):
190 if not sources.has_key(p[0]):
192 sources[p[0]][p[1]] = (re_no_epoch.sub('', p[2]), p[3])
194 for p in sources.keys():
195 for s in sources[p].keys():
196 path = os.path.join(clpool, '/'.join(sources[p][s][1].split('/')[:-1]))
197 if not os.path.exists(path):
199 if not os.path.exists(os.path.join(path, \
200 '%s_%s_changelog' % (p, sources[p][s][0]))):
201 if not unpack.has_key(os.path.join(pool, sources[p][s][1])):
202 unpack[os.path.join(pool, sources[p][s][1])] = (path, set())
203 unpack[os.path.join(pool, sources[p][s][1])][1].add(s)
205 for file in glob('%s/%s_%s*' % (path, p, sources[p][s][0])):
206 link = '%s%s' % (s, file.split('%s_%s' \
207 % (p, sources[p][s][0]))[1])
209 os.unlink(os.path.join(path, link))
212 os.link(os.path.join(path, file), os.path.join(path, link))
214 for p in unpack.keys():
215 package = os.path.splitext(os.path.basename(p))[0].split('_')
217 unpacked = UnpackedSource(p)
218 tempdir = unpacked.get_root_directory()
221 if stats['unpack'] % 100 == 0:
222 sys.stderr.write('%d packages unpacked\n' % stats['unpack'])
223 elif stats['unpack'] % 10 == 0:
224 sys.stderr.write('.')
226 for f in glob(os.path.join(tempdir, 'debian', '*%s' % file)):
227 for s in unpack[p][1]:
228 suite = os.path.join(unpack[p][0], '%s_%s' \
229 % (s, os.path.basename(f)))
230 version = os.path.join(unpack[p][0], '%s_%s_%s' % \
231 (package[0], package[1], os.path.basename(f)))
232 if not os.path.exists(version):
234 stats['created'] += 1
239 os.link(version, suite)
240 stats['created'] += 1
242 except Exception as e:
243 print 'make-changelog: unable to unpack %s\n%s' % (p, e)
246 for root, dirs, files in os.walk(clpool, topdown=False):
247 files = [f for f in files if f != filelist]
250 if root.split('/')[-1] not in sources.keys():
251 if os.path.exists(root):
252 stats['removed'] += len(os.listdir(root))
255 if os.path.exists(os.path.join(root, file)):
256 if os.stat(os.path.join(root, file)).st_nlink == 1:
257 stats['removed'] += 1
258 os.unlink(os.path.join(root, file))
261 os.rmdir(os.path.join(root, dir))
264 stats['files'] += len(files)
265 stats['files'] -= stats['removed']
267 print 'make-changelog: file exporting finished'
268 print ' * New packages unpacked: %d' % stats['unpack']
269 print ' * New files created: %d' % stats['created']
270 print ' * New files removed: %d' % stats['removed']
271 print ' * Unpack errors: %d' % stats['errors']
272 print ' * Files available into changelog pool: %d' % stats['files']
274 def generate_export_filelist(clpool):
276 for root, dirs, files in os.walk(clpool):
277 for file in [f for f in files if f != filelist]:
278 clpath = os.path.join(root, file).replace(clpool, '').strip('/')
279 source = clpath.split('/')[2]
280 elements = clpath.split('/')[3].split('_')
281 if source not in clfiles:
283 if elements[0] == source:
284 if elements[1] not in clfiles[source]:
285 clfiles[source][elements[1]] = []
286 clfiles[source][elements[1]].append(clpath)
288 if elements[0] not in clfiles[source]:
289 clfiles[source][elements[0]] = []
290 clfiles[source][elements[0]].append(clpath)
291 with open(os.path.join(clpool, filelist), 'w+') as fd:
292 safe_dump(clfiles, fd, default_flow_style=False)
295 Cnf = utils.get_conf()
297 Arguments = [('h','help','Make-Changelog::Options::Help'),
298 ('a','archive','Make-Changelog::Options::Archive','HasArg'),
299 ('s','suite','Make-Changelog::Options::Suite','HasArg'),
300 ('b','base-suite','Make-Changelog::Options::Base-Suite','HasArg'),
301 ('n','binnmu','Make-Changelog::Options::binNMU'),
302 ('e','export','Make-Changelog::Options::export'),
303 ('p','progress','Make-Changelog::Options::progress')]
305 for i in ['help', 'suite', 'base-suite', 'binnmu', 'export', 'progress']:
306 if not Cnf.has_key('Make-Changelog::Options::%s' % (i)):
307 Cnf['Make-Changelog::Options::%s' % (i)] = ''
309 apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
310 Options = Cnf.subtree('Make-Changelog::Options')
311 suite = Cnf['Make-Changelog::Options::Suite']
312 base_suite = Cnf['Make-Changelog::Options::Base-Suite']
313 binnmu = Cnf['Make-Changelog::Options::binNMU']
314 export = Cnf['Make-Changelog::Options::export']
315 progress = Cnf['Make-Changelog::Options::progress']
317 if Options['help'] or not (suite and base_suite) and not export:
320 for s in suite, base_suite:
321 if not export and not get_suite(s):
322 utils.fubar('Invalid suite "%s"' % s)
324 session = DBConn().session()
328 archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
329 exportpath = os.path.join(Cnf['Dir::Export'], cnf.exportpath)
330 export_files(session, archive, exportpath, progress)
331 generate_export_filelist(exportpath)
333 utils.fubar('No changelog export path defined')
335 display_changes(get_binary_uploads(suite, base_suite, session), 3)
337 display_changes(get_source_uploads(suite, base_suite, session), 2)
341 if __name__ == '__main__':