3 """Emulate iostat for NFS mount points using /proc/self/mountstats
7 Copyright (C) 2005, Chuck Lever <cel@netapp.com>
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License version 2 as
11 published by the Free Software Foundation.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
25 from optparse import OptionParser, OptionGroup
27 Iostats_version = '0.2'
30 """Used for a map() function
74 """DeviceData objects provide methods for parsing and displaying
75 data for a single mount grabbed from /proc/self/mountstats
78 self.__nfs_data = dict()
79 self.__rpc_data = dict()
80 self.__rpc_data['ops'] = []
82 def __parse_nfs_line(self, words):
83 if words[0] == 'device':
84 self.__nfs_data['export'] = words[1]
85 self.__nfs_data['mountpoint'] = words[4]
86 self.__nfs_data['fstype'] = words[7]
88 self.__nfs_data['statvers'] = words[8]
89 elif 'nfs' in words or 'nfs4' in words:
90 self.__nfs_data['export'] = words[0]
91 self.__nfs_data['mountpoint'] = words[3]
92 self.__nfs_data['fstype'] = words[6]
94 self.__nfs_data['statvers'] = words[7]
95 elif words[0] == 'age:':
96 self.__nfs_data['age'] = long(words[1])
97 elif words[0] == 'opts:':
98 self.__nfs_data['mountoptions'] = ''.join(words[1:]).split(',')
99 elif words[0] == 'caps:':
100 self.__nfs_data['servercapabilities'] = ''.join(words[1:]).split(',')
101 elif words[0] == 'nfsv4:':
102 self.__nfs_data['nfsv4flags'] = ''.join(words[1:]).split(',')
103 elif words[0] == 'sec:':
104 keys = ''.join(words[1:]).split(',')
105 self.__nfs_data['flavor'] = int(keys[0].split('=')[1])
106 self.__nfs_data['pseudoflavor'] = 0
107 if self.__nfs_data['flavor'] == 6:
108 self.__nfs_data['pseudoflavor'] = int(keys[1].split('=')[1])
109 elif words[0] == 'events:':
111 for key in NfsEventCounters:
112 self.__nfs_data[key] = int(words[i])
114 elif words[0] == 'bytes:':
116 for key in NfsByteCounters:
117 self.__nfs_data[key] = long(words[i])
120 def __parse_rpc_line(self, words):
121 if words[0] == 'RPC':
122 self.__rpc_data['statsvers'] = float(words[3])
123 self.__rpc_data['programversion'] = words[5]
124 elif words[0] == 'xprt:':
125 self.__rpc_data['protocol'] = words[1]
126 if words[1] == 'udp':
127 self.__rpc_data['port'] = int(words[2])
128 self.__rpc_data['bind_count'] = int(words[3])
129 self.__rpc_data['rpcsends'] = int(words[4])
130 self.__rpc_data['rpcreceives'] = int(words[5])
131 self.__rpc_data['badxids'] = int(words[6])
132 self.__rpc_data['inflightsends'] = long(words[7])
133 self.__rpc_data['backlogutil'] = long(words[8])
134 elif words[1] == 'tcp':
135 self.__rpc_data['port'] = words[2]
136 self.__rpc_data['bind_count'] = int(words[3])
137 self.__rpc_data['connect_count'] = int(words[4])
138 self.__rpc_data['connect_time'] = int(words[5])
139 self.__rpc_data['idle_time'] = int(words[6])
140 self.__rpc_data['rpcsends'] = int(words[7])
141 self.__rpc_data['rpcreceives'] = int(words[8])
142 self.__rpc_data['badxids'] = int(words[9])
143 self.__rpc_data['inflightsends'] = long(words[10])
144 self.__rpc_data['backlogutil'] = long(words[11])
145 elif words[1] == 'rdma':
146 self.__rpc_data['port'] = words[2]
147 self.__rpc_data['bind_count'] = int(words[3])
148 self.__rpc_data['connect_count'] = int(words[4])
149 self.__rpc_data['connect_time'] = int(words[5])
150 self.__rpc_data['idle_time'] = int(words[6])
151 self.__rpc_data['rpcsends'] = int(words[7])
152 self.__rpc_data['rpcreceives'] = int(words[8])
153 self.__rpc_data['badxids'] = int(words[9])
154 self.__rpc_data['backlogutil'] = int(words[10])
155 self.__rpc_data['read_chunks'] = int(words[11])
156 self.__rpc_data['write_chunks'] = int(words[12])
157 self.__rpc_data['reply_chunks'] = int(words[13])
158 self.__rpc_data['total_rdma_req'] = int(words[14])
159 self.__rpc_data['total_rdma_rep'] = int(words[15])
160 self.__rpc_data['pullup'] = int(words[16])
161 self.__rpc_data['fixup'] = int(words[17])
162 self.__rpc_data['hardway'] = int(words[18])
163 self.__rpc_data['failed_marshal'] = int(words[19])
164 self.__rpc_data['bad_reply'] = int(words[20])
165 elif words[0] == 'per-op':
166 self.__rpc_data['per-op'] = words
169 self.__rpc_data['ops'] += [op]
170 self.__rpc_data[op] = [long(word) for word in words[1:]]
172 def parse_stats(self, lines):
173 """Turn a list of lines from a mount stat file into a
174 dictionary full of stats, keyed by name
181 if (not found and words[0] != 'RPC'):
182 self.__parse_nfs_line(words)
186 self.__parse_rpc_line(words)
188 def is_nfs_mountpoint(self):
189 """Return True if this is an NFS or NFSv4 mountpoint,
190 otherwise return False
192 if self.__nfs_data['fstype'] == 'nfs':
194 elif self.__nfs_data['fstype'] == 'nfs4':
198 def compare_iostats(self, old_stats):
199 """Return the difference between two sets of stats
201 result = DeviceData()
203 # copy self into result
204 for key, value in self.__nfs_data.iteritems():
205 result.__nfs_data[key] = value
206 for key, value in self.__rpc_data.iteritems():
207 result.__rpc_data[key] = value
209 # compute the difference of each item in the list
210 # note the copy loop above does not copy the lists, just
211 # the reference to them. so we build new lists here
212 # for the result object.
213 for op in result.__rpc_data['ops']:
214 result.__rpc_data[op] = map(difference, self.__rpc_data[op], old_stats.__rpc_data[op])
216 # update the remaining keys we care about
217 result.__rpc_data['rpcsends'] -= old_stats.__rpc_data['rpcsends']
218 result.__rpc_data['backlogutil'] -= old_stats.__rpc_data['backlogutil']
220 for key in NfsEventCounters:
221 result.__nfs_data[key] -= old_stats.__nfs_data[key]
222 for key in NfsByteCounters:
223 result.__nfs_data[key] -= old_stats.__nfs_data[key]
227 def __print_data_cache_stats(self):
228 """Print the data cache hit rate
230 nfs_stats = self.__nfs_data
231 app_bytes_read = float(nfs_stats['normalreadbytes'])
232 if app_bytes_read != 0:
233 client_bytes_read = float(nfs_stats['serverreadbytes'] - nfs_stats['directreadbytes'])
234 ratio = ((app_bytes_read - client_bytes_read) * 100) / app_bytes_read
237 print 'app bytes: %f client bytes %f' % (app_bytes_read, client_bytes_read)
238 print 'Data cache hit ratio: %4.2f%%' % ratio
240 def __print_attr_cache_stats(self, sample_time):
241 """Print attribute cache efficiency stats
243 nfs_stats = self.__nfs_data
244 getattr_stats = self.__rpc_data['GETATTR']
246 if nfs_stats['inoderevalidates'] != 0:
247 getattr_ops = float(getattr_stats[1])
248 opens = float(nfs_stats['vfsopen'])
249 revalidates = float(nfs_stats['inoderevalidates']) - opens
251 ratio = ((revalidates - getattr_ops) * 100) / revalidates
255 data_invalidates = float(nfs_stats['datainvalidates'])
256 attr_invalidates = float(nfs_stats['attrinvalidates'])
259 print '%d inode revalidations, hitting in cache %4.2f%% of the time' % \
261 print '%d open operations (mandatory GETATTR requests)' % opens
263 print '%4.2f%% of GETATTRs resulted in data cache invalidations' % \
264 ((data_invalidates * 100) / getattr_ops)
266 def __print_dir_cache_stats(self, sample_time):
267 """Print directory stats
269 nfs_stats = self.__nfs_data
270 lookup_ops = self.__rpc_data['LOOKUP'][0]
271 readdir_ops = self.__rpc_data['READDIR'][0]
272 if self.__rpc_data.has_key('READDIRPLUS'):
273 readdir_ops += self.__rpc_data['READDIRPLUS'][0]
275 dentry_revals = nfs_stats['dentryrevalidates']
276 opens = nfs_stats['vfsopen']
277 lookups = nfs_stats['vfslookup']
278 getdents = nfs_stats['vfsreaddir']
281 print '%d open operations (pathname lookups)' % opens
282 print '%d dentry revalidates and %d vfs lookup requests' % \
283 (dentry_revals, lookups),
284 print 'resulted in %d LOOKUPs on the wire' % lookup_ops
285 print '%d vfs getdents calls resulted in %d READDIRs on the wire' % \
286 (getdents, readdir_ops)
288 def __print_page_stats(self, sample_time):
289 """Print page cache stats
291 nfs_stats = self.__nfs_data
293 vfsreadpage = nfs_stats['vfsreadpage']
294 vfsreadpages = nfs_stats['vfsreadpages']
295 pages_read = nfs_stats['readpages']
296 vfswritepage = nfs_stats['vfswritepage']
297 vfswritepages = nfs_stats['vfswritepages']
298 pages_written = nfs_stats['writepages']
301 print '%d nfs_readpage() calls read %d pages' % \
302 (vfsreadpage, vfsreadpage)
303 print '%d nfs_readpages() calls read %d pages' % \
304 (vfsreadpages, pages_read - vfsreadpage),
305 if vfsreadpages != 0:
306 print '(%.1f pages per call)' % \
307 (float(pages_read - vfsreadpage) / vfsreadpages)
312 print '%d nfs_updatepage() calls' % nfs_stats['vfsupdatepage']
313 print '%d nfs_writepage() calls wrote %d pages' % \
314 (vfswritepage, vfswritepage)
315 print '%d nfs_writepages() calls wrote %d pages' % \
316 (vfswritepages, pages_written - vfswritepage),
317 if (vfswritepages) != 0:
318 print '(%.1f pages per call)' % \
319 (float(pages_written - vfswritepage) / vfswritepages)
323 congestionwaits = nfs_stats['congestionwait']
324 if congestionwaits != 0:
326 print '%d congestion waits' % congestionwaits
328 def __print_rpc_op_stats(self, op, sample_time):
329 """Print generic stats for one RPC op
331 if not self.__rpc_data.has_key(op):
334 rpc_stats = self.__rpc_data[op]
335 ops = float(rpc_stats[0])
336 retrans = float(rpc_stats[1] - rpc_stats[0])
337 kilobytes = float(rpc_stats[3] + rpc_stats[4]) / 1024
338 rtt = float(rpc_stats[6])
339 exe = float(rpc_stats[7])
341 # prevent floating point exceptions
343 kb_per_op = kilobytes / ops
344 retrans_percent = (retrans * 100) / ops
345 rtt_per_op = rtt / ops
346 exe_per_op = exe / ops
349 retrans_percent = 0.0
354 print '%s' % op.lower().ljust(15),
355 print ' ops/s\t\t kB/s\t\t kB/op\t\tretrans\t\tavg RTT (ms)\tavg exe (ms)'
357 print '\t\t%7.3f' % (ops / sample_time),
358 print '\t%7.3f' % (kilobytes / sample_time),
359 print '\t%7.3f' % kb_per_op,
360 print ' %7d (%3.1f%%)' % (retrans, retrans_percent),
361 print '\t%7.3f' % rtt_per_op,
362 print '\t%7.3f' % exe_per_op
364 def ops(self, sample_time):
365 sends = float(self.__rpc_data['rpcsends'])
367 sample_time = float(self.__nfs_data['age'])
368 return (sends / sample_time)
370 def display_iostats(self, sample_time, which):
371 """Display NFS and RPC stats in an iostat-like way
373 sends = float(self.__rpc_data['rpcsends'])
375 sample_time = float(self.__nfs_data['age'])
376 # sample_time could still be zero if the export was just mounted.
377 # Set it to 1 to avoid divide by zero errors in this case since we'll
378 # likely still have relevant mount statistics to show.
383 backlog = (float(self.__rpc_data['backlogutil']) / sends) / sample_time
388 print '%s mounted on %s:' % \
389 (self.__nfs_data['export'], self.__nfs_data['mountpoint'])
392 print ' op/s\t\trpc bklog'
393 print '%7.2f' % (sends / sample_time),
394 print '\t%7.2f' % backlog
397 self.__print_rpc_op_stats('READ', sample_time)
398 self.__print_rpc_op_stats('WRITE', sample_time)
400 self.__print_rpc_op_stats('GETATTR', sample_time)
401 self.__print_rpc_op_stats('ACCESS', sample_time)
402 self.__print_attr_cache_stats(sample_time)
404 self.__print_rpc_op_stats('LOOKUP', sample_time)
405 self.__print_rpc_op_stats('READDIR', sample_time)
406 if self.__rpc_data.has_key('READDIRPLUS'):
407 self.__print_rpc_op_stats('READDIRPLUS', sample_time)
408 self.__print_dir_cache_stats(sample_time)
410 self.__print_rpc_op_stats('READ', sample_time)
411 self.__print_rpc_op_stats('WRITE', sample_time)
412 self.__print_page_stats(sample_time)
418 def parse_stats_file(filename):
419 """pop the contents of a mountstats file into a dictionary,
420 keyed by mount point. each value object is a list of the
421 lines in the mountstats file corresponding to the mount
422 point named in the key.
428 for line in f.readlines():
432 if words[0] == 'device':
434 new = [ line.strip() ]
435 elif 'nfs' in words or 'nfs4' in words:
437 new = [ line.strip() ]
439 new += [ line.strip() ]
445 def print_iostat_summary(old, new, devices, time, options):
449 # Trim device list to only include intersection of old and new data,
450 # this addresses umounts due to autofs mountpoints
451 devicelist = filter(lambda x:x in devices,old)
455 for device in devicelist:
456 stats[device] = DeviceData()
457 stats[device].parse_stats(new[device])
459 old_stats = DeviceData()
460 old_stats.parse_stats(old[device])
461 diff_stats[device] = stats[device].compare_iostats(old_stats)
465 # We now have compared data and can print a comparison
466 # ordered by mountpoint ops per second
467 devicelist.sort(key=lambda x: diff_stats[x].ops(time), reverse=True)
469 # First iteration, just sort by newly parsed ops/s
470 devicelist.sort(key=lambda x: stats[x].ops(time), reverse=True)
473 for device in devicelist:
475 diff_stats[device].display_iostats(time, options.which)
477 stats[device].display_iostats(time, options.which)
480 if (count > options.list):
484 def list_nfs_mounts(givenlist, mountstats):
485 """return a list of NFS mounts given a list to validate or
486 return a full list if the given list is empty -
487 may return an empty list if none found
490 if len(givenlist) > 0:
491 for device in givenlist:
493 stats.parse_stats(mountstats[device])
494 if stats.is_nfs_mountpoint():
497 for device, descr in mountstats.iteritems():
499 stats.parse_stats(descr)
500 if stats.is_nfs_mountpoint():
504 def iostat_command(name):
505 """iostat-like command for NFS mount points
507 mountstats = parse_stats_file('/proc/self/mountstats')
510 interval_seen = False
514 Sample iostat-like program to display NFS client per-mount'
515 statistics. The <interval> parameter specifies the amount of time in seconds
516 between each report. The first report contains statistics for the time since
517 each file system was mounted. Each subsequent report contains statistics
518 collected during the interval since the previous report. If the <count>
519 parameter is specified, the value of <count> determines the number of reports
520 generated at <interval> seconds apart. If the interval parameter is specified
521 without the <count> parameter, the command generates reports continuously.
522 If one or more <mount point> names are specified, statistics for only these
523 mount points will be displayed. Otherwise, all NFS mount points on the
526 parser = OptionParser(
527 usage="usage: %prog [ <interval> [ <count> ] ] [ <options> ] [ <mount point> ]",
528 description=mydescription,
529 version='version %s' % Iostats_version)
530 parser.set_defaults(which=0, sort=False, list=sys.maxint)
532 statgroup = OptionGroup(parser, "Statistics Options",
533 'File I/O is displayed unless one of the following is specified:')
534 statgroup.add_option('-a', '--attr',
535 action="store_const",
538 help='displays statistics related to the attribute cache')
539 statgroup.add_option('-d', '--dir',
540 action="store_const",
543 help='displays statistics related to directory operations')
544 statgroup.add_option('-p', '--page',
545 action="store_const",
548 help='displays statistics related to the page cache')
549 parser.add_option_group(statgroup)
550 displaygroup = OptionGroup(parser, "Display Options",
551 'Options affecting display format:')
552 displaygroup.add_option('-s', '--sort',
555 help="Sort NFS mount points by ops/second")
556 displaygroup.add_option('-l','--list',
560 help="only print stats for first LIST mount points")
561 parser.add_option_group(displaygroup)
563 (options, args) = parser.parse_args(sys.argv)
566 if arg == sys.argv[0]:
569 if arg in mountstats:
571 elif not interval_seen:
575 print 'Illegal <interval> value %s' % arg
580 print 'Illegal <interval> value %s' % arg
586 print 'Ilegal <count> value %s' % arg
591 print 'Illegal <count> value %s' % arg
594 # make certain devices contains only NFS mount points
595 devices = list_nfs_mounts(origdevices, mountstats)
596 if len(devices) == 0:
597 print 'No NFS mount points were found'
601 old_mountstats = None
604 if not interval_seen:
605 print_iostat_summary(old_mountstats, mountstats, devices, sample_time, options)
610 print_iostat_summary(old_mountstats, mountstats, devices, sample_time, options)
611 old_mountstats = mountstats
613 sample_time = interval
614 mountstats = parse_stats_file('/proc/self/mountstats')
615 # automount mountpoints add and drop, if automount is involved
616 # we need to recheck the devices list when reparsing
617 devices = list_nfs_mounts(origdevices,mountstats)
618 if len(devices) == 0:
619 print 'No NFS mount points were found'
624 print_iostat_summary(old_mountstats, mountstats, devices, sample_time, options)
625 old_mountstats = mountstats
627 sample_time = interval
628 mountstats = parse_stats_file('/proc/self/mountstats')
629 # automount mountpoints add and drop, if automount is involved
630 # we need to recheck the devices list when reparsing
631 devices = list_nfs_mounts(origdevices,mountstats)
632 if len(devices) == 0:
633 print 'No NFS mount points were found'
639 prog = os.path.basename(sys.argv[0])
643 except KeyboardInterrupt:
644 print 'Caught ^C... exiting'