9 STAT_VALIDITY = 300 # 5min limit on reporting stats
13 'tahoe_runtime_load_avg':
14 { 'statid': 'load_monitor.avg_load',
16 'configheader': '\n'.join(['graph_title Tahoe Runtime Load Average',
18 'graph_category tahoe',
19 'graph_info This graph shows average reactor delay',
21 'graph_config': '\n'.join(['%(name)s.label %(name)s',
22 '%(name)s.draw LINE1',
24 'graph_render': '\n'.join(['%(name)s.value %(value)s',
28 'tahoe_runtime_load_peak':
29 { 'statid': 'load_monitor.max_load',
31 'configheader': '\n'.join(['graph_title Tahoe Runtime Load Peak',
33 'graph_category tahoe',
34 'graph_info This graph shows peak reactor delay',
36 'graph_config': '\n'.join(['%(name)s.label %(name)s',
37 '%(name)s.draw LINE1',
39 'graph_render': '\n'.join(['%(name)s.value %(value)s',
43 # STORAGE ALLOCATION (BYTES)
44 'tahoe_storage_consumed':
45 { 'statid': 'storage_server.consumed',
47 'configheader': '\n'.join(['graph_title Tahoe Storage Server Space Consumed',
49 'graph_category tahoe_storage_server',
50 'graph_info This graph shows space consumed',
51 'graph_args --base 1024',
53 'graph_config': '\n'.join(['%(name)s.label %(name)s',
54 '%(name)s.draw LINE1',
56 'graph_render': '\n'.join(['%(name)s.value %(value)s',
60 'tahoe_storage_allocated':
61 { 'statid': 'storage_server.allocated',
63 'configheader': '\n'.join(['graph_title Tahoe Storage Server Space Allocated',
65 'graph_category tahoe_storage_server',
66 'graph_info This graph shows space allocated',
67 'graph_args --base 1024',
69 'graph_config': '\n'.join(['%(name)s.label %(name)s',
70 '%(name)s.draw LINE1',
72 'graph_render': '\n'.join(['%(name)s.value %(value)s',
76 'tahoe_storage_bytes_added':
77 { 'statid': 'storage_server.bytes_added',
78 'category': 'counters',
79 'configheader': '\n'.join(['graph_title Tahoe Storage Server Bytes Added',
81 'graph_category tahoe_storage_server',
82 'graph_info This graph shows cummulative bytes added',
84 'graph_config': '\n'.join(['%(name)s.label %(name)s',
85 '%(name)s.draw LINE1',
87 'graph_render': '\n'.join(['%(name)s.value %(value)s',
91 'tahoe_storage_bytes_freed':
92 { 'statid': 'storage_server.bytes_freed',
93 'category': 'counters',
94 'configheader': '\n'.join(['graph_title Tahoe Storage Server Bytes Removed',
96 'graph_category tahoe_storage_server',
97 'graph_info This graph shows cummulative bytes removed',
99 'graph_config': '\n'.join(['%(name)s.label %(name)s',
100 '%(name)s.draw LINE1',
102 'graph_render': '\n'.join(['%(name)s.value %(value)s',
106 'tahoe_storage_operations_allocate':
107 { 'statid': 'storage_server.allocate',
108 'category': 'counters',
109 'configheader': '\n'.join(['graph_title Tahoe Storage Server Allocate_Bucket Operations',
110 'graph_vlabel operations per second',
111 'graph_category tahoe_storage_server',
112 'graph_info This graph shows how many allocate_buckets operations occured per second. Each immutable file upload causes one such operation per server.',
114 'graph_config': '\n'.join(['%(name)s.label %(name)s',
115 '%(name)s.type DERIVE',
117 '%(name)s.draw LINE1',
119 'graph_render': '\n'.join(['%(name)s.value %(value)s',
123 'tahoe_storage_operations_get':
124 { 'statid': 'storage_server.get',
125 'category': 'counters',
126 'configheader': '\n'.join(['graph_title Tahoe Storage Server get_bucket Operations',
127 'graph_vlabel operations per second',
128 'graph_category tahoe_storage_server',
129 'graph_info This graph shows how many get_bucket operations occured per second. Each immutable file download/check causes one such operation per server.',
131 'graph_config': '\n'.join(['%(name)s.label %(name)s',
132 '%(name)s.type DERIVE',
134 '%(name)s.draw LINE1',
136 'graph_render': '\n'.join(['%(name)s.value %(value)s',
140 'tahoe_storage_operations_writev':
141 { 'statid': 'storage_server.writev',
142 'category': 'counters',
143 'configheader': '\n'.join(['graph_title Tahoe Storage Server writev Operations',
144 'graph_vlabel operations per second',
145 'graph_category tahoe_storage_server',
146 'graph_info This graph shows how many writev operations occured per second. Each mutable file / dirnode write causes one such operation per server.',
148 'graph_config': '\n'.join(['%(name)s.label %(name)s',
149 '%(name)s.type DERIVE',
151 '%(name)s.draw LINE1',
153 'graph_render': '\n'.join(['%(name)s.value %(value)s',
157 'tahoe_storage_operations_readv':
158 { 'statid': 'storage_server.readv',
159 'category': 'counters',
160 'configheader': '\n'.join(['graph_title Tahoe Storage Server readv Operations',
161 'graph_vlabel operations per second',
162 'graph_category tahoe_storage_server',
163 'graph_info This graph shows how many readv operations occured per second. Each dirnode read causes one such operation per server.',
165 'graph_config': '\n'.join(['%(name)s.label %(name)s',
166 '%(name)s.type DERIVE',
168 '%(name)s.draw LINE1',
170 'graph_render': '\n'.join(['%(name)s.value %(value)s',
176 'tahoe_helper_incoming_files':
177 { 'statid': 'chk_upload_helper.incoming_count',
179 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Incoming File Count',
180 'graph_vlabel n files',
181 'graph_category tahoe_helper',
182 'graph_info This graph shows number of incoming files',
184 'graph_config': '\n'.join(['%(name)s.label %(name)s',
185 '%(name)s.draw LINE1',
187 'graph_render': '\n'.join(['%(name)s.value %(value)s',
190 'tahoe_helper_incoming_filesize':
191 { 'statid': 'chk_upload_helper.incoming_size',
193 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Incoming File Size',
194 'graph_vlabel bytes',
195 'graph_category tahoe_helper',
196 'graph_info This graph shows total size of incoming files',
198 'graph_config': '\n'.join(['%(name)s.label %(name)s',
199 '%(name)s.draw LINE1',
201 'graph_render': '\n'.join(['%(name)s.value %(value)s',
204 'tahoe_helper_incoming_files_old':
205 { 'statid': 'chk_upload_helper.incoming_size_old',
207 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Incoming Old Files',
208 'graph_vlabel bytes',
209 'graph_category tahoe_helper',
210 'graph_info This graph shows total size of old incoming files',
212 'graph_config': '\n'.join(['%(name)s.label %(name)s',
213 '%(name)s.draw LINE1',
215 'graph_render': '\n'.join(['%(name)s.value %(value)s',
219 'tahoe_helper_encoding_files':
220 { 'statid': 'chk_upload_helper.encoding_count',
222 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Encoding File Count',
223 'graph_vlabel n files',
224 'graph_category tahoe_helper',
225 'graph_info This graph shows number of encoding files',
227 'graph_config': '\n'.join(['%(name)s.label %(name)s',
228 '%(name)s.draw LINE1',
230 'graph_render': '\n'.join(['%(name)s.value %(value)s',
233 'tahoe_helper_encoding_filesize':
234 { 'statid': 'chk_upload_helper.encoding_size',
236 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Encoding File Size',
237 'graph_vlabel bytes',
238 'graph_category tahoe_helper',
239 'graph_info This graph shows total size of encoding files',
241 'graph_config': '\n'.join(['%(name)s.label %(name)s',
242 '%(name)s.draw LINE1',
244 'graph_render': '\n'.join(['%(name)s.value %(value)s',
247 'tahoe_helper_encoding_files_old':
248 { 'statid': 'chk_upload_helper.encoding_size_old',
250 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Encoding Old Files',
251 'graph_vlabel bytes',
252 'graph_category tahoe_helper',
253 'graph_info This graph shows total size of old encoding files',
255 'graph_config': '\n'.join(['%(name)s.label %(name)s',
256 '%(name)s.draw LINE1',
258 'graph_render': '\n'.join(['%(name)s.value %(value)s',
262 'tahoe_helper_active_uploads':
263 { 'statid': 'chk_upload_helper.active_uploads',
265 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Active Files',
266 'graph_vlabel n files',
267 'graph_category tahoe_helper',
268 'graph_info This graph shows number of files actively being processed by the helper',
270 'graph_config': '\n'.join(['%(name)s.label %(name)s',
271 '%(name)s.draw LINE1',
273 'graph_render': '\n'.join(['%(name)s.value %(value)s',
277 'tahoe_helper_upload_requests':
278 { 'statid': 'chk_upload_helper.upload_requests',
279 'category': 'counters',
280 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Upload Requests',
281 'graph_vlabel requests',
282 'graph_category tahoe_helper',
283 'graph_info This graph shows the number of upload requests arriving at the helper',
285 'graph_config': '\n'.join(['%(name)s.label %(name)s',
286 '%(name)s.type DERIVE',
288 '%(name)s.draw LINE1',
290 'graph_render': '\n'.join(['%(name)s.value %(value)s',
293 'tahoe_helper_upload_already_present':
294 { 'statid': 'chk_upload_helper.upload_already_present',
295 'category': 'counters',
296 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Uploads Already Present',
297 'graph_vlabel requests',
298 'graph_category tahoe_helper',
299 'graph_info This graph shows the number of uploads whose files are already present in the grid',
301 'graph_config': '\n'.join(['%(name)s.label %(name)s',
302 '%(name)s.type DERIVE',
304 '%(name)s.draw LINE1',
306 'graph_render': '\n'.join(['%(name)s.value %(value)s',
309 'tahoe_helper_upload_need_upload':
310 { 'statid': 'chk_upload_helper.upload_need_upload',
311 'category': 'counters',
312 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Uploads Needing Upload',
313 'graph_vlabel requests',
314 'graph_category tahoe_helper',
315 'graph_info This graph shows the number of uploads whose files are not already present in the grid',
317 'graph_config': '\n'.join(['%(name)s.label %(name)s',
318 '%(name)s.type DERIVE',
320 '%(name)s.draw LINE1',
322 'graph_render': '\n'.join(['%(name)s.value %(value)s',
325 'tahoe_helper_encoded_bytes':
326 { 'statid': 'chk_upload_helper.encoded_bytes',
327 'category': 'counters',
328 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Encoded Bytes',
329 'graph_vlabel bytes',
330 'graph_category tahoe_helper',
331 'graph_info This graph shows the number of bytes encoded by the helper',
333 'graph_config': '\n'.join(['%(name)s.label %(name)s',
334 '%(name)s.type DERIVE',
336 '%(name)s.draw LINE1',
338 'graph_render': '\n'.join(['%(name)s.value %(value)s',
341 'tahoe_helper_fetched_bytes':
342 { 'statid': 'chk_upload_helper.fetched_bytes',
343 'category': 'counters',
344 'configheader': '\n'.join(['graph_title Tahoe Upload Helper Fetched Bytes',
345 'graph_vlabel bytes',
346 'graph_category tahoe_helper',
347 'graph_info This graph shows the number of bytes fetched by the helper',
349 'graph_config': '\n'.join(['%(name)s.label %(name)s',
350 '%(name)s.type DERIVE',
352 '%(name)s.draw LINE1',
354 'graph_render': '\n'.join(['%(name)s.value %(value)s',
359 'tahoe_uploader_bytes_uploaded':
360 { 'statid': 'uploader.bytes_uploaded',
361 'category': 'counters',
362 'configheader': '\n'.join(['graph_title Tahoe Uploader Bytes Uploaded',
363 'graph_vlabel bytes',
364 'graph_category tahoe_traffic',
365 'graph_info This graph shows the number of bytes uploaded',
367 'graph_config': '\n'.join(['%(name)s.label %(name)s',
368 '%(name)s.type DERIVE',
370 '%(name)s.draw LINE1',
372 'graph_render': '\n'.join(['%(name)s.value %(value)s',
375 'tahoe_uploader_files_uploaded':
376 { 'statid': 'uploader.files_uploaded',
377 'category': 'counters',
378 'configheader': '\n'.join(['graph_title Tahoe Uploader Bytes Uploaded',
379 'graph_vlabel files',
380 'graph_category tahoe_traffic',
381 'graph_info This graph shows the number of files uploaded',
383 'graph_config': '\n'.join(['%(name)s.label %(name)s',
384 '%(name)s.type DERIVE',
386 '%(name)s.draw LINE1',
388 'graph_render': '\n'.join(['%(name)s.value %(value)s',
391 'tahoe_mutable_files_published':
392 { 'statid': 'mutable.files_published',
393 'category': 'counters',
394 'configheader': '\n'.join(['graph_title Tahoe Mutable Files Published',
395 'graph_vlabel files',
396 'graph_category tahoe_traffic',
397 'graph_info This graph shows the number of mutable files published',
399 'graph_config': '\n'.join(['%(name)s.label %(name)s',
400 '%(name)s.type DERIVE',
402 '%(name)s.draw LINE1',
404 'graph_render': '\n'.join(['%(name)s.value %(value)s',
407 'tahoe_mutable_files_retrieved':
408 { 'statid': 'mutable.files_retrieved',
409 'category': 'counters',
410 'configheader': '\n'.join(['graph_title Tahoe Mutable Files Retrieved',
411 'graph_vlabel files',
412 'graph_category tahoe_traffic',
413 'graph_info This graph shows the number of files retrieved',
415 'graph_config': '\n'.join(['%(name)s.label %(name)s',
416 '%(name)s.type DERIVE',
418 '%(name)s.draw LINE1',
420 'graph_render': '\n'.join(['%(name)s.value %(value)s',
426 def smash_name(name):
427 return re.sub('[^a-zA-Z0-9]', '_', name)
429 def open_stats(fname):
430 f = open(fname, 'rb')
431 stats = pickle.load(f)
436 graph_name = os.path.basename(argv[0])
437 if graph_name.endswith('.py'):
438 graph_name = graph_name[:-3]
440 plugin_conf = PLUGINS.get(graph_name)
442 for k,v in os.environ.items():
443 if k.startswith('statsfile'):
447 raise RuntimeError("No 'statsfile' env var found")
449 stats = open_stats(stats_file)
452 def output_nodes(output_section, check_time):
453 for tubid, nodestats in stats.items():
454 if check_time and (now - nodestats.get('timestamp', 0)) > STAT_VALIDITY:
456 name = smash_name("%s_%s" % (nodestats['nickname'], tubid[:4]))
457 #value = nodestats['stats'][plugin_conf['category']].get(plugin_conf['statid'])
458 category = plugin_conf['category']
459 statid = plugin_conf['statid']
460 value = nodestats['stats'][category].get(statid)
461 if value is not None:
462 args = { 'name': name, 'value': value }
463 print plugin_conf[output_section] % args
466 if sys.argv[1] == 'config':
467 print plugin_conf['configheader']
468 output_nodes('graph_config', False)
471 output_nodes('graph_render', True)
473 if __name__ == '__main__':