1
2
3
4
5
6
7
8
9
10
11 __doc__ = """RRDUtil
12
13 Wrapper routines around the rrdtool library.
14 """
15
16 import logging
17 log = logging.getLogger("zen.RRDUtil")
18
19 import os
20 import re
21 import rrdtool
22 import string
23
24 from Products.ZenUtils.Utils import zenPath, rrd_daemon_args, rrd_daemon_retry
25
26
27 EMPTY_RRD = zenPath('perf', 'empty.rrd')
28
29 _UNWANTED_CHARS = ''.join(
30 set(string.punctuation + string.ascii_letters) - set(['.', '-', '+'])
31 )
32 _LAST_RRDFILE_WRITE = {}
36 """
37 Sanity check on the min, max values
38
39 @param x: RRD min or max value
40 @type x: number
41 @return: Either the number or 'U' (for undefined)
42 @rtype: number or string
43 """
44 if x is None or x == '' or x == -1 or x == '-1':
45 return 'U'
46 return x
47
50 """
51 Convert any value that is passed in to a string that is acceptable to use
52 for RRDtool's start and end parameters. Raises ValueError if this is not
53 possible.
54
55 See the AT-STYLE TIME SPECIFICATION and TIME REFERENCE SPECIFICATION
56 sections of the following document.
57
58 http://oss.oetiker.ch/rrdtool/doc/rrdfetch.en.html
59
60 Note: Currently this method is only fixing floats by turning them into
61 strings with no decimal places.
62 """
63
64 try:
65 result = int(val)
66 return str(result)
67 except ValueError:
68 pass
69
70 return str(val)
71
74 """
75 Parses a list of RRDtool gopts for DEFs. Runs all of the filenames
76 referenced by those DEFs through the fixRRDFilename method to make sure
77 that they will exist and not cause the rendering to fail.
78 """
79 fixed_gopts = []
80
81 def_match = re.compile(r'^DEF:([^=]+)=([^:]+)').match
82 for gopt in gopts:
83 match = def_match(gopt)
84 if not match:
85 fixed_gopts.append(gopt)
86 continue
87
88 rrd_filename = match.group(2)
89 fixed_gopts.append(gopt.replace(
90 rrd_filename, fixRRDFilename(rrd_filename)))
91
92 return fixed_gopts
93
96 """
97 Attempting to render a graph containing a DEF referencing a non-existent
98 filename will cause the entire graph to fail to render. This method is a
99 helper to verify existence of an RRD file. If the file doesn't exist, a
100 placeholder RRD filename with no values in it will be returned instead.
101 """
102 if os.path.isfile(filename):
103 return filename
104
105 if not os.path.isfile(EMPTY_RRD):
106 rrdtool.create(EMPTY_RRD, "--step", '300', 'DS:ds0:GAUGE:900:U:U',
107 'RRA:AVERAGE:0.5:1:1', 'RRA:MAX:0.5:1:1', 'RRA:LAST:0.5:1:1')
108
109 return EMPTY_RRD
110
111 -def read(path, consolidationFunction, start, end):
116 return rrdtool_fn()
117 except rrdtool.error, err:
118 import sys
119 err_str = '%s: %s' % (err.__class__.__name__, err)
120 msg = 'Failed to read RRD file %s. %s' % (path, err_str)
121 raise StandardError(msg), None, sys.exc_info()[2]
122
124 """
125 Wrapper class around rrdtool
126 """
127
128 - def __init__(self, defaultRrdCreateCommand, defaultCycleTime):
129 """
130 Initializer
131
132 The RRD creation command is only used if the RRD file doesn't
133 exist and no rrdCommand was specified with the save() method.
134
135 @param defaultRrdCreateCommand: RRD creation command
136 @type defaultRrdCreateCommand: string
137 @param defaultCycleTime: expected time to periodically collect data
138 @type defaultCycleTime: integer
139 """
140 self.defaultRrdCreateCommand = defaultRrdCreateCommand
141 self.defaultCycleTime = defaultCycleTime
142 self.dataPoints = 0
143 self.cycleDataPoints = 0
144
145
147 """
148 Report on the number of data points collected in a cycle,
149 and reset the counter for a new cycle.
150
151 @return: number of data points collected during the cycle
152 @rtype: number
153 """
154 result = self.cycleDataPoints
155 self.cycleDataPoints = 0
156 return result
157
158
171
172
174 """
175 Return the step value for the provided cycleTime. This is a hook for
176 altering the default step calculation.
177 """
178 return int(cycleTime)
179
180
182 """
183 Return the heartbeat value for the provided cycleTime. This is a hook
184 for altering the default heartbeat calculation.
185 """
186 return int(cycleTime) * 3
187
188
189 - def put(self, path, value, rrdType, rrdCommand=None, cycleTime=None,
190 min='U', max='U', useRRDDaemon=True, timestamp='N', start=None,
191 allowStaleDatapoint=True):
192 """
193 Save the value provided in the command to the RRD file specified in path.
194
195 If the RRD file does not exist, use the rrdType, rrdCommand, min and
196 max parameters to create the file.
197
198 @param path: name for a datapoint in a path (eg device/component/datasource_datapoint)
199 @type path: string
200 @param value: value to store into the RRD file
201 @type value: number
202 @param rrdType: RRD data type (eg ABSOLUTE, DERIVE, COUNTER)
203 @type rrdType: string
204 @param rrdCommand: RRD file creation command
205 @type rrdCommand: string
206 @param cycleTime: length of a cycle
207 @type cycleTime: number
208 @param min: minimum value acceptable for this metric
209 @type min: number
210 @param max: maximum value acceptable for this metric
211 @type max: number
212 @param allowStaleDatapoint: attempt to write datapoint even if a newer datapoint has already been written
213 @type allowStaleDatapoint: boolean
214 @return: the parameter value converted to a number
215 @rtype: number or None
216 """
217 if value is None: return None
218
219 self.dataPoints += 1
220 self.cycleDataPoints += 1
221
222 if cycleTime is None:
223 cycleTime = self.defaultCycleTime
224
225 filename = self.performancePath(path) + '.rrd'
226 if not rrdCommand:
227 rrdCommand = self.defaultRrdCreateCommand
228 if not os.path.exists(filename):
229 log.debug("Creating new RRD file %s", filename)
230 dirname = os.path.dirname(filename)
231 if not os.path.exists(dirname):
232 os.makedirs(dirname, 0750)
233
234 min, max = map(_checkUndefined, (min, max))
235 dataSource = 'DS:%s:%s:%d:%s:%s' % (
236 'ds0', rrdType, self.getHeartbeat(cycleTime), min, max)
237 args = [str(filename), "--step",
238 str(self.getStep(cycleTime)),]
239 if start is not None:
240 args.extend(["--start", "%d" % start])
241 elif timestamp != 'N':
242 args.extend(["--start", "%d" % int(timestamp) - 10])
243
244 args.append(str(dataSource))
245 args.extend(rrdCommand.split())
246 rrdtool.create(*args),
247
248 daemon_args = rrd_daemon_args() if useRRDDaemon else tuple()
249
250
251 value = str(value).translate(None, _UNWANTED_CHARS)
252
253 if rrdType in ('COUNTER', 'DERIVE'):
254 try:
255
256
257 value = long(float(value))
258 except (TypeError, ValueError):
259 return None
260 else:
261 try:
262 value = float(value)
263 except (TypeError, ValueError):
264 return None
265 try:
266 @rrd_daemon_retry
267 def rrdtool_fn():
268 return rrdtool.update(str(filename), *(daemon_args + ('%s:%s' % (timestamp, value),)))
269 if timestamp == 'N' or allowStaleDatapoint:
270 rrdtool_fn()
271 else:
272
273 lastTs = _LAST_RRDFILE_WRITE.get(filename, None)
274 if lastTs is None:
275 try:
276 lastTs = _LAST_RRDFILE_WRITE[filename] = rrdtool.last(
277 *(daemon_args + (str(filename),)))
278 except Exception as ex:
279 lastTs = 0
280 log.exception("Could not determine last update to %r", filename)
281
282 if lastTs < timestamp:
283 _LAST_RRDFILE_WRITE[filename] = timestamp
284 if log.getEffectiveLevel() < logging.DEBUG:
285 log.debug('%s: %r, currentTs = %s, lastTs = %s', filename, value, timestamp, lastTs)
286 rrdtool_fn()
287 else:
288 if log.getEffectiveLevel() < logging.DEBUG:
289 log.debug("ignoring write %s:%s", filename, timestamp)
290 return None
291
292 log.debug('%s: %r, @ %s', str(filename), value, timestamp)
293 except rrdtool.error, err:
294
295 log.error('rrdtool reported error %s %s', err, path)
296
297 return value
298
299
300 - def save(self, path, value, rrdType, rrdCommand=None, cycleTime=None,
301 min='U', max='U', useRRDDaemon=True, timestamp='N', start=None,
302 allowStaleDatapoint=True):
303 """
304 Save the value provided in the command to the RRD file specified in path.
305 Afterward, fetch the latest value for this point and return it.
306
307 If the RRD file does not exist, use the rrdType, rrdCommand, min and
308 max parameters to create the file.
309
310 @param path: name for a datapoint in a path (eg device/component/datasource_datapoint)
311 @type path: string
312 @param value: value to store into the RRD file
313 @type value: number
314 @param rrdType: RRD data type (eg ABSOLUTE, DERIVE, COUNTER)
315 @type rrdType: string
316 @param rrdCommand: RRD file creation command
317 @type rrdCommand: string
318 @param cycleTime: length of a cycle
319 @type cycleTime: number
320 @param min: minimum value acceptable for this metric
321 @type min: number
322 @param max: maximum value acceptable for this metric
323 @type max: number
324 @param allowStaleDatapoint: attempt to write datapoint even if a newer datapoint has already been written
325 @type allowStaleDatapoint: boolean
326 @return: the parameter value converted to a number
327 @rtype: number or None
328 """
329 value = self.put(path, value, rrdType, rrdCommand, cycleTime, min, max, useRRDDaemon, timestamp, start, allowStaleDatapoint)
330
331 if value is None:
332 return None
333
334 if rrdType in ('COUNTER', 'DERIVE'):
335 filename = self.performancePath(path) + '.rrd'
336 if cycleTime is None:
337 cycleTime = self.defaultCycleTime
338
339 @rrd_daemon_retry
340 def rrdtool_fn():
341 daemon_args = rrd_daemon_args() if useRRDDaemon else tuple()
342 return rrdtool.fetch(filename, 'AVERAGE',
343 '-s', 'now-%d' % (cycleTime*2),
344 '-e', 'now', *daemon_args)
345 startStop, names, values = rrdtool_fn()
346
347 values = [ v[0] for v in values if v[0] is not None ]
348 if values: value = values[-1]
349 else: value = None
350 return value
351