1
2
3
4
5
6
7
8
9
10
11
12
13
14 __doc__="""RenderServer
15
16 Frontend that passes RRD graph options to rrdtool to render,
17 and then returns an URL to access the rendered graphic file.
18 """
19
20 import os
21 import time
22 import logging
23 import urllib
24 import zlib
25 import mimetypes
26
27 from AccessControl import ClassSecurityInfo
28 from Globals import InitializeClass
29 from Globals import DTMLFile
30 from sets import Set
31
32 try:
33 import rrdtool
34 except ImportError:
35 pass
36
37 try:
38 from base64 import urlsafe_b64decode
39 raise ImportError
40 except ImportError:
42 import base64
43 return base64.decodestring(s.replace('-','+').replace('_','/'))
44
45 from Products.ZenUtils.PObjectCache import PObjectCache
46 from Products.ZenUtils.Utils import zenPath
47
48 from RRDToolItem import RRDToolItem
49
50 from Products.ZenModel.PerformanceConf import performancePath
51 import glob
52 import tarfile
53
54 log = logging.getLogger("RenderServer")
55
56
65
66
67 addRenderServer = DTMLFile('dtml/addRenderServer',globals())
68
69
71 """
72 Base class for turning graph requests into graphics.
73 NB: Any log messages get logged into the event.log file.
74 """
75
76 meta_type = "RenderServer"
77
78 cacheName = 'RRDRenderCache'
79
80 security = ClassSecurityInfo()
81
82 - def __init__(self, id, tmpdir = '/tmp/renderserver', cachetimeout=300):
83 self.id = id
84 self.tmpdir = tmpdir
85 self.cachetimeout = cachetimeout
86
87
89 """
90 Check the cmds list for DEF commands. For each one check that the rrd
91 file specified actually exists. Return a list of commands which
92 excludes commands referencing or depending on non-existent RRD files.
93
94 @param cmds: list of RRD commands
95 @return: sanitized list of RRD commands
96 """
97 newCmds = []
98 badNames = Set()
99 for cmd in cmds:
100 if cmd.startswith('DEF:'):
101
102 vName, rrdFile = cmd.split(':')[1].split('=', 1)
103 if not os.path.isfile(rrdFile):
104 badNames.add(vName)
105 parts = rrdFile.split('/')
106 try:
107 devIndex = parts.index('Devices') + 1
108 except ValueError:
109 devIndex = -1
110 devName = devIndex > 0 and parts[devIndex] or ''
111 compIndex = len(parts) - 2
112 compName = compIndex > devIndex and parts[compIndex] or ''
113 dpName = parts[-1].rsplit('.', 1)[0]
114 desc = ' '.join([p for p in (devName,compName,dpName) if p])
115 newCmds.append('COMMENT:MISSING RRD FILE\: %s' % desc)
116 continue
117
118 elif cmd.startswith('VDEF:') or cmd.startswith('CDEF:'):
119 vName, expression = cmd.split(':', 1)[1].split('=', 1)
120 if Set(expression.split(',')) & badNames:
121 badNames.add(vName)
122 continue
123
124 elif not cmd.startswith('COMMENT'):
125 try:
126 vName = cmd.split(':')[1].split('#')[0]
127 except IndexError:
128 vName = None
129 if vName in badNames:
130 continue
131 newCmds.append(cmd)
132 return newCmds
133
134
135 security.declareProtected('View', 'render')
136 - def render(self, gopts=None, start=None, end=None, drange=None,
137 remoteUrl=None, width=None, ftype='PNG', getImage=True,
138 graphid='', comment=None, ms=None, REQUEST=None):
139 """
140 Render a graph and return it
141
142 @param gopts: RRD graph creation options
143 @param start: requested start of data to graph
144 @param end: requested start of data to graph
145 @param drange: min/max values of the graph
146 @param remoteUrl: if the RRD is not here, where it lives
147 @param width: size of graphic to create
148 @param ftype: file type of graphic (eg PNG)
149 @param getImage: return the graph or a script location
150 @param graphid: (hopefully) unique identifier of a graph
151 @param comment: RRD graph comment
152 @param ms: a timestamp used to force IE to reload images
153 @param REQUEST: URL-marshalled object containg URL options
154 @return: graph or script location
155 """
156 gopts = zlib.decompress(urlsafe_b64decode(gopts))
157 gopts = gopts.split('|')
158 gopts = self.removeInvalidRRDReferences(gopts)
159 gopts.append('--width=%s' % width)
160 if start:
161 gopts.append('--start=%s' % start)
162 if end:
163 gopts.append('--end=%s' % end)
164 drange = int(drange)
165 id = self.graphId(gopts, drange, ftype)
166 graph = self.getGraph(id, ftype, REQUEST)
167 if not graph:
168 if not os.path.exists(self.tmpdir):
169 os.makedirs(self.tmpdir, 0750)
170 filename = "%s/graph-%s" % (self.tmpdir,id)
171 if remoteUrl:
172 f = open(filename, "w")
173 f.write(urllib.urlopen(remoteUrl).read())
174 f.close()
175 else:
176 if ftype.lower()=='html':
177 imgtype = 'PNG'
178 else:
179 imgtype = ftype
180 gopts.insert(0, "--imgformat=%s" % imgtype)
181
182 end = int(time.time())-300
183 start = end - drange
184 gopts.insert(0, 'COMMENT:%s\\c' % comment)
185 gopts.insert(0, '--end=%d' % end)
186 gopts.insert(0, '--start=%d' % start)
187 gopts.insert(0, filename)
188 log.debug("RRD graphing options: %r", (gopts,))
189 try:
190 rrdtool.graph(*gopts)
191 except Exception, ex:
192 if ex.args[0].find('No such file or directory') > -1:
193 return None
194 log.exception("Failed to generate a graph")
195 log.warn(" ".join(gopts))
196 return None
197
198 self.addGraph(id, filename)
199 graph = self.getGraph(id, ftype, REQUEST)
200
201 if getImage:
202 return graph
203 else:
204 return """
205 <script>
206 parent.location.hash = '%s:%s;';
207 </script>
208 """ % (graphid, str(bool(graph)))
209
210
211 - def deleteRRDFiles(self, device,
212 datasource=None, datapoint=None,
213 remoteUrl=None, REQUEST=None):
214 """
215 Delete RRD files associated with the given device id.
216 If datapoint is not None then delete the file corresponding to that dp.
217 Else if datasource is not None then delete the files corresponding to
218 all datapoints in the datasource.
219 Else delete all RRD files associated with the given device.
220
221 @param device: device name
222 @param datasource: RRD datasource (DS) name
223 @param datapoint: RRD datapoint name (lives in a DS)
224 @param remoteUrl: if the RRD is not here, where it lives
225 @param REQUEST: URL-marshalled object containg URL options
226 """
227 devDir = performancePath('/Devices/%s' % device)
228 if not os.path.isdir(devDir):
229 return
230 fileNames = []
231 dirNames = []
232 if datapoint:
233 fileNames = [
234 performancePath('/Devices/%s/%s.rrd' % (device, datapoint))]
235 elif datasource:
236 rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource)
237 fileNames = glob.glob(performancePath(rrdPath))
238 else:
239 for dPath, dNames, dFiles in os.walk(devDir, topdown=False):
240 fileNames += [os.path.join(dPath, f) for f in dFiles]
241 dirNames += [os.path.join(dPath, d) for d in dNames]
242 dirNames.append(devDir)
243 for fileName in fileNames:
244 try:
245 os.remove(fileName)
246 except OSError:
247 log.warn("File %s does not exist" % fileName)
248 for dirName in dirNames:
249 try:
250 os.rmdir(dirName)
251 except OSError:
252 log.warn('Directory %s could not be removed' % dirName)
253 if remoteUrl:
254 urllib.urlopen(remoteUrl)
255
256
258 """
259 Tar up RRD files into a nice, neat package
260
261 @param device: device name
262 @param REQUEST: URL-marshalled object containg URL options
263 """
264 srcdir = performancePath('/Devices/%s' % device)
265 tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
266 log.debug( "tarring up %s into %s" % ( srcdir, tarfilename ))
267 tar = tarfile.open(tarfilename, "w:gz")
268 for file in os.listdir(srcdir):
269 tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file))
270 tar.close()
271
273 """
274 Untar a package of RRDFiles
275
276 @param device: device name
277 @param REQUEST: URL-marshalled object containg URL options
278 """
279 destdir = performancePath('/Devices/%s' % device)
280 tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
281 log.debug( "Untarring %s into %s" % ( tarfilename, destdir ))
282 tar = tarfile.open(tarfilename, "r:gz")
283 for file in tar.getmembers():
284 tar.extract(file, destdir)
285 tar.close()
286
288 """
289 Receive a device's RRD Files from another server
290 This function is called by sendRRDFiles()
291
292 @param REQUEST: 'tarfile', 'tarfilename'
293 @type REQUEST: URL-marshalled parameters
294 """
295 tarfile = REQUEST.get('tarfile')
296 tarfilename = REQUEST.get('tarfilename')
297 log.debug( "Receiving %s ..." % ( tarfilename ))
298 f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb')
299 f.write(urllib.unquote(tarfile))
300 f.close()
301
303 """
304 Move a package of RRDFiles
305
306 @param device: device name
307 @param server: another RenderServer instance
308 @param REQUEST: URL-marshalled object containg URL options
309 """
310 tarfilename = '%s.tgz' % device
311 f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb')
312 tarfilebody=f.read()
313 f.close()
314
315 params = urllib.urlencode({'tarfilename': tarfilename,
316 'tarfile':tarfilebody})
317
318
319 perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server)
320 if perfMon.renderurl.startswith('http'):
321 remoteUrl = '%s/receiveRRDFiles' % (perfMon.renderurl)
322 log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl ))
323 urllib.urlopen(remoteUrl, params)
324
325
326 - def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
327 """
328 Send a device's RRD files to another server
329
330 @param device: device name
331 @param destServer: another RenderServer instance
332 @param srcServer: another RenderServer instance
333 @param REQUEST: URL-marshalled object containg URL options
334 """
335 monitors = self.dmd.getDmdRoot("Monitors")
336 destPerfMon = monitors.getPerformanceMonitor(destServer)
337 if srcServer:
338 srcPerfMon = monitors.getPerformanceMonitor(srcServer)
339 remoteUrl = '%s/moveRRDFiles?device=%s&destServer=%s' % (srcPerfMon.renderurl, device, destServer)
340 urllib.urlopen(remoteUrl)
341
342 else:
343 self.packageRRDFiles(device, REQUEST)
344 self.sendRRDFiles(device, destServer, REQUEST)
345 if destPerfMon.renderurl.startswith('http'):
346 remoteUrl = '%s/unpackageRRDFiles?device=%s' % (destPerfMon.renderurl, device)
347 urllib.urlopen(remoteUrl)
348 else:
349 self.unpackageRRDFiles(device, REQUEST)
350
351 security.declareProtected('View', 'plugin')
352 - def plugin(self, name, REQUEST=None):
353 """
354 Render a custom graph and return it
355
356 @param name: plugin name from Products/ZenRRD/plugins
357 @return: graph or None
358 """
359 try:
360 m = zenPath('Products/ZenRRD/plugins/%s.py' % name)
361 log.debug( "Trying plugin %s to generate a graph..." % m )
362 graph = None
363 exec open(m)
364 return graph
365 except Exception, ex:
366 log.exception("Failed generating graph from plugin %s" % name)
367 raise
368
369
370 security.declareProtected('GenSummary', 'summary')
372 """
373 Return summary information as a list but no graph
374
375 @param gopts: RRD graph options
376 @return: values from the graph
377 """
378 gopts.insert(0, '/dev/null')
379 try:
380 values = rrdtool.graph(*gopts)[2]
381 except Exception, ex:
382 if ex.args[0].find('No such file or directory') > -1:
383 return None
384 log.exception("Failed while generating summary")
385 log.warn(" ".join(gopts))
386 raise
387 return values
388
389
390 security.declareProtected('GenSummary', 'fetchValues')
391 - def fetchValues(self, paths, cf, resolution, start, end=""):
392 if not end:
393 end = "now"
394 values = []
395 try:
396 for path in paths:
397 values.append(rrdtool.fetch(path, cf, "-r %d" % resolution,
398 "-s %s" % start,"-e %s" % end))
399 return values
400 except NameError:
401 log.exception("It appears that the rrdtool bindings are not installed properly.")
402 except Exception, ex:
403 if ex.args[0].find('No such file or directory') > -1:
404 return None
405 log.exception("Failed while generating current values")
406 raise
407
408
409 security.declareProtected('GenSummary', 'currentValues')
411 """
412 Return the latest values recorded in the RRD file
413 """
414 try:
415 def value(p):
416 v = None
417 info = None
418 try:
419 info = rrdtool.info(p)
420 except:
421 log.debug('%s not found' % p)
422 if info:
423 last = info['last_update']
424 step = info['step']
425 v = rrdtool.graph('/dev/null',
426 'DEF:x=%s:ds0:AVERAGE' % p,
427 'VDEF:v=x,LAST',
428 'PRINT:v:%.2lf',
429 '--start=%d'%(last-step),
430 '--end=%d'%last)
431 v = float(v[2][0])
432 if str(v) == 'nan': v = None
433 return v
434 return map(value, paths)
435
436 except NameError:
437 log.exception("It appears that the rrdtool bindings are not installed properly.")
438
439 except Exception, ex:
440 if ex.args[0].find('No such file or directory') > -1:
441 return None
442 log.exception("Failed while generating current values")
443 raise
444
445
446 - def rrdcmd(self, gopts, ftype='PNG'):
447 """
448 Generate the RRD command using the graphing options specified.
449
450 @param gopts: RRD graphing options
451 @param ftype: graphic file type (eg PNG)
452 @return: RRD command usable on the command-line
453 @rtype: string
454 """
455 filename, gopts = self._setfile(gopts, ftype)
456 return "rrdtool graph " + " ".join(gopts)
457
458
459 - def graphId(self, gopts, drange, ftype):
460 """
461 Generate a graph id based on a hash of values
462
463 @param gopts: RRD graphing options
464 @param drange: min/max values of the graph
465 @param ftype: graphic file's type (eg PNG)
466 @return: An id for this graph usable in URLs
467 @rtype: string
468 """
469 import md5
470 id = md5.new(''.join(gopts)).hexdigest()
471 id += str(drange) + '.' + ftype.lower()
472 return id
473
475 try:
476 f = open(filename)
477 graph = f.read()
478 f.close()
479 return graph
480 except IOError:
481 log.info("File: %s not created yet." % filename);
482 return None
483
484
486 """
487 Make a new cache if we need one
488 """
489 if not hasattr(self, '_v_cache') or not self._v_cache:
490 tmpfolder = self.getPhysicalRoot().temp_folder
491 if not hasattr(tmpfolder, self.cacheName):
492 cache = PObjectCache(self.cacheName, self.cachetimeout)
493 tmpfolder._setObject(self.cacheName, cache)
494 self._v_cache = tmpfolder._getOb(self.cacheName)
495 return self._v_cache
496
497
499 """
500 Add a graph to temporary folder
501
502 @param id: graph id
503 @param filename: cacheable graphic file
504 """
505 cache = self.setupCache()
506 graph = self._loadfile(filename)
507 if graph:
508 cache.addToCache(id, graph)
509 try:
510 os.remove(filename)
511 except OSError, e:
512 if e.errno == 2:
513 log.debug("Unable to remove cached graph %s: %s" \
514 % (e.strerror, e.filename))
515 else:
516 raise e
517 cache.cleanCache()
518
519
520 - def getGraph(self, id, ftype, REQUEST):
521 """
522 Get a previously generated graph
523
524 @param id: graph id
525 @param ftype: file type of graphic (eg PNG)
526 @param REQUEST: graph id
527 """
528 cache = self.setupCache()
529 ftype = ftype.lower()
530
531 if REQUEST:
532 mimetype = mimetypes.guess_type('.%s'%ftype)[0]
533 if not mimetype:
534 mimetype = 'image/%s' % ftype
535 response = REQUEST.RESPONSE
536 response.setHeader('Content-Type', mimetype)
537
538 return cache.checkCache(id)
539
540
541 InitializeClass(RenderServer)
542