1
2
3
4
5
6
7
8
9
10
11 __doc__="""RenderServer
12
13 Frontend that passes RRD graph options to rrdtool to render,
14 and then returns an URL to access the rendered graphic file.
15 """
16
17 import os
18 import time
19 import logging
20 import json
21 import urllib
22 import zlib
23 import mimetypes
24 import glob
25 import tarfile
26 import md5
27 import tempfile
28 from Products.ZenUtils import Map
29 from AccessControl import ClassSecurityInfo
30 from Globals import InitializeClass
31 from Globals import DTMLFile
32
33 try:
34 import rrdtool
35 except ImportError:
36 pass
37
38 from base64 import b64encode, urlsafe_b64decode, urlsafe_b64encode
39 from urllib import urlencode
40
41 from Products.ZenRRD.RRDUtil import fixMissingRRDs
42 from Products.ZenUtils.Utils import zenPath, rrd_daemon_args, rrd_daemon_retry
43
44 from RRDToolItem import RRDToolItem
45
46 from Products.ZenModel.PerformanceConf import performancePath
47
48 log = logging.getLogger("zen.RenderServer")
59
60
61 addRenderServer = DTMLFile('dtml/addRenderServer',globals())
62
63 DEFAULT_TIMEOUT=300
64 _cache = Map.Locked(Map.Timed({}, DEFAULT_TIMEOUT))
68 """
69 Base class for turning graph requests into graphics.
70 NB: Any log messages get logged into the event.log file.
71 """
72
73 meta_type = "RenderServer"
74
75 cacheName = 'RRDRenderCache'
76
77 security = ClassSecurityInfo()
78
80 self.id = id
81 self.tmpdir = tmpdir
82 self.cachetimeout = cachetimeout
83
84
85 security.declareProtected('View', 'render')
86 - def render(self, gopts=None, start=None, end=None, drange=None,
87 remoteUrl=None, width=None, ftype='PNG', getImage=True,
88 graphid='', comment=None, ms=None, remoteHost=None, REQUEST=None, zenrenderRequest=None):
89 """
90 Render a graph and return it
91
92 @param gopts: RRD graph creation options
93 @param start: requested start of data to graph
94 @param end: requested start of data to graph
95 @param drange: min/max values of the graph
96 @param remoteUrl: if the RRD is not here, where it lives -DEPRECATED use remoteHost instead
97 @param width: size of graphic to create
98 @param ftype: file type of graphic (eg PNG)
99 @param getImage: return the graph or a script location
100 @param graphid: (hopefully) unique identifier of a graph
101 @param comment: RRD graph comment
102 @param ms: a timestamp used to force IE to reload images
103 @param remoteHost: Forward current RRD request to renderserver at remoteHost. eg http://remotezenrender:8091/
104 @param REQUEST: URL-marshalled object containg URL options
105 @return: graph or script location
106 """
107
108
109 for tries in range(3):
110 try:
111 gopts = zlib.decompress(urlsafe_b64decode(gopts))
112 except Exception:
113 gopts = urllib.unquote(gopts)
114 else:
115 break
116
117 comment = urllib.unquote(comment) if comment is not None else ''
118
119 gopts = gopts.split('|')
120 gopts = fixMissingRRDs(gopts)
121 gopts.append('HRULE:INF#00000000')
122 gopts.append('--width=%s' % width)
123 if start:
124 gopts.append('--start=%s' % start)
125 if end:
126 gopts.append('--end=%s' % end)
127 drange = int(drange)
128 id = self.graphId(gopts, drange, ftype)
129 graph = self.getGraph(id, ftype, REQUEST)
130 if not graph:
131 if not os.path.exists(self.tmpdir):
132 os.makedirs(self.tmpdir, 0750)
133 fd, filename = tempfile.mkstemp(dir=self.tmpdir, suffix=id)
134 if remoteHost or remoteUrl:
135 if remoteHost:
136 encodedOpts = urlsafe_b64encode(
137 zlib.compress('|'.join(gopts), 9))
138 params = {
139 'gopts': encodedOpts,
140 'drange': drange,
141 'width': width,
142 }
143 remote = "%s/render?%s" %(remoteHost, urlencode(params))
144 else:
145 remote = remoteUrl
146 f = open(filename, "w")
147 response = urllib.urlopen(remote).read()
148 f.write(response)
149 f.close()
150 else:
151 if ftype.lower()=='html':
152 imgtype = 'PNG'
153 else:
154 imgtype = ftype
155 gopts.insert(0, "--imgformat=%s" % imgtype)
156
157 end = int(time.time())-300
158 start = end - drange
159 if comment is not None:
160 gopts.insert(0, 'COMMENT:%s\\c' % comment)
161 gopts.insert(0, '--end=%d' % end)
162 gopts.insert(0, '--start=%d' % start)
163 gopts.insert(0, filename)
164 log.debug("RRD graphing options: %r", (gopts,))
165 try:
166 @rrd_daemon_retry
167 def rrdtool_fn():
168 rrdtool.graph(*(gopts + list(rrd_daemon_args())))
169 rrdtool_fn()
170
171 except Exception, ex:
172 if ex.args[0].find('No such file or directory') > -1:
173 return None
174 log.exception("Failed to generate a graph")
175 log.warn(" ".join(gopts))
176 return None
177
178 self.addGraph(id, filename, fd)
179 graph = self.getGraph(id, ftype, REQUEST)
180
181 if getImage:
182 return graph
183 else:
184 success = bool(graph)
185 ret = {'success':success}
186 if success:
187 ret['data'] = b64encode(graph)
188 if REQUEST:
189 REQUEST.RESPONSE.setHeader('Content-Type', 'text/javascript')
190 elif zenrenderRequest:
191 zenrenderRequest.setHeader('Content-Type', 'text/javascript')
192 return """Zenoss.SWOOP_CALLBACKS["%s"]('%s')""" % (graphid, json.dumps(ret))
193
194
195 - def deleteRRDFiles(self, device,
196 datasource=None, datapoint=None,
197 remoteUrl=None, REQUEST=None):
198 """
199 Delete RRD files associated with the given device id.
200 If datapoint is not None then delete the file corresponding to that dp.
201 Else if datasource is not None then delete the files corresponding to
202 all datapoints in the datasource.
203 Else delete all RRD files associated with the given device.
204
205 @param device: device name
206 @param datasource: RRD datasource (DS) name
207 @param datapoint: RRD datapoint name (lives in a DS)
208 @param remoteUrl: if the RRD is not here, where it lives
209 @param REQUEST: URL-marshalled object containg URL options
210 """
211
212
213 if remoteUrl:
214 urllib.urlopen(remoteUrl)
215
216
217
218 devDir = performancePath('/Devices/%s' % device)
219 if not os.path.isdir(devDir):
220 return
221 fileNames = []
222 dirNames = []
223 if datapoint:
224 fileNames = [
225 performancePath('/Devices/%s/%s.rrd' % (device, datapoint))]
226 elif datasource:
227 rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource)
228 fileNames = glob.glob(performancePath(rrdPath))
229 else:
230 for dPath, dNames, dFiles in os.walk(devDir, topdown=False):
231 fileNames += [os.path.join(dPath, f) for f in dFiles]
232 dirNames += [os.path.join(dPath, d) for d in dNames]
233 dirNames.append(devDir)
234 for fileName in fileNames:
235 try:
236 os.remove(fileName)
237 except OSError:
238 log.warn("File %s does not exist" % fileName)
239 for dirName in dirNames:
240 try:
241 os.rmdir(dirName)
242 except OSError:
243 log.warn('Directory %s could not be removed' % dirName)
244
245
247 """
248 Tar up RRD files into a nice, neat package
249
250 @param device: device name
251 @param REQUEST: URL-marshalled object containg URL options
252 """
253 srcdir = performancePath('/Devices/%s' % device)
254 tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
255 log.debug( "tarring up %s into %s" % ( srcdir, tarfilename ))
256 tar = tarfile.open(tarfilename, "w:gz")
257 for file in os.listdir(srcdir):
258 tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file))
259 tar.close()
260
262 """
263 Untar a package of RRDFiles
264
265 @param device: device name
266 @param REQUEST: URL-marshalled object containg URL options
267 """
268 destdir = performancePath('/Devices/%s' % device)
269 tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
270 log.debug( "Untarring %s into %s" % ( tarfilename, destdir ))
271 tar = tarfile.open(tarfilename, "r:gz")
272 for file in tar.getmembers():
273 tar.extract(file, destdir)
274 tar.close()
275
277 """
278 Receive a device's RRD Files from another server
279 This function is called by sendRRDFiles()
280
281 @param REQUEST: 'tarfile', 'tarfilename'
282 @type REQUEST: URL-marshalled parameters
283 """
284 tarfile = REQUEST.get('tarfile')
285 tarfilename = REQUEST.get('tarfilename')
286 log.debug( "Receiving %s ..." % ( tarfilename ))
287 f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb')
288 f.write(urllib.unquote(tarfile))
289 f.close()
290
292 """
293 Move a package of RRDFiles
294
295 @param device: device name
296 @param server: another RenderServer instance
297 @param REQUEST: URL-marshalled object containg URL options
298 """
299 tarfilename = '%s.tgz' % device
300 f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb')
301 tarfilebody=f.read()
302 f.close()
303
304 params = urllib.urlencode({'tarfilename': tarfilename,
305 'tarfile':tarfilebody})
306
307
308 perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server)
309 if perfMon.getRemoteRenderUrl().startswith('http'):
310 remoteUrl = '%s/receiveRRDFiles' % (perfMon.getRemoteRenderUrl())
311 log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl ))
312 urllib.urlopen(remoteUrl, params)
313
314
315 - def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
339
340 security.declareProtected('View', 'plugin')
341 - def plugin(self, name, REQUEST=None):
342 """
343 Render a custom graph and return it
344
345 @param name: plugin name from Products/ZenRRD/plugins
346 @return: graph or None
347 """
348 try:
349 m = zenPath('Products/ZenRRD/plugins/%s.py' % name)
350 log.debug( "Trying plugin %s to generate a graph..." % m )
351 graph = None
352 exec open(m)
353 return graph
354 except Exception:
355 log.exception("Failed generating graph from plugin %s" % name)
356 raise
357
358
359 security.declareProtected('GenSummary', 'summary')
361 """
362 Return summary information as a list but no graph
363
364 @param gopts: RRD graph options
365 @return: values from the graph
366 """
367 gopts = fixMissingRRDs(gopts)
368 gopts.insert(0, '/dev/null')
369 log.debug("RRD summary options: %r", (gopts,))
370 try:
371 @rrd_daemon_retry
372 def rrdtool_fn():
373 return rrdtool.graph(*(gopts+list(rrd_daemon_args())))[2]
374 values = rrdtool_fn()
375 except Exception, ex:
376 if ex.args[0].find('No such file or directory') > -1:
377 return None
378 log.exception("Failed while generating summary")
379 log.warn(" ".join(gopts))
380 raise
381 return values
382
383
384 security.declareProtected('GenSummary', 'fetchValues')
385 - def fetchValues(self, paths, cf, resolution, start, end=""):
386 """
387 Return the values recorded in the RRD file between the start and end period
388
389 @param paths: path names to files
390 @param cf: RRD consolidation function to use
391 @param resolution: requested resolution of RRD data
392 @param start: requested start of data to graph
393 @param end: requested start of data to graph
394 @return: values from the RRD files in the paths
395 """
396 if not end:
397 end = "now"
398 values = []
399 try:
400 for path in paths:
401 @rrd_daemon_retry
402 def rrdtool_fn():
403 try:
404 values.append(rrdtool.fetch(
405 path, cf, "-r %d" % resolution,
406 "-s %s" % start, "-e %s" % end,
407 *rrd_daemon_args()))
408
409 except rrdtool.error, ex:
410 if 'No such file or directory' in ex.message:
411 values.append(None)
412 else:
413 raise
414
415 rrdtool_fn()
416 return values
417 except NameError:
418 log.exception("It appears that the rrdtool bindings are not installed properly.")
419 except Exception, ex:
420 log.exception("Failed while generating current values")
421 raise
422
423
424 security.declareProtected('GenSummary', 'currentValues')
426 """
427 Return the latest values recorded in the RRD file
428
429 @param paths: path names to files
430 @return: values from the RRD files in the path
431 """
432 try:
433 def value(p):
434 v = None
435 info = None
436 try:
437 @rrd_daemon_retry
438 def rrdtool_fn():
439 return rrdtool.info(p, *rrd_daemon_args())
440 info = rrdtool_fn()
441 except Exception:
442 log.debug('%s not found' % p)
443 if info:
444 last = info['last_update']
445 step = info['step']
446 gopts = ['/dev/null',
447 'DEF:x=%s:ds0:AVERAGE' % p,
448 'VDEF:v=x,LAST',
449 'PRINT:v:%.2lf',
450 '--start=%d'%(last-step),
451 '--end=%d'%last]
452 log.debug("RRD currentValue options: %r", (gopts,))
453 @rrd_daemon_retry
454 def rrdtool_fn():
455 return rrdtool.graph(*(gopts+list(rrd_daemon_args())))
456 v = rrdtool_fn()
457 v = float(v[2][0])
458 if str(v) == 'nan': v = None
459 return v
460 return map(value, paths)
461
462 except NameError:
463 log.exception("It appears that the rrdtool bindings are not installed properly.")
464
465 except Exception, ex:
466 if ex.args[0].find('No such file or directory') > -1:
467 return None
468 log.exception("Failed while generating current values")
469 raise
470
471 - def rrdcmd(self, gopts, ftype='PNG'):
472 """
473 Generate the RRD command using the graphing options specified.
474
475 @param gopts: RRD graphing options
476 @param ftype: graphic file type (eg PNG)
477 @return: RRD command usable on the command-line
478 @rtype: string
479 """
480 filename, gopts = self._setfile(gopts, ftype)
481 return "rrdtool graph " + " ".join(gopts)
482
483
484 - def graphId(self, gopts, drange, ftype):
485 """
486 Generate a graph id based on a hash of values
487
488 @param gopts: RRD graphing options
489 @param drange: min/max values of the graph
490 @param ftype: graphic file's type (eg PNG)
491 @return: An id for this graph usable in URLs
492 @rtype: string
493 """
494 id = md5.new(''.join(gopts)).hexdigest()
495 id += str(drange) + '.' + ftype.lower()
496 return id
497
499 try:
500 f = open(filename)
501 graph = f.read()
502 f.close()
503 return graph
504 except IOError:
505 log.info("File: %s not created yet." % filename);
506 return None
507
508
510 """
511 Make a new cache if we need one
512 """
513 if not hasattr(self, '_v_cache') or not self._v_cache:
514 self._v_cache = _cache
515 self._v_cache.map.timeout = self.cachetimeout
516 return self._v_cache
517
518
520 """
521 Add a graph to temporary folder
522
523 @param id: graph id
524 @param filename: cacheable graphic file
525 """
526 cache = self.setupCache()
527 graph = self._loadfile(filename)
528 if graph:
529 cache[id] = graph
530 try:
531 os.close(fd)
532 os.remove(filename)
533 except OSError, e:
534 if e.errno == 2:
535 log.debug("Unable to remove cached graph %s: %s" \
536 % (e.strerror, e.filename))
537 else:
538 raise e
539
540 - def getGraph(self, id, ftype, REQUEST):
541 """
542 Get a previously generated graph
543
544 @param id: graph id
545 @param ftype: file type of graphic (eg PNG)
546 @param REQUEST: graph id
547 """
548 cache = self.setupCache()
549 ftype = ftype.lower()
550
551 if REQUEST:
552 mimetype = mimetypes.guess_type('%s.%s' % (id, ftype))[0]
553 if mimetype is None:
554 mimetype = 'image/%s' % ftype
555 response = REQUEST.RESPONSE
556 response.setHeader('Content-Type', mimetype)
557
558 return cache.get(id, None)
559
560 InitializeClass(RenderServer)
561