Package Products :: Package ZenRRD :: Module RenderServer |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRRD.RenderServer

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2007, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 or (at your
      8  # option) any later version as published by the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13
     14  __doc__="""RenderServer
     15
     16  Frontend that passes RRD graph options to rrdtool to render,
     17  and then returns an URL to access the rendered graphic file.
     18  """
     19
     20  import os
     21  import time
     22  import logging
     23  import json
     24  import urllib
     25  import zlib
     26  import mimetypes
     27  import glob
     28  import tarfile
     29  import md5
     30
     31  from AccessControl import ClassSecurityInfo
     32  from Globals import InitializeClass
     33  from Globals import DTMLFile
     34
     35  try:
     36      import rrdtool
     37  except ImportError:
     38      pass
     39
     40  from base64 import b64encode, urlsafe_b64decode
     41
     42  from Products.ZenRRD.RRDUtil import fixMissingRRDs
     43  from Products.ZenUtils.PObjectCache import PObjectCache
     44  from Products.ZenUtils.Utils import zenPath, rrd_daemon_running
     45
     46  from RRDToolItem import RRDToolItem
     47
     48  from Products.ZenModel.PerformanceConf import performancePath
     49
     50  log = logging.getLogger("RenderServer")
     51
     52
     53 -def manage_addRenderServer(context, id, REQUEST = None):
     54      """
     55      Make a RenderServer
     56      """
     57      rs = RenderServer(id)
     58      context._setObject(id, rs)
     59      if REQUEST is not None:
     60          REQUEST['RESPONSE'].redirect(context.absolute_url()+'/manage_main')
     61
     62
     63  addRenderServer = DTMLFile('dtml/addRenderServer',globals())
     64
     65
     66 -class RenderServer(RRDToolItem):
     67      """
     68      Base class for turning graph requests into graphics.
     69      NB: Any log messages get logged into the event.log file.
     70      """
     71
     72      meta_type = "RenderServer"
     73
     74      cacheName = 'RRDRenderCache'
     75
     76      security = ClassSecurityInfo()
     77
     78 -    def __init__(self, id, tmpdir = '/tmp/renderserver', cachetimeout=300):
     79          self.id = id
     80          self.tmpdir = tmpdir
     81          self.cachetimeout = cachetimeout
     82
     83
     84      security.declareProtected('View', 'render')
     85 -    def render(self, gopts=None, start=None, end=None, drange=None,
     86                 remoteUrl=None, width=None, ftype='PNG', getImage=True,
     87                 graphid='', comment=None, ms=None, REQUEST=None, zenrenderRequest=None):
     88          """
     89          Render a graph and return it
     90
     91          @param gopts: RRD graph creation options
     92          @param start: requested start of data to graph
     93          @param end: requested start of data to graph
     94          @param drange: min/max values of the graph
     95          @param remoteUrl: if the RRD is not here, where it lives
     96          @param width: size of graphic to create
     97          @param ftype: file type of graphic (eg PNG)
     98          @param getImage: return the graph or a script location
     99          @param graphid: (hopefully) unique identifier of a graph
    100          @param comment: RRD graph comment
    101          @param ms: a timestamp used to force IE to reload images
    102          @param REQUEST: URL-marshalled object containg URL options
    103          @return: graph or script location
    104          """
    105
    106          # gopts may have repeated url quoting, possibly from multiple hops thru remote zenhubs
    107          # extra quoting will create invalid zlib padding characters ('%3D' instead of '=')
    108          for tries in range(3):
    109              try:
    110                  gopts = zlib.decompress(urlsafe_b64decode(gopts))
    111              except Exception:
    112                  gopts = urllib.unquote(gopts)
    113              else:
    114                  break
    115
    116          comment = urllib.unquote(comment) if comment is not None else ''
    117
    118          gopts = gopts.split('|')
    119          gopts = fixMissingRRDs(gopts)
    120          gopts.append('HRULE:INF#00000000')
    121          gopts.append('--width=%s' % width)
    122          if start:
    123              gopts.append('--start=%s' % start)
    124          if end:
    125              gopts.append('--end=%s' % end)
    126          drange = int(drange)
    127          id = self.graphId(gopts, drange, ftype)
    128          graph = self.getGraph(id, ftype, REQUEST)
    129          if not graph:
    130              if not os.path.exists(self.tmpdir):
    131                  os.makedirs(self.tmpdir, 0750)
    132              filename = "%s/graph-%s" % (self.tmpdir,id)
    133              if remoteUrl:
    134                  f = open(filename, "w")
    135                  f.write(urllib.urlopen(remoteUrl).read())
    136                  f.close()
    137              else:
    138                  if ftype.lower()=='html':
    139                      imgtype = 'PNG'
    140                  else:
    141                      imgtype = ftype
    142                  daemon = rrd_daemon_running()
    143                  if daemon:
    144                      gopts.insert(0, '--daemon=%s' % daemon)
    145                  gopts.insert(0, "--imgformat=%s" % imgtype)
    146                  #gopts.insert(0, "--lazy")
    147                  end = int(time.time())-300
    148                  start = end - drange
    149                  if comment is not None:
    150                      gopts.insert(0, 'COMMENT:%s\\c' % comment)
    151                  gopts.insert(0, '--end=%d' % end)
    152                  gopts.insert(0, '--start=%d' % start)
    153                  gopts.insert(0, filename)
    154                  log.debug("RRD graphing options: %r", (gopts,))
    155                  try:
    156                      rrdtool.graph(*gopts)
    157                  except Exception, ex:
    158                      if ex.args[0].find('No such file or directory') > -1:
    159                          return None
    160                      log.exception("Failed to generate a graph")
    161                      log.warn(" ".join(gopts))
    162                      return None
    163
    164              self.addGraph(id, filename)
    165              graph = self.getGraph(id, ftype, REQUEST)
    166
    167          if getImage:
    168              return graph
    169          else:
    170              success = bool(graph)
    171              ret = {'success':success}
    172              if success:
    173                  ret['data'] = b64encode(graph)
    174              if REQUEST:
    175                  REQUEST.RESPONSE.setHeader('Content-Type', 'text/javascript')
    176              elif zenrenderRequest:
    177                  zenrenderRequest.setHeader('Content-Type', 'text/javascript')
    178              return """Zenoss.SWOOP_CALLBACKS["%s"]('%s')""" % (graphid, json.dumps(ret))
    179
    180
    181 -    def deleteRRDFiles(self, device,
    182                          datasource=None, datapoint=None,
    183                          remoteUrl=None, REQUEST=None):
    184          """
    185          Delete RRD files associated with the given device id.
    186          If datapoint is not None then delete the file corresponding to that dp.
    187          Else if datasource is not None then delete the files corresponding to
    188            all datapoints in the datasource.
    189          Else delete all RRD files associated with the given device.
    190
    191          @param device: device name
    192          @param datasource: RRD datasource (DS) name
    193          @param datapoint: RRD datapoint name (lives in a DS)
    194          @param remoteUrl: if the RRD is not here, where it lives
    195          @param REQUEST: URL-marshalled object containg URL options
    196          """
    197          devDir = performancePath('/Devices/%s' % device)
    198          if not os.path.isdir(devDir):
    199              return
    200          fileNames = []
    201          dirNames = []
    202          if datapoint:
    203              fileNames = [
    204                  performancePath('/Devices/%s/%s.rrd' % (device, datapoint))]
    205          elif datasource:
    206              rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource)
    207              fileNames = glob.glob(performancePath(rrdPath))
    208          else:
    209              for dPath, dNames, dFiles in os.walk(devDir, topdown=False):
    210                  fileNames += [os.path.join(dPath, f) for f in dFiles]
    211                  dirNames += [os.path.join(dPath, d) for d in dNames]
    212              dirNames.append(devDir)
    213          for fileName in fileNames:
    214              try:
    215                  os.remove(fileName)
    216              except OSError:
    217                  log.warn("File %s does not exist" % fileName)
    218          for dirName in dirNames:
    219              try:
    220                  os.rmdir(dirName)
    221              except OSError:
    222                  log.warn('Directory %s could not be removed' % dirName)
    223          if remoteUrl:
    224              urllib.urlopen(remoteUrl)
    225
    226
    227 -    def packageRRDFiles(self, device, REQUEST=None):
    228          """
    229          Tar up RRD files into a nice, neat package
    230
    231          @param device: device name
    232          @param REQUEST: URL-marshalled object containg URL options
    233          """
    234          srcdir = performancePath('/Devices/%s' % device)
    235          tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
    236          log.debug( "tarring up %s into %s" % ( srcdir, tarfilename ))
    237          tar = tarfile.open(tarfilename, "w:gz")
    238          for file in os.listdir(srcdir):
    239              tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file))
    240          tar.close()
    241
    242 -    def unpackageRRDFiles(self, device, REQUEST=None):
    243          """
    244          Untar a package of RRDFiles
    245
    246          @param device: device name
    247          @param REQUEST: URL-marshalled object containg URL options
    248          """
    249          destdir = performancePath('/Devices/%s' % device)
    250          tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
    251          log.debug( "Untarring %s into %s" % ( tarfilename, destdir ))
    252          tar = tarfile.open(tarfilename, "r:gz")
    253          for file in tar.getmembers():
    254              tar.extract(file, destdir)
    255          tar.close()
    256
    257 -    def receiveRRDFiles(self, REQUEST=None):
    258          """
    259          Receive a device's RRD Files from another server
    260          This function is called by sendRRDFiles()
    261
    262          @param REQUEST: 'tarfile', 'tarfilename'
    263          @type REQUEST: URL-marshalled parameters
    264          """
    265          tarfile = REQUEST.get('tarfile')
    266          tarfilename = REQUEST.get('tarfilename')
    267          log.debug( "Receiving %s ..." % ( tarfilename ))
    268          f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb')
    269          f.write(urllib.unquote(tarfile))
    270          f.close()
    271
    272 -    def sendRRDFiles(self, device, server, REQUEST=None):
    273          """
    274          Move a package of RRDFiles
    275
    276          @param device: device name
    277          @param server: another RenderServer instance
    278          @param REQUEST: URL-marshalled object containg URL options
    279          """
    280          tarfilename = '%s.tgz' % device
    281          f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb')
    282          tarfilebody=f.read()
    283          f.close()
    284          # urlencode the id, title and file
    285          params = urllib.urlencode({'tarfilename': tarfilename,
    286              'tarfile':tarfilebody})
    287
    288          # send the file to Zope
    289          perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server)
    290          if perfMon.renderurl.startswith('http'):
    291              remoteUrl = '%s/receiveRRDFiles' % (perfMon.renderurl)
    292              log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl ))
    293              urllib.urlopen(remoteUrl, params)
    294
    295
    296 -    def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
    297          """
    298          Send a device's RRD files to another server
    299
    300          @param device: device name
    301          @param destServer: another RenderServer instance
    302          @param srcServer: another RenderServer instance
    303          @param REQUEST: URL-marshalled object containg URL options
    304          """
    305          monitors = self.dmd.getDmdRoot("Monitors")
    306          destPerfMon = monitors.getPerformanceMonitor(destServer)
    307          if srcServer:
    308              srcPerfMon = monitors.getPerformanceMonitor(srcServer)
    309              remoteUrl = '%s/moveRRDFiles?device=%s&destServer=%s' % (srcPerfMon.renderurl, device, destServer)
    310              urllib.urlopen(remoteUrl)
    311
    312          else:
    313              self.packageRRDFiles(device, REQUEST)
    314              self.sendRRDFiles(device, destServer, REQUEST)
    315              if destPerfMon.renderurl.startswith('http'):
    316                  remoteUrl = '%s/unpackageRRDFiles?device=%s' % (destPerfMon.renderurl, device)
    317                  urllib.urlopen(remoteUrl)
    318              else:
    319                  self.unpackageRRDFiles(device, REQUEST)
    320
    321      security.declareProtected('View', 'plugin')
    322 -    def plugin(self, name, REQUEST=None):
    323          """
    324          Render a custom graph and return it
    325
    326          @param name: plugin name from Products/ZenRRD/plugins
    327          @return: graph or None
    328          """
    329          try:
    330              m = zenPath('Products/ZenRRD/plugins/%s.py' % name)
    331              log.debug( "Trying plugin %s to generate a graph..." % m )
    332              graph = None
    333              exec open(m)
    334              return graph
    335          except Exception:
    336              log.exception("Failed generating graph from plugin %s" % name)
    337              raise
    338
    339
    340      security.declareProtected('GenSummary', 'summary')
    341 -    def summary(self, gopts):
    342          """
    343          Return summary information as a list but no graph
    344
    345          @param gopts: RRD graph options
    346          @return: values from the graph
    347          """
    348          gopts = fixMissingRRDs(gopts)
    349          gopts.insert(0, '/dev/null') #no graph generated
    350          daemon = rrd_daemon_running()
    351          if daemon:
    352              gopts.insert(0, '--daemon=%s' % daemon)
    353          try:
    354              values = rrdtool.graph(*gopts)[2]
    355          except Exception, ex:
    356              if ex.args[0].find('No such file or directory') > -1:
    357                  return None
    358              log.exception("Failed while generating summary")
    359              log.warn(" ".join(gopts))
    360              raise
    361          return values
    362
    363
    364      security.declareProtected('GenSummary', 'fetchValues')
    365 -    def fetchValues(self, paths, cf, resolution, start, end=""):
    366          """
    367          Return the values recorded in the RRD file between the start and end period
    368
    369          @param paths: path names to files
    370          @param cf: RRD consolidation function to use
    371          @param resolution: requested resolution of RRD data
    372          @param start: requested start of data to graph
    373          @param end: requested start of data to graph
    374          @return: values from the RRD files in the paths
    375          """
    376          if not end:
    377              end = "now"
    378          values = []
    379          daemon = rrd_daemon_running()
    380          args = ('--daemon', daemon) if daemon else ()
    381          try:
    382              for path in paths:
    383                  values.append(rrdtool.fetch(path, cf, "-r %d" % resolution,
    384                      "-s %s" % start,"-e %s" % end, *args))
    385              return values
    386          except NameError:
    387              log.exception("It appears that the rrdtool bindings are not installed properly.")
    388          except Exception, ex:
    389              if ex.args[0].find('No such file or directory') > -1:
    390                  return None
    391              log.exception("Failed while generating current values")
    392              raise
    393
    394
    395      security.declareProtected('GenSummary', 'currentValues')
    396 -    def currentValues(self, paths):
    397          """
    398          Return the latest values recorded in the RRD file
    399
    400          @param paths: path names to files
    401          @return: values from the RRD files in the path
    402          """
    403          try:
    404              def value(p):
    405                  v = None
    406                  info = None
    407                  try:
    408                      info = rrdtool.info(p)
    409                  except:
    410                      log.debug('%s not found' % p)
    411                  if info:
    412                      last = info['last_update']
    413                      step = info['step']
    414                      v = rrdtool.graph('/dev/null',
    415                                        'DEF:x=%s:ds0:AVERAGE' % p,
    416                                        'VDEF:v=x,LAST',
    417                                        'PRINT:v:%.2lf',
    418                                        '--start=%d'%(last-step),
    419                                        '--end=%d'%last)
    420                      v = float(v[2][0])
    421                      if str(v) == 'nan': v = None
    422                  return v
    423              return map(value, paths)
    424
    425          except NameError:
    426              log.exception("It appears that the rrdtool bindings are not installed properly.")
    427
    428          except Exception, ex:
    429              if ex.args[0].find('No such file or directory') > -1:
    430                  return None
    431              log.exception("Failed while generating current values")
    432              raise
    433
    434 -    def rrdcmd(self, gopts, ftype='PNG'):
    435          """
    436          Generate the RRD command using the graphing options specified.
    437
    438          @param gopts: RRD graphing options
    439          @param ftype: graphic file type (eg PNG)
    440          @return: RRD command usable on the command-line
    441          @rtype: string
    442          """
    443          filename, gopts = self._setfile(gopts, ftype)
    444          return "rrdtool graph " + " ".join(gopts)
    445
    446
    447 -    def graphId(self, gopts, drange, ftype):
    448          """
    449          Generate a graph id based on a hash of values
    450
    451          @param gopts: RRD graphing options
    452          @param drange: min/max values of the graph
    453          @param ftype: graphic file's type (eg PNG)
    454          @return: An id for this graph usable in URLs
    455          @rtype: string
    456          """
    457          id = md5.new(''.join(gopts)).hexdigest()
    458          id += str(drange) + '.' + ftype.lower()
    459          return id
    460
    461 -    def _loadfile(self, filename):
    462          try:
    463              f = open(filename)
    464              graph = f.read()
    465              f.close()
    466              return graph
    467          except IOError:
    468              log.info("File: %s not created yet." % filename);
    469              return None
    470
    471
    472 -    def setupCache(self):
    473          """
    474          Make a new cache if we need one
    475          """
    476          if not hasattr(self, '_v_cache') or not self._v_cache:
    477              tmpfolder = self.getPhysicalRoot().temp_folder
    478              if not hasattr(tmpfolder, self.cacheName):
    479                  cache = PObjectCache(self.cacheName, self.cachetimeout)
    480                  tmpfolder._setObject(self.cacheName, cache)
    481              self._v_cache = tmpfolder._getOb(self.cacheName)
    482          return self._v_cache
    483
    484
    485 -    def addGraph(self, id, filename):
    486          """
    487          Add a graph to temporary folder
    488
    489          @param id: graph id
    490          @param filename: cacheable graphic file
    491          """
    492          cache = self.setupCache()
    493          graph = self._loadfile(filename)
    494          if graph:
    495              cache.addToCache(id, graph)
    496              try:
    497                  os.remove(filename)
    498              except OSError, e:
    499                  if e.errno == 2:
    500                      log.debug("Unable to remove cached graph %s: %s" \
    501                          % (e.strerror, e.filename))
    502                  else:
    503                      raise e
    504          cache.cleanCache()
    505
    506
    507 -    def getGraph(self, id, ftype, REQUEST):
    508          """
    509          Get a previously generated graph
    510
    511          @param id: graph id
    512          @param ftype: file type of graphic (eg PNG)
    513          @param REQUEST: graph id
    514          """
    515          cache = self.setupCache()
    516          ftype = ftype.lower()
    517
    518          if REQUEST:
    519              mimetype = mimetypes.guess_type('%s.%s' % (id, ftype))[0]
    520              if mimetype is None:
    521                  mimetype = 'image/%s' % ftype
    522              response = REQUEST.RESPONSE
    523              response.setHeader('Content-Type', mimetype)
    524
    525          return cache.checkCache(id)
    526
    527
    528  InitializeClass(RenderServer)
    529