Package Products :: Package ZenRRD :: Module RenderServer |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRRD.RenderServer

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2007, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 as published by
      8  # the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13
     14  __doc__="""RenderServer
     15
     16  Frontend that passes RRD graph options to rrdtool to render,
     17  and then returns an URL to access the rendered graphic file.
     18  """
     19
     20  import os
     21  import time
     22  import logging
     23  import urllib
     24  import zlib
     25  import mimetypes
     26
     27  from AccessControl import ClassSecurityInfo
     28  from Globals import InitializeClass
     29  from Globals import DTMLFile
     30  from sets import Set
     31
     32  try:
     33      import rrdtool
     34  except ImportError:
     35      pass
     36
     37  try:
     38      from base64 import urlsafe_b64decode
     39      raise ImportError
     40  except ImportError:
     41 -    def urlsafe_b64decode(s):
     42          import base64
     43          return base64.decodestring(s.replace('-','+').replace('_','/'))
     44
     45  from Products.ZenUtils.PObjectCache import PObjectCache
     46  from Products.ZenUtils.Utils import zenPath
     47
     48  from RRDToolItem import RRDToolItem
     49
     50  from Products.ZenModel.PerformanceConf import performancePath
     51  import glob
     52  import tarfile
     53
     54  log = logging.getLogger("RenderServer")
     55
     56
     57 -def manage_addRenderServer(context, id, REQUEST = None):
     58      """
     59      Make a RenderServer
     60      """
     61      rs = RenderServer(id)
     62      context._setObject(id, rs)
     63      if REQUEST is not None:
     64          REQUEST['RESPONSE'].redirect(context.absolute_url()+'/manage_main')
     65
     66
     67  addRenderServer = DTMLFile('dtml/addRenderServer',globals())
     68
     69
     70 -class RenderServer(RRDToolItem):
     71      """
     72      Base class for turning graph requests into graphics.
     73      NB: Any log messages get logged into the event.log file.
     74      """
     75
     76      meta_type = "RenderServer"
     77
     78      cacheName = 'RRDRenderCache'
     79
     80      security = ClassSecurityInfo()
     81
     82 -    def __init__(self, id, tmpdir = '/tmp/renderserver', cachetimeout=300):
     83          self.id = id
     84          self.tmpdir = tmpdir
     85          self.cachetimeout = cachetimeout
     86
     87
     88 -    def removeInvalidRRDReferences(self, cmds):
     89          """
     90          Check the cmds list for DEF commands.  For each one check that the rrd
     91          file specified actually exists.  Return a list of commands which
     92          excludes commands referencing or depending on non-existent RRD files.
     93
     94          @param cmds: list of RRD commands
     95          @return: sanitized list of RRD commands
     96          """
     97          newCmds = []
     98          badNames = Set()
     99          for cmd in cmds:
    100              if cmd.startswith('DEF:'):
    101                  # Check for existence of the RRD file
    102                  vName, rrdFile = cmd.split(':')[1].split('=', 1)
    103                  if not os.path.isfile(rrdFile):
    104                      badNames.add(vName)
    105                      parts = rrdFile.split('/')
    106                      try:
    107                          devIndex = parts.index('Devices') + 1
    108                      except ValueError:
    109                          devIndex = -1
    110                      devName = devIndex > 0 and parts[devIndex] or ''
    111                      compIndex = len(parts) - 2
    112                      compName = compIndex > devIndex and parts[compIndex] or ''
    113                      dpName = parts[-1].rsplit('.', 1)[0]
    114                      desc = ' '.join([p for p in (devName,compName,dpName) if p])
    115                      newCmds.append('COMMENT:MISSING RRD FILE\: %s' % desc)
    116                      continue
    117
    118              elif cmd.startswith('VDEF:') or cmd.startswith('CDEF:'):
    119                  vName, expression = cmd.split(':', 1)[1].split('=', 1)
    120                  if Set(expression.split(',')) & badNames:
    121                      badNames.add(vName)
    122                      continue
    123
    124              elif not cmd.startswith('COMMENT'):
    125                  try:
    126                      vName = cmd.split(':')[1].split('#')[0]
    127                  except IndexError:
    128                      vName = None
    129                  if vName in badNames:
    130                      continue
    131              newCmds.append(cmd)
    132          return newCmds
    133
    134
    135      security.declareProtected('View', 'render')
    136 -    def render(self, gopts=None, start=None, end=None, drange=None,
    137                 remoteUrl=None, width=None, ftype='PNG', getImage=True,
    138                 graphid='', comment=None, ms=None, REQUEST=None):
    139          """
    140          Render a graph and return it
    141
    142          @param gopts: RRD graph creation options
    143          @param start: requested start of data to graph
    144          @param end: requested start of data to graph
    145          @param drange: min/max values of the graph
    146          @param remoteUrl: if the RRD is not here, where it lives
    147          @param width: size of graphic to create
    148          @param ftype: file type of graphic (eg PNG)
    149          @param getImage: return the graph or a script location
    150          @param graphid: (hopefully) unique identifier of a graph
    151          @param comment: RRD graph comment
    152          @param ms: a timestamp used to force IE to reload images
    153          @param REQUEST: URL-marshalled object containg URL options
    154          @return: graph or script location
    155          """
    156          gopts = zlib.decompress(urlsafe_b64decode(gopts))
    157          gopts = gopts.split('|')
    158          gopts = self.removeInvalidRRDReferences(gopts)
    159          gopts.append('--width=%s' % width)
    160          if start:
    161              gopts.append('--start=%s' % start)
    162          if end:
    163              gopts.append('--end=%s' % end)
    164          drange = int(drange)
    165          id = self.graphId(gopts, drange, ftype)
    166          graph = self.getGraph(id, ftype, REQUEST)
    167          if not graph:
    168              if not os.path.exists(self.tmpdir):
    169                  os.makedirs(self.tmpdir, 0750)
    170              filename = "%s/graph-%s" % (self.tmpdir,id)
    171              if remoteUrl:
    172                  f = open(filename, "w")
    173                  f.write(urllib.urlopen(remoteUrl).read())
    174                  f.close()
    175              else:
    176                  if ftype.lower()=='html':
    177                      imgtype = 'PNG'
    178                  else:
    179                      imgtype = ftype
    180                  gopts.insert(0, "--imgformat=%s" % imgtype)
    181                  #gopts.insert(0, "--lazy")
    182                  end = int(time.time())-300
    183                  start = end - drange
    184                  gopts.insert(0, 'COMMENT:%s\\c' % comment)
    185                  gopts.insert(0, '--end=%d' % end)
    186                  gopts.insert(0, '--start=%d' % start)
    187                  gopts.insert(0, filename)
    188                  log.debug("RRD graphing options: %r", (gopts,))
    189                  try:
    190                      rrdtool.graph(*gopts)
    191                  except Exception, ex:
    192                      if ex.args[0].find('No such file or directory') > -1:
    193                          return None
    194                      log.exception("Failed to generate a graph")
    195                      log.warn(" ".join(gopts))
    196                      return None
    197
    198              self.addGraph(id, filename)
    199              graph = self.getGraph(id, ftype, REQUEST)
    200
    201          if getImage:
    202              return graph
    203          else:
    204              return """
    205              <script>
    206                  parent.location.hash = '%s:%s;';
    207              </script>
    208              """ % (graphid, str(bool(graph)))
    209
    210
    211 -    def deleteRRDFiles(self, device,
    212                          datasource=None, datapoint=None,
    213                          remoteUrl=None, REQUEST=None):
    214          """
    215          Delete RRD files associated with the given device id.
    216          If datapoint is not None then delete the file corresponding to that dp.
    217          Else if datasource is not None then delete the files corresponding to
    218            all datapoints in the datasource.
    219          Else delete all RRD files associated with the given device.
    220
    221          @param device: device name
    222          @param datasource: RRD datasource (DS) name
    223          @param datapoint: RRD datapoint name (lives in a DS)
    224          @param remoteUrl: if the RRD is not here, where it lives
    225          @param REQUEST: URL-marshalled object containg URL options
    226          """
    227          devDir = performancePath('/Devices/%s' % device)
    228          if not os.path.isdir(devDir):
    229              return
    230          fileNames = []
    231          dirNames = []
    232          if datapoint:
    233              fileNames = [
    234                  performancePath('/Devices/%s/%s.rrd' % (device, datapoint))]
    235          elif datasource:
    236              rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource)
    237              fileNames = glob.glob(performancePath(rrdPath))
    238          else:
    239              for dPath, dNames, dFiles in os.walk(devDir, topdown=False):
    240                  fileNames += [os.path.join(dPath, f) for f in dFiles]
    241                  dirNames += [os.path.join(dPath, d) for d in dNames]
    242              dirNames.append(devDir)
    243          for fileName in fileNames:
    244              try:
    245                  os.remove(fileName)
    246              except OSError:
    247                  log.warn("File %s does not exist" % fileName)
    248          for dirName in dirNames:
    249              try:
    250                  os.rmdir(dirName)
    251              except OSError:
    252                  log.warn('Directory %s could not be removed' % dirName)
    253          if remoteUrl:
    254              urllib.urlopen(remoteUrl)
    255
    256
    257 -    def packageRRDFiles(self, device, REQUEST=None):
    258          """
    259          Tar up RRD files into a nice, neat package
    260
    261          @param device: device name
    262          @param REQUEST: URL-marshalled object containg URL options
    263          """
    264          srcdir = performancePath('/Devices/%s' % device)
    265          tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
    266          log.debug( "tarring up %s into %s" % ( srcdir, tarfilename ))
    267          tar = tarfile.open(tarfilename, "w:gz")
    268          for file in os.listdir(srcdir):
    269              tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file))
    270          tar.close()
    271
    272 -    def unpackageRRDFiles(self, device, REQUEST=None):
    273          """
    274          Untar a package of RRDFiles
    275
    276          @param device: device name
    277          @param REQUEST: URL-marshalled object containg URL options
    278          """
    279          destdir = performancePath('/Devices/%s' % device)
    280          tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
    281          log.debug( "Untarring %s into %s" % ( tarfilename, destdir ))
    282          tar = tarfile.open(tarfilename, "r:gz")
    283          for file in tar.getmembers():
    284              tar.extract(file, destdir)
    285          tar.close()
    286
    287 -    def receiveRRDFiles(self, REQUEST=None):
    288          """
    289          Receive a device's RRD Files from another server
    290          This function is called by sendRRDFiles()
    291
    292          @param REQUEST: 'tarfile', 'tarfilename'
    293          @type REQUEST: URL-marshalled parameters
    294          """
    295          tarfile = REQUEST.get('tarfile')
    296          tarfilename = REQUEST.get('tarfilename')
    297          log.debug( "Receiving %s ..." % ( tarfilename ))
    298          f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb')
    299          f.write(urllib.unquote(tarfile))
    300          f.close()
    301
    302 -    def sendRRDFiles(self, device, server, REQUEST=None):
    303          """
    304          Move a package of RRDFiles
    305
    306          @param device: device name
    307          @param server: another RenderServer instance
    308          @param REQUEST: URL-marshalled object containg URL options
    309          """
    310          tarfilename = '%s.tgz' % device
    311          f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb')
    312          tarfilebody=f.read()
    313          f.close()
    314          # urlencode the id, title and file
    315          params = urllib.urlencode({'tarfilename': tarfilename,
    316              'tarfile':tarfilebody})
    317
    318          # send the file to Zope
    319          perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server)
    320          if perfMon.renderurl.startswith('http'):
    321              remoteUrl = '%s/receiveRRDFiles' % (perfMon.renderurl)
    322              log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl ))
    323              urllib.urlopen(remoteUrl, params)
    324
    325
    326 -    def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
    327          """
    328          Send a device's RRD files to another server
    329
    330          @param device: device name
    331          @param destServer: another RenderServer instance
    332          @param srcServer: another RenderServer instance
    333          @param REQUEST: URL-marshalled object containg URL options
    334          """
    335          monitors = self.dmd.getDmdRoot("Monitors")
    336          destPerfMon = monitors.getPerformanceMonitor(destServer)
    337          if srcServer:
    338              srcPerfMon = monitors.getPerformanceMonitor(srcServer)
    339              remoteUrl = '%s/moveRRDFiles?device=%s&destServer=%s' % (srcPerfMon.renderurl, device, destServer)
    340              urllib.urlopen(remoteUrl)
    341
    342          else:
    343              self.packageRRDFiles(device, REQUEST)
    344              self.sendRRDFiles(device, destServer, REQUEST)
    345              if destPerfMon.renderurl.startswith('http'):
    346                  remoteUrl = '%s/unpackageRRDFiles?device=%s' % (destPerfMon.renderurl, device)
    347                  urllib.urlopen(remoteUrl)
    348              else:
    349                  self.unpackageRRDFiles(device, REQUEST)
    350
    351      security.declareProtected('View', 'plugin')
    352 -    def plugin(self, name, REQUEST=None):
    353          """
    354          Render a custom graph and return it
    355
    356          @param name: plugin name from Products/ZenRRD/plugins
    357          @return: graph or None
    358          """
    359          try:
    360              m = zenPath('Products/ZenRRD/plugins/%s.py' % name)
    361              log.debug( "Trying plugin %s to generate a graph..." % m )
    362              graph = None
    363              exec open(m)
    364              return graph
    365          except Exception, ex:
    366              log.exception("Failed generating graph from plugin %s" % name)
    367              raise
    368
    369
    370      security.declareProtected('GenSummary', 'summary')
    371 -    def summary(self, gopts):
    372          """
    373          Return summary information as a list but no graph
    374
    375          @param gopts: RRD graph options
    376          @return: values from the graph
    377          """
    378          gopts.insert(0, '/dev/null') #no graph generated
    379          try:
    380              values = rrdtool.graph(*gopts)[2]
    381          except Exception, ex:
    382              if ex.args[0].find('No such file or directory') > -1:
    383                  return None
    384              log.exception("Failed while generating summary")
    385              log.warn(" ".join(gopts))
    386              raise
    387          return values
    388
    389
    390      security.declareProtected('GenSummary', 'fetchValues')
    391 -    def fetchValues(self, paths, cf, resolution, start, end=""):
    392          if not end:
    393              end = "now"
    394          values = []
    395          try:
    396              for path in paths:
    397                  values.append(rrdtool.fetch(path, cf, "-r %d" % resolution,
    398                      "-s %s" % start,"-e %s" % end))
    399              return values
    400          except NameError:
    401              log.exception("It appears that the rrdtool bindings are not installed properly.")
    402          except Exception, ex:
    403              if ex.args[0].find('No such file or directory') > -1:
    404                  return None
    405              log.exception("Failed while generating current values")
    406              raise
    407
    408
    409      security.declareProtected('GenSummary', 'currentValues')
    410 -    def currentValues(self, paths):
    411          """
    412          Return the latest values recorded in the RRD file
    413          """
    414          try:
    415              def value(p):
    416                  v = None
    417                  info = None
    418                  try:
    419                      info = rrdtool.info(p)
    420                  except:
    421                      log.debug('%s not found' % p)
    422                  if info:
    423                      last = info['last_update']
    424                      step = info['step']
    425                      v = rrdtool.graph('/dev/null',
    426                                        'DEF:x=%s:ds0:AVERAGE' % p,
    427                                        'VDEF:v=x,LAST',
    428                                        'PRINT:v:%.2lf',
    429                                        '--start=%d'%(last-step),
    430                                        '--end=%d'%last)
    431                      v = float(v[2][0])
    432                      if str(v) == 'nan': v = None
    433                  return v
    434              return map(value, paths)
    435
    436          except NameError:
    437              log.exception("It appears that the rrdtool bindings are not installed properly.")
    438
    439          except Exception, ex:
    440              if ex.args[0].find('No such file or directory') > -1:
    441                  return None
    442              log.exception("Failed while generating current values")
    443              raise
    444
    445
    446 -    def rrdcmd(self, gopts, ftype='PNG'):
    447          """
    448          Generate the RRD command using the graphing options specified.
    449
    450          @param gopts: RRD graphing options
    451          @param ftype: graphic file type (eg PNG)
    452          @return: RRD command usable on the command-line
    453          @rtype: string
    454          """
    455          filename, gopts = self._setfile(gopts, ftype)
    456          return "rrdtool graph " + " ".join(gopts)
    457
    458
    459 -    def graphId(self, gopts, drange, ftype):
    460          """
    461          Generate a graph id based on a hash of values
    462
    463          @param gopts: RRD graphing options
    464          @param drange: min/max values of the graph
    465          @param ftype: graphic file's type (eg PNG)
    466          @return: An id for this graph usable in URLs
    467          @rtype: string
    468          """
    469          import md5
    470          id = md5.new(''.join(gopts)).hexdigest()
    471          id += str(drange) + '.' + ftype.lower()
    472          return id
    473
    474 -    def _loadfile(self, filename):
    475          try:
    476              f = open(filename)
    477              graph = f.read()
    478              f.close()
    479              return graph
    480          except IOError:
    481              log.info("File: %s not created yet." % filename);
    482              return None
    483
    484
    485 -    def setupCache(self):
    486          """
    487          Make a new cache if we need one
    488          """
    489          if not hasattr(self, '_v_cache') or not self._v_cache:
    490              tmpfolder = self.getPhysicalRoot().temp_folder
    491              if not hasattr(tmpfolder, self.cacheName):
    492                  cache = PObjectCache(self.cacheName, self.cachetimeout)
    493                  tmpfolder._setObject(self.cacheName, cache)
    494              self._v_cache = tmpfolder._getOb(self.cacheName)
    495          return self._v_cache
    496
    497
    498 -    def addGraph(self, id, filename):
    499          """
    500          Add a graph to temporary folder
    501
    502          @param id: graph id
    503          @param filename: cacheable graphic file
    504          """
    505          cache = self.setupCache()
    506          graph = self._loadfile(filename)
    507          if graph:
    508              cache.addToCache(id, graph)
    509              try:
    510                  os.remove(filename)
    511              except OSError, e:
    512                  if e.errno == 2:
    513                      log.debug("Unable to remove cached graph %s: %s" \
    514                          % (e.strerror, e.filename))
    515                  else:
    516                      raise e
    517          cache.cleanCache()
    518
    519
    520 -    def getGraph(self, id, ftype, REQUEST):
    521          """
    522          Get a previously generated graph
    523
    524          @param id: graph id
    525          @param ftype: file type of graphic (eg PNG)
    526          @param REQUEST: graph id
    527          """
    528          cache = self.setupCache()
    529          ftype = ftype.lower()
    530
    531          if REQUEST:
    532              mimetype = mimetypes.guess_type('.%s'%ftype)[0]
    533              if not mimetype:
    534                  mimetype = 'image/%s' % ftype
    535              response = REQUEST.RESPONSE
    536              response.setHeader('Content-Type', mimetype)
    537
    538          return cache.checkCache(id)
    539
    540
    541  InitializeClass(RenderServer)
    542