Package Products :: Package ZenRRD :: Module RenderServer |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRRD.RenderServer

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2007, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  __doc__="""RenderServer
     12
     13  Frontend that passes RRD graph options to rrdtool to render,
     14  and then returns an URL to access the rendered graphic file.
     15  """
     16
     17  import os
     18  import time
     19  import logging
     20  import json
     21  import urllib
     22  import zlib
     23  import mimetypes
     24  import glob
     25  import tarfile
     26  import md5
     27  import tempfile
     28  from Products.ZenUtils import Map
     29  from AccessControl import ClassSecurityInfo
     30  from Globals import InitializeClass
     31  from Globals import DTMLFile
     32
     33  try:
     34      import rrdtool
     35  except ImportError:
     36      pass
     37
     38  from base64 import b64encode, urlsafe_b64decode, urlsafe_b64encode
     39  from urllib import urlencode
     40
     41  from Products.ZenRRD.RRDUtil import fixMissingRRDs
     42  from Products.ZenUtils.Utils import zenPath, rrd_daemon_args, rrd_daemon_retry
     43
     44  from RRDToolItem import RRDToolItem
     45
     46  from Products.ZenModel.PerformanceConf import performancePath
     47
     48  log = logging.getLogger("zen.RenderServer")
     49
     50
     51 -def manage_addRenderServer(context, id, REQUEST = None):
     52      """
     53      Make a RenderServer
     54      """
     55      rs = RenderServer(id)
     56      context._setObject(id, rs)
     57      if REQUEST is not None:
     58          REQUEST['RESPONSE'].redirect(context.absolute_url()+'/manage_main')
     59
     60
     61  addRenderServer = DTMLFile('dtml/addRenderServer',globals())
     62
     63  DEFAULT_TIMEOUT=300
     64  _cache = Map.Locked(Map.Timed({}, DEFAULT_TIMEOUT))
     65
     66
     67 -class RenderServer(RRDToolItem):
     68      """
     69      Base class for turning graph requests into graphics.
     70      NB: Any log messages get logged into the event.log file.
     71      """
     72
     73      meta_type = "RenderServer"
     74
     75      cacheName = 'RRDRenderCache'
     76
     77      security = ClassSecurityInfo()
     78
     79 -    def __init__(self, id, tmpdir = '/tmp/renderserver', cachetimeout=DEFAULT_TIMEOUT):
     80          self.id = id
     81          self.tmpdir = tmpdir
     82          self.cachetimeout = cachetimeout
     83
     84
     85      security.declareProtected('View', 'render')
     86 -    def render(self, gopts=None, start=None, end=None, drange=None,
     87                 remoteUrl=None, width=None, ftype='PNG', getImage=True,
     88                 graphid='', comment=None, ms=None, remoteHost=None, REQUEST=None, zenrenderRequest=None):
     89          """
     90          Render a graph and return it
     91
     92          @param gopts: RRD graph creation options
     93          @param start: requested start of data to graph
     94          @param end: requested start of data to graph
     95          @param drange: min/max values of the graph
     96          @param remoteUrl: if the RRD is not here, where it lives -DEPRECATED use remoteHost instead
     97          @param width: size of graphic to create
     98          @param ftype: file type of graphic (eg PNG)
     99          @param getImage: return the graph or a script location
    100          @param graphid: (hopefully) unique identifier of a graph
    101          @param comment: RRD graph comment
    102          @param ms: a timestamp used to force IE to reload images
    103          @param remoteHost: Forward current RRD request to renderserver at remoteHost. eg http://remotezenrender:8091/
    104          @param REQUEST: URL-marshalled object containg URL options
    105          @return: graph or script location
    106          """
    107          # gopts may have repeated url quoting, possibly from multiple hops thru remote zenhubs
    108          # extra quoting will create invalid zlib padding characters ('%3D' instead of '=')
    109          for tries in range(3):
    110              try:
    111                  gopts = zlib.decompress(urlsafe_b64decode(gopts))
    112              except Exception:
    113                  gopts = urllib.unquote(gopts)
    114              else:
    115                  break
    116
    117          comment = urllib.unquote(comment) if comment is not None else ''
    118
    119          gopts = gopts.split('|')
    120          gopts = fixMissingRRDs(gopts)
    121          gopts.append('HRULE:INF#00000000')
    122          gopts.append('--width=%s' % width)
    123          if start:
    124              gopts.append('--start=%s' % start)
    125          if end:
    126              gopts.append('--end=%s' % end)
    127          drange = int(drange)
    128          id = self.graphId(gopts, drange, ftype)
    129          graph = self.getGraph(id, ftype, REQUEST)
    130          if not graph:
    131              if not os.path.exists(self.tmpdir):
    132                  os.makedirs(self.tmpdir, 0750)
    133              fd, filename = tempfile.mkstemp(dir=self.tmpdir, suffix=id)
    134              if remoteHost or remoteUrl:
    135                  if remoteHost:
    136                      encodedOpts = urlsafe_b64encode(
    137                          zlib.compress('|'.join(gopts), 9))
    138                      params = {
    139                          'gopts': encodedOpts,
    140                          'drange': drange,
    141                          'width': width,
    142                      }
    143                      remote = "%s/render?%s" %(remoteHost, urlencode(params))
    144                  else:
    145                      remote = remoteUrl
    146                  f = open(filename, "w")
    147                  response = urllib.urlopen(remote).read()
    148                  f.write(response)
    149                  f.close()
    150              else:
    151                  if ftype.lower()=='html':
    152                      imgtype = 'PNG'
    153                  else:
    154                      imgtype = ftype
    155                  gopts.insert(0, "--imgformat=%s" % imgtype)
    156                  #gopts.insert(0, "--lazy")
    157                  end = int(time.time())-300
    158                  start = end - drange
    159                  if comment is not None:
    160                      gopts.insert(0, 'COMMENT:%s\\c' % comment)
    161                  gopts.insert(0, '--end=%d' % end)
    162                  gopts.insert(0, '--start=%d' % start)
    163                  gopts.insert(0, filename)
    164                  log.debug("RRD graphing options: %r", (gopts,))
    165                  try:
    166                      @rrd_daemon_retry
    167                      def rrdtool_fn():
    168                          rrdtool.graph(*(gopts + list(rrd_daemon_args())))
    169                      rrdtool_fn()
    170
    171                  except Exception, ex:
    172                      if ex.args[0].find('No such file or directory') > -1:
    173                          return None
    174                      log.exception("Failed to generate a graph")
    175                      log.warn(" ".join(gopts))
    176                      return None
    177
    178              self.addGraph(id, filename, fd)
    179              graph = self.getGraph(id, ftype, REQUEST)
    180
    181          if getImage:
    182              return graph
    183          else:
    184              success = bool(graph)
    185              ret = {'success':success}
    186              if success:
    187                  ret['data'] = b64encode(graph)
    188              if REQUEST:
    189                  REQUEST.RESPONSE.setHeader('Content-Type', 'text/javascript')
    190              elif zenrenderRequest:
    191                  zenrenderRequest.setHeader('Content-Type', 'text/javascript')
    192              return """Zenoss.SWOOP_CALLBACKS["%s"]('%s')""" % (graphid, json.dumps(ret))
    193
    194
    195 -    def deleteRRDFiles(self, device,
    196                          datasource=None, datapoint=None,
    197                          remoteUrl=None, REQUEST=None):
    198          """
    199          Delete RRD files associated with the given device id.
    200          If datapoint is not None then delete the file corresponding to that dp.
    201          Else if datasource is not None then delete the files corresponding to
    202            all datapoints in the datasource.
    203          Else delete all RRD files associated with the given device.
    204
    205          @param device: device name
    206          @param datasource: RRD datasource (DS) name
    207          @param datapoint: RRD datapoint name (lives in a DS)
    208          @param remoteUrl: if the RRD is not here, where it lives
    209          @param REQUEST: URL-marshalled object containg URL options
    210          """
    211          # If remoteUrl is specified, open/invoke that first because we
    212          # probably want to delete RRD files on some other machine.
    213          if remoteUrl:
    214              urllib.urlopen(remoteUrl)
    215          # Carry on with deleting local RRD files; however, if remoteUrl was
    216          # specified, then the devDir path (probably) doesn't exist on this
    217          # machine.
    218          devDir = performancePath('/Devices/%s' % device)
    219          if not os.path.isdir(devDir):
    220              return
    221          fileNames = []
    222          dirNames = []
    223          if datapoint:
    224              fileNames = [
    225                  performancePath('/Devices/%s/%s.rrd' % (device, datapoint))]
    226          elif datasource:
    227              rrdPath = '/Devices/%s/%s_*.rrd' % (device, datasource)
    228              fileNames = glob.glob(performancePath(rrdPath))
    229          else:
    230              for dPath, dNames, dFiles in os.walk(devDir, topdown=False):
    231                  fileNames += [os.path.join(dPath, f) for f in dFiles]
    232                  dirNames += [os.path.join(dPath, d) for d in dNames]
    233              dirNames.append(devDir)
    234          for fileName in fileNames:
    235              try:
    236                  os.remove(fileName)
    237              except OSError:
    238                  log.warn("File %s does not exist" % fileName)
    239          for dirName in dirNames:
    240              try:
    241                  os.rmdir(dirName)
    242              except OSError:
    243                  log.warn('Directory %s could not be removed' % dirName)
    244
    245
    246 -    def packageRRDFiles(self, device, REQUEST=None):
    247          """
    248          Tar up RRD files into a nice, neat package
    249
    250          @param device: device name
    251          @param REQUEST: URL-marshalled object containg URL options
    252          """
    253          srcdir = performancePath('/Devices/%s' % device)
    254          tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
    255          log.debug( "tarring up %s into %s" % ( srcdir, tarfilename ))
    256          tar = tarfile.open(tarfilename, "w:gz")
    257          for file in os.listdir(srcdir):
    258              tar.add('%s/%s' % (srcdir, file), '/%s' % os.path.basename(file))
    259          tar.close()
    260
    261 -    def unpackageRRDFiles(self, device, REQUEST=None):
    262          """
    263          Untar a package of RRDFiles
    264
    265          @param device: device name
    266          @param REQUEST: URL-marshalled object containg URL options
    267          """
    268          destdir = performancePath('/Devices/%s' % device)
    269          tarfilename = '%s/%s.tgz' % (self.tmpdir, device)
    270          log.debug( "Untarring %s into %s" % ( tarfilename, destdir ))
    271          tar = tarfile.open(tarfilename, "r:gz")
    272          for file in tar.getmembers():
    273              tar.extract(file, destdir)
    274          tar.close()
    275
    276 -    def receiveRRDFiles(self, REQUEST=None):
    277          """
    278          Receive a device's RRD Files from another server
    279          This function is called by sendRRDFiles()
    280
    281          @param REQUEST: 'tarfile', 'tarfilename'
    282          @type REQUEST: URL-marshalled parameters
    283          """
    284          tarfile = REQUEST.get('tarfile')
    285          tarfilename = REQUEST.get('tarfilename')
    286          log.debug( "Receiving %s ..." % ( tarfilename ))
    287          f=open('%s/%s' % (self.tmpdir, tarfilename), 'wb')
    288          f.write(urllib.unquote(tarfile))
    289          f.close()
    290
    291 -    def sendRRDFiles(self, device, server, REQUEST=None):
    292          """
    293          Move a package of RRDFiles
    294
    295          @param device: device name
    296          @param server: another RenderServer instance
    297          @param REQUEST: URL-marshalled object containg URL options
    298          """
    299          tarfilename = '%s.tgz' % device
    300          f=open('%s/%s' % (self.tmpdir, tarfilename), 'rb')
    301          tarfilebody=f.read()
    302          f.close()
    303          # urlencode the id, title and file
    304          params = urllib.urlencode({'tarfilename': tarfilename,
    305              'tarfile':tarfilebody})
    306
    307          # send the file to Zope
    308          perfMon = self.dmd.getDmdRoot("Monitors").getPerformanceMonitor(server)
    309          if perfMon.getRemoteRenderUrl().startswith('http'):
    310              remoteUrl = '%s/receiveRRDFiles' % (perfMon.getRemoteRenderUrl())
    311              log.debug( "Sending %s to %s ..." % ( tarfilename, remoteUrl ))
    312              urllib.urlopen(remoteUrl, params)
    313
    314
    315 -    def moveRRDFiles(self, device, destServer, srcServer=None, REQUEST=None):
    316          """
    317          Send a device's RRD files to another server
    318
    319          @param device: device name
    320          @param destServer: another RenderServer instance
    321          @param srcServer: another RenderServer instance
    322          @param REQUEST: URL-marshalled object containg URL options
    323          """
    324          monitors = self.dmd.getDmdRoot("Monitors")
    325          destPerfMon = monitors.getPerformanceMonitor(destServer)
    326          if srcServer:
    327              srcPerfMon = monitors.getPerformanceMonitor(srcServer)
    328              remoteUrl = '%s/moveRRDFiles?device=%s&destServer=%s' % (srcPerfMon.getRemoteRenderUrl(), device, destServer)
    329              urllib.urlopen(remoteUrl)
    330
    331          else:
    332              self.packageRRDFiles(device, REQUEST)
    333              self.sendRRDFiles(device, destServer, REQUEST)
    334              if destPerfMon.getRemoteRenderUrl().startswith('http'):
    335                  remoteUrl = '%s/unpackageRRDFiles?device=%s' % (destPerfMon.getRemoteRenderUrl(), device)
    336                  urllib.urlopen(remoteUrl)
    337              else:
    338                  self.unpackageRRDFiles(device, REQUEST)
    339
    340      security.declareProtected('View', 'plugin')
    341 -    def plugin(self, name, REQUEST=None):
    342          """
    343          Render a custom graph and return it
    344
    345          @param name: plugin name from Products/ZenRRD/plugins
    346          @return: graph or None
    347          """
    348          try:
    349              m = zenPath('Products/ZenRRD/plugins/%s.py' % name)
    350              log.debug( "Trying plugin %s to generate a graph..." % m )
    351              graph = None
    352              exec open(m)
    353              return graph
    354          except Exception:
    355              log.exception("Failed generating graph from plugin %s" % name)
    356              raise
    357
    358
    359      security.declareProtected('GenSummary', 'summary')
    360 -    def summary(self, gopts):
    361          """
    362          Return summary information as a list but no graph
    363
    364          @param gopts: RRD graph options
    365          @return: values from the graph
    366          """
    367          gopts = fixMissingRRDs(gopts)
    368          gopts.insert(0, '/dev/null') #no graph generated
    369          log.debug("RRD summary options: %r", (gopts,))
    370          try:
    371              @rrd_daemon_retry
    372              def rrdtool_fn():
    373                  return rrdtool.graph(*(gopts+list(rrd_daemon_args())))[2]
    374              values = rrdtool_fn()
    375          except Exception, ex:
    376              if ex.args[0].find('No such file or directory') > -1:
    377                  return None
    378              log.exception("Failed while generating summary")
    379              log.warn(" ".join(gopts))
    380              raise
    381          return values
    382
    383
    384      security.declareProtected('GenSummary', 'fetchValues')
    385 -    def fetchValues(self, paths, cf, resolution, start, end=""):
    386          """
    387          Return the values recorded in the RRD file between the start and end period
    388
    389          @param paths: path names to files
    390          @param cf: RRD consolidation function to use
    391          @param resolution: requested resolution of RRD data
    392          @param start: requested start of data to graph
    393          @param end: requested start of data to graph
    394          @return: values from the RRD files in the paths
    395          """
    396          if not end:
    397              end = "now"
    398          values = []
    399          try:
    400              for path in paths:
    401                  @rrd_daemon_retry
    402                  def rrdtool_fn():
    403                      try:
    404                          values.append(rrdtool.fetch(
    405                              path, cf, "-r %d" % resolution,
    406                              "-s %s" % start, "-e %s" % end,
    407                              *rrd_daemon_args()))
    408
    409                      except rrdtool.error, ex:
    410                          if 'No such file or directory' in ex.message:
    411                              values.append(None)
    412                          else:
    413                              raise
    414
    415                  rrdtool_fn()
    416              return values
    417          except NameError:
    418              log.exception("It appears that the rrdtool bindings are not installed properly.")
    419          except Exception, ex:
    420              log.exception("Failed while generating current values")
    421              raise
    422
    423
    424      security.declareProtected('GenSummary', 'currentValues')
    425 -    def currentValues(self, paths):
    426          """
    427          Return the latest values recorded in the RRD file
    428
    429          @param paths: path names to files
    430          @return: values from the RRD files in the path
    431          """
    432          try:
    433              def value(p):
    434                  v = None
    435                  info = None
    436                  try:
    437                      @rrd_daemon_retry
    438                      def rrdtool_fn():
    439                          return rrdtool.info(p, *rrd_daemon_args())
    440                      info = rrdtool_fn()
    441                  except Exception:
    442                      log.debug('%s not found' % p)
    443                  if info:
    444                      last = info['last_update']
    445                      step = info['step']
    446                      gopts = ['/dev/null',
    447                               'DEF:x=%s:ds0:AVERAGE' % p,
    448                               'VDEF:v=x,LAST',
    449                               'PRINT:v:%.2lf',
    450                               '--start=%d'%(last-step),
    451                               '--end=%d'%last]
    452                      log.debug("RRD currentValue options: %r", (gopts,))
    453                      @rrd_daemon_retry
    454                      def rrdtool_fn():
    455                          return rrdtool.graph(*(gopts+list(rrd_daemon_args())))
    456                      v = rrdtool_fn()
    457                      v = float(v[2][0])
    458                      if str(v) == 'nan': v = None
    459                  return v
    460              return map(value, paths)
    461
    462          except NameError:
    463              log.exception("It appears that the rrdtool bindings are not installed properly.")
    464
    465          except Exception, ex:
    466              if ex.args[0].find('No such file or directory') > -1:
    467                  return None
    468              log.exception("Failed while generating current values")
    469              raise
    470
    471 -    def rrdcmd(self, gopts, ftype='PNG'):
    472          """
    473          Generate the RRD command using the graphing options specified.
    474
    475          @param gopts: RRD graphing options
    476          @param ftype: graphic file type (eg PNG)
    477          @return: RRD command usable on the command-line
    478          @rtype: string
    479          """
    480          filename, gopts = self._setfile(gopts, ftype)
    481          return "rrdtool graph " + " ".join(gopts)
    482
    483
    484 -    def graphId(self, gopts, drange, ftype):
    485          """
    486          Generate a graph id based on a hash of values
    487
    488          @param gopts: RRD graphing options
    489          @param drange: min/max values of the graph
    490          @param ftype: graphic file's type (eg PNG)
    491          @return: An id for this graph usable in URLs
    492          @rtype: string
    493          """
    494          id = md5.new(''.join(gopts)).hexdigest()
    495          id += str(drange) + '.' + ftype.lower()
    496          return id
    497
    498 -    def _loadfile(self, filename):
    499          try:
    500              f = open(filename)
    501              graph = f.read()
    502              f.close()
    503              return graph
    504          except IOError:
    505              log.info("File: %s not created yet." % filename);
    506              return None
    507
    508
    509 -    def setupCache(self):
    510          """
    511          Make a new cache if we need one
    512          """
    513          if not hasattr(self, '_v_cache') or not self._v_cache:
    514              self._v_cache = _cache
    515              self._v_cache.map.timeout = self.cachetimeout
    516          return self._v_cache
    517
    518
    519 -    def addGraph(self, id, filename, fd):
    520          """
    521          Add a graph to temporary folder
    522
    523          @param id: graph id
    524          @param filename: cacheable graphic file
    525          """
    526          cache = self.setupCache()
    527          graph = self._loadfile(filename)
    528          if graph:
    529              cache[id] = graph
    530              try:
    531                  os.close(fd)
    532                  os.remove(filename)
    533              except OSError, e:
    534                  if e.errno == 2:
    535                      log.debug("Unable to remove cached graph %s: %s" \
    536                          % (e.strerror, e.filename))
    537                  else:
    538                      raise e
    539
    540 -    def getGraph(self, id, ftype, REQUEST):
    541          """
    542          Get a previously generated graph
    543
    544          @param id: graph id
    545          @param ftype: file type of graphic (eg PNG)
    546          @param REQUEST: graph id
    547          """
    548          cache = self.setupCache()
    549          ftype = ftype.lower()
    550
    551          if REQUEST:
    552              mimetype = mimetypes.guess_type('%s.%s' % (id, ftype))[0]
    553              if mimetype is None:
    554                  mimetype = 'image/%s' % ftype
    555              response = REQUEST.RESPONSE
    556              response.setHeader('Content-Type', mimetype)
    557
    558          return cache.get(id, None)
    559
    560  InitializeClass(RenderServer)
    561