Package Products :: Package ZenRRD :: Module RRDUtil |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRRD.RRDUtil

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2007, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  __doc__ = """RRDUtil
     12
     13  Wrapper routines around the rrdtool library.
     14  """
     15
     16  import logging
     17  log = logging.getLogger("zen.RRDUtil")
     18
     19  import os
     20  import re
     21  import rrdtool
     22  import string
     23
     24  from Products.ZenUtils.Utils import zenPath, rrd_daemon_args, rrd_daemon_retry
     25
     26
     27  EMPTY_RRD = zenPath('perf', 'empty.rrd')
     28
     29  _UNWANTED_CHARS = ''.join(
     30          set(string.punctuation + string.ascii_letters) - set(['.', '-', '+'])
     31      )
     32  _LAST_RRDFILE_WRITE = {}
     33
     34
     35 -def _checkUndefined(x):
     36      """
     37      Sanity check on the min, max values
     38
     39      @param x: RRD min or max value
     40      @type x: number
     41      @return: Either the number or 'U' (for undefined)
     42      @rtype: number or string
     43      """
     44      if x is None or x == '' or x == -1 or x == '-1':
     45          return 'U'
     46      return x
     47
     48
     49 -def convertToRRDTime(val):
     50      """
     51      Convert any value that is passed in to a string that is acceptable to use
     52      for RRDtool's start and end parameters. Raises ValueError if this is not
     53      possible.
     54
     55      See the AT-STYLE TIME SPECIFICATION and TIME REFERENCE SPECIFICATION
     56      sections of the following document.
     57
     58          http://oss.oetiker.ch/rrdtool/doc/rrdfetch.en.html
     59
     60      Note: Currently this method is only fixing floats by turning them into
     61            strings with no decimal places.
     62      """
     63      # Integers are ok. This will also strip decimal precision.
     64      try:
     65          result = int(val)
     66          return str(result)
     67      except ValueError:
     68          pass
     69
     70      return str(val)
     71
     72
     73 -def fixMissingRRDs(gopts):
     74      """
     75      Parses a list of RRDtool gopts for DEFs. Runs all of the filenames
     76      referenced by those DEFs through the fixRRDFilename method to make sure
     77      that they will exist and not cause the rendering to fail.
     78      """
     79      fixed_gopts = []
     80
     81      def_match = re.compile(r'^DEF:([^=]+)=([^:]+)').match
     82      for gopt in gopts:
     83          match = def_match(gopt)
     84          if not match:
     85              fixed_gopts.append(gopt)
     86              continue
     87
     88          rrd_filename = match.group(2)
     89          fixed_gopts.append(gopt.replace(
     90              rrd_filename, fixRRDFilename(rrd_filename)))
     91
     92      return fixed_gopts
     93
     94
     95 -def fixRRDFilename(filename):
     96      """
     97      Attempting to render a graph containing a DEF referencing a non-existent
     98      filename will cause the entire graph to fail to render. This method is a
     99      helper to verify existence of an RRD file. If the file doesn't exist, a
    100      placeholder RRD filename with no values in it will be returned instead.
    101      """
    102      if os.path.isfile(filename):
    103          return filename
    104
    105      if not os.path.isfile(EMPTY_RRD):
    106          rrdtool.create(EMPTY_RRD, "--step", '300', 'DS:ds0:GAUGE:900:U:U',
    107              'RRA:AVERAGE:0.5:1:1', 'RRA:MAX:0.5:1:1', 'RRA:LAST:0.5:1:1')
    108
    109      return EMPTY_RRD
    110
    111 -def read(path, consolidationFunction, start, end):
    112      try:
    113          @rrd_daemon_retry
    114          def rrdtool_fn():
    115              return rrdtool.fetch(path, consolidationFunction, start, end, *rrd_daemon_args())
    116          return rrdtool_fn()
    117      except rrdtool.error, err:
    118          import sys
    119          err_str = '%s: %s' % (err.__class__.__name__, err)
    120          msg = 'Failed to read RRD file %s. %s' % (path, err_str)
    121          raise StandardError(msg), None, sys.exc_info()[2]
    122
    123 -class RRDUtil(object):
    124      """
    125      Wrapper class around rrdtool
    126      """
    127
    128 -    def __init__(self, defaultRrdCreateCommand, defaultCycleTime):
    129          """
    130          Initializer
    131
    132          The RRD creation command is only used if the RRD file doesn't
    133          exist and no rrdCommand was specified with the save() method.
    134
    135          @param defaultRrdCreateCommand: RRD creation command
    136          @type defaultRrdCreateCommand: string
    137          @param defaultCycleTime: expected time to periodically collect data
    138          @type defaultCycleTime: integer
    139          """
    140          self.defaultRrdCreateCommand = defaultRrdCreateCommand
    141          self.defaultCycleTime = defaultCycleTime
    142          self.dataPoints = 0
    143          self.cycleDataPoints = 0
    144
    145
    146 -    def endCycle(self):
    147          """
    148          Report on the number of data points collected in a cycle,
    149          and reset the counter for a new cycle.
    150
    151          @return: number of data points collected during the cycle
    152          @rtype: number
    153          """
    154          result = self.cycleDataPoints
    155          self.cycleDataPoints = 0
    156          return result
    157
    158
    159 -    def performancePath(self, path):
    160          """
    161          Given a path, return its location from $ZENHOME and the
    162          perf/ directories.
    163
    164          @param path: name for a datapoint in a path (eg device/component/datasource_datapoint)
    165          @type path: string
    166          @return: absolute path
    167          @rtype: string
    168          """
    169          from Products.ZenModel.PerformanceConf import performancePath
    170          return performancePath(path)
    171
    172
    173 -    def getStep(self, cycleTime):
    174          """
    175          Return the step value for the provided cycleTime. This is a hook for
    176          altering the default step calculation.
    177          """
    178          return int(cycleTime)
    179
    180
    181 -    def getHeartbeat(self, cycleTime):
    182          """
    183          Return the heartbeat value for the provided cycleTime. This is a hook
    184          for altering the default heartbeat calculation.
    185          """
    186          return int(cycleTime) * 3
    187
    188
    189 -    def put(self, path, value, rrdType, rrdCommand=None, cycleTime=None,
    190               min='U', max='U', useRRDDaemon=True, timestamp='N', start=None,
    191               allowStaleDatapoint=True):
    192          """
    193          Save the value provided in the command to the RRD file specified in path.
    194
    195          If the RRD file does not exist, use the rrdType, rrdCommand, min and
    196          max parameters to create the file.
    197
    198          @param path: name for a datapoint in a path (eg device/component/datasource_datapoint)
    199          @type path: string
    200          @param value: value to store into the RRD file
    201          @type value: number
    202          @param rrdType: RRD data type (eg ABSOLUTE, DERIVE, COUNTER)
    203          @type rrdType: string
    204          @param rrdCommand: RRD file creation command
    205          @type rrdCommand: string
    206          @param cycleTime: length of a cycle
    207          @type cycleTime: number
    208          @param min: minimum value acceptable for this metric
    209          @type min: number
    210          @param max: maximum value acceptable for this metric
    211          @type max: number
    212          @param allowStaleDatapoint: attempt to write datapoint even if a newer datapoint has already been written
    213          @type allowStaleDatapoint: boolean
    214          @return: the parameter value converted to a number
    215          @rtype: number or None
    216          """
    217          if value is None: return None
    218
    219          self.dataPoints += 1
    220          self.cycleDataPoints += 1
    221
    222          if cycleTime is None:
    223              cycleTime = self.defaultCycleTime
    224
    225          filename = self.performancePath(path) + '.rrd'
    226          if not rrdCommand:
    227              rrdCommand = self.defaultRrdCreateCommand
    228          if not os.path.exists(filename):
    229              log.debug("Creating new RRD file %s", filename)
    230              dirname = os.path.dirname(filename)
    231              if not os.path.exists(dirname):
    232                  os.makedirs(dirname, 0750)
    233
    234              min, max = map(_checkUndefined, (min, max))
    235              dataSource = 'DS:%s:%s:%d:%s:%s' % (
    236                  'ds0', rrdType, self.getHeartbeat(cycleTime), min, max)
    237              args = [str(filename), "--step",
    238                  str(self.getStep(cycleTime)),]
    239              if start is not None:
    240                  args.extend(["--start", "%d" % start])
    241              elif timestamp != 'N':
    242                  args.extend(["--start", "%d" % int(timestamp) - 10])
    243
    244              args.append(str(dataSource))
    245              args.extend(rrdCommand.split())
    246              rrdtool.create(*args),
    247
    248          daemon_args = rrd_daemon_args() if useRRDDaemon else tuple()
    249
    250          # remove unwanted chars (this is actually pretty quick)
    251          value = str(value).translate(None, _UNWANTED_CHARS)
    252
    253          if rrdType in ('COUNTER', 'DERIVE'):
    254              try:
    255                  # cast to float first because long('100.0') will fail with a
    256                  # ValueError
    257                  value = long(float(value))
    258              except (TypeError, ValueError):
    259                  return None
    260          else:
    261              try:
    262                  value = float(value)
    263              except (TypeError, ValueError):
    264                  return None
    265          try:
    266              @rrd_daemon_retry
    267              def rrdtool_fn():
    268                  return rrdtool.update(str(filename), *(daemon_args + ('%s:%s' % (timestamp, value),)))
    269              if timestamp == 'N' or allowStaleDatapoint:
    270                  rrdtool_fn()
    271              else:
    272                  # try to detect when the last datasample was collected
    273                  lastTs = _LAST_RRDFILE_WRITE.get(filename, None)
    274                  if lastTs is None:
    275                      try:
    276                          lastTs = _LAST_RRDFILE_WRITE[filename] = rrdtool.last(
    277                              *(daemon_args + (str(filename),)))
    278                      except Exception as ex:
    279                           lastTs = 0
    280                           log.exception("Could not determine last update to %r", filename)
    281                  # if the current datapoint is newer than the last datapoint, then write
    282                  if lastTs < timestamp:
    283                      _LAST_RRDFILE_WRITE[filename] = timestamp
    284                      if log.getEffectiveLevel() < logging.DEBUG:
    285                          log.debug('%s: %r, currentTs = %s, lastTs = %s', filename, value, timestamp, lastTs)
    286                      rrdtool_fn()
    287                  else:
    288                      if log.getEffectiveLevel() < logging.DEBUG:
    289                          log.debug("ignoring write %s:%s", filename, timestamp)
    290                      return None
    291
    292              log.debug('%s: %r, @ %s', str(filename), value, timestamp)
    293          except rrdtool.error, err:
    294              # may get update errors when updating too quickly
    295              log.error('rrdtool reported error %s %s', err, path)
    296
    297          return value
    298
    299
    300 -    def save(self, path, value, rrdType, rrdCommand=None, cycleTime=None,
    301               min='U', max='U', useRRDDaemon=True, timestamp='N', start=None,
    302               allowStaleDatapoint=True):
    303          """
    304          Save the value provided in the command to the RRD file specified in path.
    305          Afterward, fetch the latest value for this point and return it.
    306
    307          If the RRD file does not exist, use the rrdType, rrdCommand, min and
    308          max parameters to create the file.
    309
    310          @param path: name for a datapoint in a path (eg device/component/datasource_datapoint)
    311          @type path: string
    312          @param value: value to store into the RRD file
    313          @type value: number
    314          @param rrdType: RRD data type (eg ABSOLUTE, DERIVE, COUNTER)
    315          @type rrdType: string
    316          @param rrdCommand: RRD file creation command
    317          @type rrdCommand: string
    318          @param cycleTime: length of a cycle
    319          @type cycleTime: number
    320          @param min: minimum value acceptable for this metric
    321          @type min: number
    322          @param max: maximum value acceptable for this metric
    323          @type max: number
    324          @param allowStaleDatapoint: attempt to write datapoint even if a newer datapoint has already been written
    325          @type allowStaleDatapoint: boolean
    326          @return: the parameter value converted to a number
    327          @rtype: number or None
    328          """
    329          value = self.put(path, value, rrdType, rrdCommand, cycleTime, min, max, useRRDDaemon, timestamp, start, allowStaleDatapoint)
    330
    331          if value is None:
    332              return None
    333
    334          if rrdType in ('COUNTER', 'DERIVE'):
    335              filename = self.performancePath(path) + '.rrd'
    336              if cycleTime is None:
    337                  cycleTime = self.defaultCycleTime
    338
    339              @rrd_daemon_retry
    340              def rrdtool_fn():
    341                  daemon_args = rrd_daemon_args() if useRRDDaemon else tuple()
    342                  return rrdtool.fetch(filename, 'AVERAGE',
    343                                      '-s', 'now-%d' % (cycleTime*2),
    344                                      '-e', 'now', *daemon_args)
    345              startStop, names, values = rrdtool_fn()
    346
    347              values = [ v[0] for v in values if v[0] is not None ]
    348              if values: value = values[-1]
    349              else: value = None
    350          return value
    351