Package Products :: Package ZenUtils :: Module ZenCheckRRD |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenCheckRRD

      1  #!/usr/bin/env python
      2  ##############################################################################
      3  #
      4  # Copyright (C) Zenoss, Inc. 2010, all rights reserved.
      5  #
      6  # This content is made available according to terms specified in
      7  # License.zenoss under the directory where your Zenoss product is installed.
      8  #
      9  ##############################################################################
     10
     11
     12  import os
     13  import re
     14  import sys
     15  from pickle import dump, load
     16  from subprocess import Popen, PIPE
     17  import logging
     18  log = logging.getLogger('zen.checkRRD')
     19
     20  import Globals
     21  import transaction
     22  from Products.ZenUtils.Utils import zenPath
     23  from Products.ZenUtils.ZenScriptBase import ZenScriptBase
     24
     25
     26  CACHE_FILE = zenPath('var', 'zencheckrrd.cache')
     27  rrdMatch = re.compile('DEF:[^=]+=([^:]+)').match
     28
     29 -class collectorStats:
     30 -    def __init__(self, id, hostname):
     31          self.id = id
     32          self.hostname = hostname
     33          self.expectedComponents = 0
     34          self.stale = 0
     35          self.missing = 0
     36          self.orphan = 0
     37          self.expectedFiles = set()
     38          self.staleFiles = set()
     39          self.allFiles = set()
     40
     41 -class ZenCheckRRD(ZenScriptBase):
     42 -    def __init__(self):
     43          ZenScriptBase.__init__(self, connect=True)
     44
     45
     46 -    def buildOptions(self):
     47          ZenScriptBase.buildOptions(self)
     48          self.parser.add_option('--age', dest='age',
     49              type='int', default=1,
     50              help="Number of days old to consider fresh (default=1)")
     51          self.parser.add_option('--all', dest='all',
     52              action="store_true", default=False,
     53              help="Check all data points. Not just ones used in graphs")
     54          self.parser.add_option('--pathcache', dest='pathcache',
     55              action="store_true", default=False,
     56              help="Cache the full list of RRD file paths in the model")
     57          self.parser.add_option('--devicesonly', dest='devicesonly',
     58              action="store_true", default=False,
     59              help="Only check for device files. Not components")
     60          self.parser.add_option('--collector', dest='collector',
     61              help="Name of specific collector to check (optional)")
     62          self.parser.add_option('--file', dest='file',
     63              help="Output filename")
     64
     65
     66 -    def run(self):
     67          if not self.options.file:
     68              log.critical("You must specify the output file.")
     69              return
     70
     71          try:
     72              outfile = open(self.options.file, 'w')
     73          except IOError, ex:
     74              log.critical("Unable to open %s for writing: %s",
     75                  self.options.file, ex)
     76              return
     77
     78          if self.options.all:
     79              log.info("Starting check for missing, stale or orphaned RRD files")
     80              log.info("Results based on all RRD files defined by data points")
     81          else:
     82              log.info("Starting check for missing or stale RRD files")
     83              log.info("Results based on all RRD files used in graphs")
     84
     85          if self.options.collector:
     86              try:
     87                  collectors = [self.dmd.Monitors.Performance._getOb(
     88                      self.options.collector)]
     89              except AttributeError:
     90                  log.critical("No collector named %s could be found. Exiting",
     91                      self.options.collector)
     92                  return
     93          else:
     94              collectors = self.dmd.Monitors.Performance.objectValues(
     95                  spec="PerformanceConf")
     96
     97          collectors = [collectorStats(x.id, getattr(x, 'hostname', x.id)) \
     98                          for x in collectors]
     99          self._getExpectedFiles(collectors)
    100
    101          for collector in collectors:
    102              if len(collector.expectedFiles) == 0:
    103                  collector.expected = 0
    104                  log.debug("No expected files found for collector %s",
    105                      collector.id)
    106                  continue
    107
    108              self._getCollectorFiles(collector)
    109              if self.options.all:
    110                  for path in sorted(collector.allFiles - collector.expectedFiles):
    111                      outfile.write("orphaned:%s:%s\n" % (collector.id, path))
    112                      collector.orphan += 1
    113
    114              for path in sorted(collector.expectedFiles - collector.staleFiles):
    115                  if path in collector.allFiles:
    116                      outfile.write("stale:%s:%s\n" % (collector.id, path))
    117                      collector.stale += 1
    118                  else:
    119                      outfile.write("missing:%s:%s\n" % (collector.id, path))
    120                      collector.missing += 1
    121
    122          outfile.close()
    123          self.report(collectors)
    124
    125
    126 -    def report(self, collectors):
    127          totalExpectedRRDs = sum(len(x.expectedFiles) for x in collectors)
    128          totalAllRRDs = sum(len(x.allFiles) for x in collectors)
    129          totalMissingRRDs = sum(x.missing for x in collectors)
    130          totalStaleRRDs = sum(x.stale for x in collectors)
    131          #totalComponentRRDs = sum(x.expectedComponents for x in collectors)
    132          #totalDeviceRRDs = totalExpectedRRDs - totalComponentRRDs
    133          header = """
    134                                On-disk Expected   Missing    Stale
    135  Collector                        RRDs     RRDs      RRDs     RRDs"""
    136          delimLen = 65
    137          if self.options.all:
    138              header += " Orphans"
    139              delimLen = 75
    140          print header
    141          print '-' * delimLen
    142
    143          collectorNames = dict(zip(map(lambda x: x.id, collectors), collectors))
    144          for name in sorted(collectorNames.keys()):
    145              collector = collectorNames[name]
    146              expected = len(collector.expectedFiles)
    147              all = len(collector.allFiles)
    148              line = "%-30s %6s   %6s    %6s   %6s" % (
    149                     name, all, expected, collector.missing, collector.stale)
    150              if self.options.all:
    151                  line += " %6s" % collector.orphan
    152              print line
    153
    154          print '-' * delimLen
    155          trailer = "%-30s %6s   %6s    %6s   %6s" % (
    156                     'Total', totalAllRRDs, totalExpectedRRDs,
    157                     totalMissingRRDs, totalStaleRRDs)
    158          if self.options.all:
    159              trailer += " %6s" % sum(x.orphan for x in collectors)
    160          print trailer
    161
    162
    163 -    def _getExpectedFiles(self, collectors):
    164          rrdFiles = set()
    165          componentRrdFiles = set()
    166
    167          if self.options.pathcache and os.path.isfile(CACHE_FILE):
    168              log.debug("Reading list of expected RRD files from cache...")
    169              f = open('.rrdcheck.state', 'r')
    170              rrdFiles = load(f)
    171              f.close()
    172          else:
    173              log.debug("Building list of expected device RRD files..")
    174              for device in self.dmd.Devices.getSubDevicesGen():
    175                  if not device.monitorDevice(): continue
    176                  rrdFiles.update(self._getRRDPaths(device))
    177                  device._p_deactivate()
    178
    179              if not self.options.devicesonly:
    180                  log.debug("Building list of expected component RRD files..")
    181                  for component in self._getAllMonitoredComponents():
    182                      componentRrdFiles.update(self._getRRDPaths(component))
    183                      component._p_deactivate()
    184
    185              # Dump the cache in case we want to use it next time.
    186              f = open('.rrdcheck.state', 'w')
    187              dump(rrdFiles, f)
    188              f.close()
    189
    190          collectorNames = dict(zip(map(lambda x: x.id, collectors), collectors))
    191          for collectorName, path in rrdFiles:
    192              collector = collectorNames.get(collectorName, None)
    193              if collector:
    194                  collector.expectedFiles.add(path)
    195
    196          for collectorName, path in componentRrdFiles:
    197              collector = collectorNames.get(collectorName, None)
    198              if collector:
    199                  collector.expectedComponents += 1
    200                  collector.expectedFiles.add(path)
    201
    202
    203 -    def _getRRDPaths(self, ob):
    204          ob_rrds = set()
    205          path = ob.fullRRDPath()
    206          perfServer = ob.getPerformanceServer()
    207          if not perfServer: return []
    208          if self.options.all:
    209              for t in ob.getRRDTemplates():
    210                  for ds in t.datasources():
    211                      for dp in ds.datapoints():
    212                          ob_rrds.add((perfServer.id,
    213                              os.path.join(path, "%s_%s.rrd" % (ds.id, dp.id))))
    214          else:
    215              for t in ob.getRRDTemplates():
    216                  for g in t.graphDefs():
    217                      for cmd in g.getGraphCmds(ob, path):
    218                          match = rrdMatch(cmd)
    219                          if match:
    220                              ob_rrds.add((perfServer.id, match.group(1)))
    221          transaction.abort()
    222          return ob_rrds
    223
    224
    225 -    def _getAllMonitoredComponents(self):
    226          for component in self.dmd.Devices.getMonitoredComponents():
    227              if not component.snmpIgnore():
    228                  yield component
    229
    230
    231 -    def _getCollectorFiles(self, collector):
    232          def parseOutput(output):
    233              files = set()
    234              for line in ( l.strip() for l in output.split('\n') if l ):
    235                  files.add(line)
    236              return files
    237
    238          log.debug("Checking collector %s (%s) for RRD files",
    239              collector.id, collector.hostname)
    240
    241          allCmd = "find %s -name *.rrd" % zenPath('perf', 'Devices')
    242          staleCmd = "%s -mtime -%s" % (allCmd, self.options.age)
    243
    244          if collector.hostname == 'localhost':
    245              allOutput = Popen([allCmd],
    246                  shell=True, stdout=PIPE).communicate()[0]
    247              staleOutput = Popen([staleCmd],
    248                  shell=True, stdout=PIPE).communicate()[0]
    249          else:
    250              # Quick check to see if we can SSH to the collector.
    251              p1 = Popen(["echo", "0"], stdout=PIPE)
    252              p2 = Popen(["nc", "-w", "4", collector.hostname, "22"],
    253                  stdin=p1.stdout, stdout=PIPE, stderr=PIPE)
    254
    255              if os.waitpid(p2.pid, 0)[1] != 0:
    256                  log.warn("Unable to SSH to collector %s (%s)",
    257                           collector.id, collector.hostname)
    258                  return
    259
    260              allOutput = Popen(["ssh", collector.hostname, allCmd],
    261                  stdout=PIPE).communicate()[0]
    262              staleOutput = Popen(["ssh", collector.hostname, staleCmd],
    263                  stdout=PIPE).communicate()[0]
    264
    265          collector.allFiles = parseOutput(allOutput)
    266          collector.staleFiles = parseOutput(staleOutput)
    267
    268
    269  if __name__ == '__main__':
    270      zrc = ZenCheckRRD()
    271      zrc.run()
    272