Package Products :: Package ZenModel :: Module BatchDeviceDumper |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.BatchDeviceDumper

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2011, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  __doc__ = """zenbatchdump
     12
     13  zenbatchdump dumps a list of devices to a file.
     14  """
     15
     16  import sys
     17  import re
     18  from datetime import datetime
     19  import platform
     20  from collections import defaultdict
     21
     22  import Globals
     23
     24  from Products.ZenUtils.ZCmdBase import ZCmdBase
     25  from Products.ZenModel.DeviceClass import DeviceClass
     26  from Products.ZenModel.Device import Device
     27  from Products.ZenModel.DeviceOrganizer import DeviceOrganizer
     28
     29
     30 -class BatchDeviceDumper(ZCmdBase):
     31      """
     32      Base class
     33      """
     34
     35      sample_configs = """#
     36  # zenbatchdump run on host zenoss41 on date 2011-10-16 16:34:23.569920
     37  # with --root=Devices/Server/Linux
     38  # To load this Device dump file, use:
     39  #   zenbatchload <file>
     40
     41
     42  '/Locations'
     43  '/Locations/TestZenBatchDumper'
     44  '/Locations/TestZenBatchDumper/City1'
     45  '/Locations/TestZenBatchDumper/City1/Building1'
     46  '/Locations/TestZenBatchDumper/City2'
     47
     48
     49  '/Systems'
     50  '/Systems/TestZenBatchDumper'
     51  '/Systems/TestZenBatchDumper/System1'
     52  '/Systems/TestZenBatchDumper/Scary/System2'
     53  '/Systems/TestZenBatchDumper/Scary/System3'
     54
     55
     56  '/Groups'
     57  '/Groups/TestZenBatchDumper'
     58  '/Groups/TestZenBatchDumper/Production'
     59  '/Groups/TestZenBatchDumper/Production/Critical'
     60  '/Groups/TestZenBatchDumper/Production/Secondary'
     61  '/Groups/TestZenBatchDumper/TEST'
     62  '/Groups/TestZenBatchDumper/DEV'
     63
     64  '/Devices/TestZenBatchDumper'  zCollectorPlugins=['zenoss.snmp.NewDeviceMap', 'zenoss.snmp.DeviceMap', 'HPDeviceMap', 'DellDeviceMap', 'zenoss.snmp.InterfaceMap', 'zenoss.snmp.RouteMap', 'zenoss.snmp.IpServiceMap', 'zenoss.snmp.HRFileSystemMap', 'zenoss.snmp.HRSWInstalledMap', 'zenoss.snmp.HRSWRunMap', 'zenoss.snmp.CpuMap', 'HPCPUMap', 'DellCPUMap', 'DellPCIMap'], zIcon='/zport/dmd/img/icons/server.png'
     65
     66  '/Devices/TestZenBatchDumper/Server'  zCollectorPlugins=['zenoss.snmp.NewDeviceMap', 'zenoss.snmp.DeviceMap', 'HPDeviceMap', 'DellDeviceMap', 'zenoss.snmp.InterfaceMap', 'zenoss.snmp.RouteMap', 'zenoss.snmp.IpServiceMap', 'zenoss.snmp.HRFileSystemMap', 'zenoss.snmp.HRSWInstalledMap', 'zenoss.snmp.HRSWRunMap', 'zenoss.snmp.CpuMap', 'HPCPUMap', 'DellCPUMap', 'DellPCIMap'], zIcon='/zport/dmd/img/icons/server.png'
     67
     68  '/Devices/TestZenBatchDumper/Server/Linux'  zCollectorPlugins=['zenoss.snmp.NewDeviceMap', 'zenoss.snmp.DeviceMap', 'HPDeviceMap', 'DellDeviceMap', 'zenoss.snmp.InterfaceMap', 'zenoss.snmp.RouteMap', 'zenoss.snmp.IpServiceMap', 'zenoss.snmp.HRFileSystemMap', 'zenoss.snmp.HRSWRunMap', 'zenoss.snmp.CpuMap', 'HPCPUMap', 'DellCPUMap', 'DellPCIMap'], zHardDiskMapMatch='^[hs]d[a-z]\\d+$|c\\d+t\\d+d\\d+s\\d+$|^cciss\\/c\\dd\\dp\\d$|^dm\\-\\d$', zIcon='/zport/dmd/img/icons/server-linux.png', zIpServiceMapMaxPort=8090
     69
     70  'localhost' setHWProductKey=('.1.3.6.1.4.1.8072.3.2.10', 'net snmp'), setHWSerialNumber='', setHWTag='', setLastChange=DateTime('2011/10/16 09:07:53.208444 GMT-7'), setManageIp='127.0.0.1', setOSProductKey='Linux 2.6.18-164.el5', setPriority=3, setProdState=1000, setPerformanceMonitor='localhost'
     71
     72  'thor' setLocation='/Locations/TestZenBatchDumper/City1', setSystems=['/Systems/TestZenBatchDumper/Scary/System2', '/Systems/TestZenBatchDumper/System1'], setGroups=['/Groups/TestZenBatchDumper/TEST', '/Groups/TestZenBatchDumper/DEV'], setHWProductKey=('.1.3.6.1.4.1.8072.3.2.10', 'net snmp'), setHWSerialNumber='', setHWTag='', setLastChange=DateTime('2011/10/16 09:22:59.450108 GMT-7'), setManageIp='192.168.55.225', setOSProductKey='Linux 2.6.32-25-server', setPriority=4, setProdState=1000, setPerformanceMonitor='localhost'
     73
     74  'loki' setLocation='/Locations/TestZenBatchDumper/City2', setSystems=['/Systems/TestZenBatchDumper/System1'], setGroups=['/Groups/TestZenBatchDumper/Production/Critical'], setHWProductKey=('.1.3.6.1.4.1.8072.3.2.10', 'net snmp'), setHWSerialNumber='', setHWTag='', setLastChange=DateTime('2011/10/16 09:19:59.450108 GMT-7'), setManageIp='192.168.55.223', setOSProductKey='Linux 2.6.32-25-server', setPriority=4, setProdState=1000, setPerformanceMonitor='localhost'
     75
     76  '/Devices/TestZenBatchDumper/Server/Windows'  zCollectorPlugins=['zenoss.snmp.NewDeviceMap', 'zenoss.snmp.DeviceMap', 'HPDeviceMap', 'DellDeviceMap', 'zenoss.snmp.InterfaceMap', 'zenoss.snmp.RouteMap', 'zenoss.snmp.IpServiceMap', 'zenoss.snmp.HRFileSystemMap', 'zenoss.snmp.HRSWInstalledMap', 'zenoss.snmp.HRSWRunMap', 'zenoss.snmp.CpuMap', 'HPCPUMap', 'DellCPUMap', 'DellPCIMap', 'zenoss.snmp.InformantHardDiskMap', 'zenoss.wmi.WinServiceMap'], zHardDiskMapMatch='.*', zIcon='/zport/dmd/img/icons/server-windows.png', zWinEventlog=True, zWmiMonitorIgnore=False
     77
     78  '/Devices/TestZenBatchDumper/Server/Windows/WMI'  zCollectorPlugins=['zenoss.wmi.WindowsDeviceMap', 'zenoss.wmi.WinServiceMap', 'zenoss.wmi.CpuMap', 'zenoss.wmi.FileSystemMap', 'zenoss.wmi.IpInterfaceMap', 'zenoss.wmi.IpRouteMap', 'zenoss.wmi.MemoryMap', 'zenoss.wmi.ProcessMap', 'zenoss.wmi.SoftwareMap'], zDeviceTemplates=['Device_WMI'], zWinPerfCycleSeconds=300, zWinPerfCyclesPerConnection=10
     79
     80  '192.168.5.219' zWinPassword='easyPassword', zWinUser='admin', setLocation='/Locations/TestZenBatchDumper/City1/Building1', setGroups=['/Groups/TestZenBatchDumper/Production/Secondary'], setHWProductKey=('Unknown', 'Chassis Manufacture'), setHWSerialNumber='Chassis Serial Number', setHWTag='Asset-1234567890', setLastChange=DateTime('2011/10/16 09:19:56.920976 GMT-7'), setManageIp='192.168.55.229', setOSProductKey=('Windows 7 Home Premium ', 'Microsoft'), setPriority=3, setProdState=1000, setPerformanceMonitor='localhost'
     81
     82
     83  # Dumped:
     84  #            Locations: 5
     85  #               Groups: 7
     86  #              Systems: 5
     87  #        DeviceClasses: 5
     88  #              Devices: 4
     89  """
     90
     91 -    def __init__(self, *args, **kwargs):
     92          ZCmdBase.__init__(self, *args, **kwargs)
     93          self.defaults = {}
     94          self.emittedDeviceClasses = set()
     95
     96 -    def _prepRoot(self):
     97          """
     98          initializes and verify the device root and prune options properly
     99
    100          @return: was initialization successful
    101          @rtype: bool
    102          """
    103
    104          if self.options.root == "":
    105              self.root = self.dmd.Devices
    106              self.options.prune = False
    107          else:
    108              try:
    109                  self.root = self.dmd.unrestrictedTraverse(self.options.root)
    110              except KeyError:
    111                  self.log.error("%s is not a valid DeviceOrganizer path under %s\n" % (self.options.root, self.dmd.getPrimaryUrlPath()))
    112                  return False
    113
    114          self.rootPath = self.root.getPrimaryUrlPath()
    115          return True
    116
    117 -    def _emitProps(self, obj):
    118          """
    119          Returns string of object local zProperties, cProperties and "setter" properties suitable for ZenBatchLoader
    120
    121          @parameter obj: a Device or DeviceClass (or perhaps Location later)
    122          @type obj: ZenModelRM
    123          @return: string containing local zProperties as documented in above sample
    124          @rtype str
    125          """
    126          props = []
    127          # description has neither setter nor getter so we special-case it here
    128          desc = getattr(obj,"description","")
    129          if desc:
    130              props.append("%s=%s" % ('description',repr(desc)))
    131          for prop in ((x['id'], repr(x['value'])) for x in obj.exportZProperties() if x['islocal']):
    132              key = prop[0]
    133              if obj.zenPropIsPassword(key):
    134                  val = repr(getattr(obj, key, ''))
    135                  prop = (key, val)
    136              props.append("%s=%s" % prop)
    137
    138          for cProp in obj.custPropertyMap():
    139              if cProp['id'] == 'cDateTest': continue
    140              value = getattr(obj,cProp['id'],'')
    141              if value and value != '':
    142                  props.append("%s=%s" % (cProp['id'], repr(value)))
    143
    144          # Export out setter method data
    145          ignoreSetters = ('setLastPollSnmpUpTime', 'setSnmpLastCollection', 'setSiteManager', 'setLocation', 'setGroups', 'setSystems')
    146          for setMethod in [setter for setter in dir(obj) if setter.startswith('set')]:
    147              if setMethod in ignoreSetters:
    148                  continue
    149              getMethod = setMethod.replace('set', 'get', 1)
    150              getter = getattr(obj, getMethod, None)
    151              if getter and callable(getter):
    152                  # deal with braindamaged get/setProdState
    153                  if setMethod == 'setProdState':
    154                      states = obj.getProdStateConversions()
    155                      for state in states:
    156                          if getter() in state:
    157                              value = state[1]
    158                  else:
    159                      value = getter()
    160                  if value and value != '':
    161                      props.append("%s=%s" % (setMethod, repr(value)))
    162              else:
    163                  # for setters that have no getter, try a bare attribute
    164                  value = getattr(obj, setMethod[3:].lower(), None)
    165                  if value and value != '':
    166                      props.append("%s=%s" % (setMethod, repr(value)))
    167
    168          # There's always got to be a weirdie....
    169          if ('getPerformanceServerName' in dir(obj)):
    170              props.append("setPerformanceMonitor=" + repr(obj.getPerformanceServerName()))
    171          return props
    172
    173 -    def _emitDev(self, dev):
    174          """
    175          Returns a device and its zProperties in strings appropriate for ZenBatchLoader
    176
    177          @parameter dev: Device object to emit
    178          @type dev: Device
    179          @return: strings containing device name and list of Device-local zProperties and cProperties
    180          @rtype: tuple of strings
    181          """
    182
    183          result = self._emitProps(dev)
    184          location = dev.location()
    185          if location:
    186              result.append("setLocation=" + repr("/".join(location.getPrimaryPath()[4:])))
    187
    188          systems = dev.systems()
    189          if systems:
    190              result.append("setSystems=" + repr(["/"+"/".join(system.getPrimaryPath()[4:]) for system in systems]))
    191
    192          groups = dev.groups()
    193          if groups:
    194              result.append("setGroups=" + repr(["/"+"/".join(grp.getPrimaryPath()[4:]) for grp in groups]))
    195
    196          return (repr(dev.getId()),result)
    197
    198 -    def _emitOrg(self, org):
    199          """
    200          Returns a device organizer with its type and local properties
    201
    202          @parameter org: DeviceOrganizer to emit
    203          @type org: DeviceOrganizer
    204          @return: strings containing the device organizer name, type and properties
    205          @rtype: tuple of strings
    206          """
    207
    208          path = org.getPrimaryPath()
    209          name = "'/%s' " % "/".join(path[3:])
    210          # Avoid things that override base classes for the moment (eg uses zPythonClass)
    211          props = [] if isinstance(org, DeviceClass) and 'ZenPacks' in org.zPythonClass else self._emitProps(org)
    212
    213          if '/Locations/' in path:
    214              props.append('setAddress="%s"' % org.address)
    215
    216          return (name, props)
    217
    218 -    def _backtraceOrg(self, outFile, obj):
    219          """
    220          Recurse upward from a device emitting parent DeviceClasses if not already emitted
    221
    222          @parameter outFile: file object to which output is written
    223          @type outFile: file or other object with .write() method that is simillar
    224          @parameter dev: Device/DeviceClass for whom we emit parent Organizer paths
    225          @type dev: Device or DeviceClass
    226          @return: number of DeviceClasses emitted
    227          @rtype: int
    228          """
    229
    230          result = 0
    231          if isinstance(obj, Device):
    232              # back out to first containing DeviceOrganizer
    233              obj = obj.getPrimaryParent().getPrimaryParent()
    234          if not obj in self.emittedDeviceClasses:
    235              parent = obj.getPrimaryParent()
    236              # don't recurse to dmd
    237              if parent.getPrimaryPath()[2:] != self.dmd.getPrimaryPath()[2:]:
    238                  result = self._backtraceOrg(outFile, parent)
    239              (name, props) = self._emitOrg(obj);
    240              outFile.write("\n%s %s\n" % (name, ", ".join(props)))
    241              self.emittedDeviceClasses.add(obj)
    242              result += 1
    243          return result
    244
    245 -    def listLSGOTree(self, outFile, branch):
    246          """
    247          Recurse through the Locations, Systems and Groups trees printing out Organizers with properties
    248          return number of Devices emitted
    249
    250          @parameter outFile: output object to which we write output
    251          @type outFile: file or other object with .write() method that is simillar
    252          @parameter branch: object reference to current tree branch (only used for recursion, should be called with None)
    253          @type branch: DeviceOrganizer
    254          @return: number of Locations, Systems or Groups dumped
    255          @rtype: int
    256          """
    257          if getattr(self.options,'rootPath',None) is None:
    258              # BatchDeviceDumper.run() already calls ._prepRoot() before we get here, but leaving this in
    259              # for unit tests and unexpected uses
    260              if not self._prepRoot():
    261                  return -1;
    262
    263          result = 0
    264
    265          if not isinstance(branch, DeviceOrganizer):
    266              raise TypeError("listLSGOTree must start in a DeviceOrganizer not (%s)") % branch
    267
    268          # Hidden option for pruning LSG Organizers as pruned ones may get referenced by unpruned devices
    269          # This is to be used by unit tests to simplify output
    270          if getattr(self.options, 'pruneLSGO', None) and not isinstance(self.root, DeviceClass) and \
    271                  not (branch.getPrimaryUrlPath().startswith(self.rootPath) or \
    272                       self.root.getPrimaryUrlPath().startswith(branch.getPrimaryUrlPath())):
    273              return result
    274
    275          outFile.write("\n")
    276          (name, props) = self._emitOrg(branch)
    277          result += 1
    278          outFile.write("\n%s %s\n" % (name, ", ".join(props)))
    279
    280          for org in branch.children():
    281             result += self.listLSGOTree(outFile, org)
    282          return result
    283
    284 -    def makeRegexMatcher(self):
    285          regex = re.compile(self.options.regex)
    286          return lambda dev: dev is not None and regex.match(dev.id)
    287
    288 -    def chooseDevice(self, root, matcher=None):
    289          for dev in root.getDevices():
    290              # can likely remove this next line, as I only call this from within the dmd.Devices tree
    291              # was here for when I could be traversing the LSGOrganizers and finding devices there
    292              dev = self.dmd.unrestrictedTraverse(dev.getPrimaryPath())
    293              if not dev in self.root.getSubDevices():
    294                  continue
    295              if 'ZenPack' in dev.zPythonClass:
    296                  continue
    297              if matcher:
    298                  if matcher(dev):
    299                      yield dev
    300              else:
    301                  yield dev
    302
    303 -    def listDeviceTree(self, outFile, branch=None):
    304          """
    305          Recurse through the Devices tree printing out Organizers and Devices with properties
    306          return number of Devices emitted
    307
    308          @parameter outFile: output object to which we write output
    309          @type outFile: file or other object with .write() method that is simillar
    310          @parameter branch: object reference to current tree branch (only used for recursion, should be called with None)
    311          @type branch: DeviceClass (or perhaps DeviceOrganizer at worst)
    312          @return: number of leaf Devices and DeviceClasses dumped
    313          @rtype: dict
    314          """
    315          if getattr(self.options,'rootPath',None) is None:
    316              # BatchDeviceDumper.run() already calls ._prepRoot() before we get here, but leaving this in
    317              # for unit tests and unexpected uses
    318              if not self._prepRoot():
    319                  return { 'fail' : True }
    320
    321          if branch is None:
    322              branch = self.dmd.Devices
    323
    324          if not isinstance(branch, DeviceClass):
    325              raise TypeError("listDeviceTree must start in a DeviceClass not " + repr(branch))
    326
    327          result = defaultdict(int)
    328
    329          # Dump DeviceClass if not pruned
    330          if not self.options.prune or branch.getPrimaryUrlPath() in self.rootPath:
    331              outFile.write("\n")
    332              (name, props) = self._emitOrg(branch)
    333              result['DeviceClasses'] += 1
    334              outFile.write("\n%s %s\n" % (name, ", ".join(props)))
    335              self.emittedDeviceClasses.add(branch)
    336
    337          # Dump all elligible Devices in this DeviceClass (pruning occurs in .chooseDevice())
    338          for dev in self.chooseDevice(branch,self.makeRegexMatcher()):
    339              (name,props) = self._emitDev(dev)
    340              # enusre that if we've pruned Organizers above this Device that we emit them first
    341              result['DeviceClasses'] += self._backtraceOrg(outFile, dev)
    342              outFile.write("\n%s %s\n" % (name, ", ".join(props)))
    343              result['Devices'] += 1
    344
    345          # Recurse on down the Tree
    346          for org in branch.children():
    347             found = self.listDeviceTree(outFile, org)
    348             result['Devices'] += found['Devices']
    349             result['DeviceClasses'] += found['DeviceClasses']
    350          return result
    351
    352 -    def buildOptions(self):
    353          """
    354          Add our command-line options to the basics
    355          """
    356          ZCmdBase.buildOptions(self)
    357
    358          self.parser.add_option('--root',
    359               dest = "root", default = "",
    360               help = "Set the root Device Path to dump (eg: /Devices/Servers or /Devices/Network/Cisco/Nexus; default: /Devices)")
    361
    362          self.parser.add_option('--outFile',
    363               dest = 'outFile', default = sys.__stdout__,
    364               help = "Specify file to which zenbatchdump will write output")
    365
    366          self.parser.add_option('--regex',
    367               dest = 'regex', default = '.*',
    368               help = "Specify include filter for device objects")
    369
    370          self.parser.add_option('--prune',
    371               dest = 'prune', default = False,
    372               action = 'store_true',
    373               help = "Should DeviceClasses only be dumped if part of root path")
    374
    375 -    def run(self):
    376          """
    377          Run the batch device dump
    378          """
    379          # make sure we have somewhere to write output
    380          if isinstance(self.options.outFile, str):
    381              try:
    382                  outFile = open(self.options.outFile, "w")
    383              except IOError as e:
    384                  self.log.error("Cannot open file %s for writing: %s" % (self.options.outFile,e))
    385                  sys.exit(1)
    386          else:
    387              outFile = self.options.outFile
    388              self.options.outFile = outFile.name
    389
    390          # ensure we have a valid root
    391          if self.options.root:
    392              if self.options.root[0] == '/':
    393                  self.options.root = self.options.root[1:]
    394              if not self._prepRoot():
    395                  outFile.close()
    396                  sys.exit(2)
    397
    398          self.printHeader(outFile)
    399          foundLSGO = {}
    400          foundLSGO['Locations'] = self.listLSGOTree(outFile, self.dmd.Locations)
    401          foundLSGO['Systems'] = self.listLSGOTree(outFile, self.dmd.Systems)
    402          foundLSGO['Groups'] = self.listLSGOTree(outFile, self.dmd.Groups)
    403          foundDevices = self.listDeviceTree(outFile)
    404          self.printTrailer(outFile, foundLSGO, foundDevices)
    405          outFile.close()
    406
    407 -    def printHeader(self, outFile):
    408          curDate = datetime.now()
    409          hostname = platform.node()
    410          outFile.write("# zenbatchdump run on host %s on date %s\n" % (hostname,str(curDate)))
    411          outFile.write("# with --root=%s\n" % self.options.root)
    412          outFile.write("# To load this Device dump file, use:\n")
    413          outFile.write("#   zenbatchload <file>\n")
    414
    415 -    def printTrailer(self, outFile, foundLSGO, foundDevices):
    416          outFile.write("\n# Dumped:\n")
    417          for type in foundLSGO:
    418              outFile.write("#        %13s: %d\n" % (type, foundLSGO[type]))
    419          for type in foundDevices:
    420              outFile.write("#        %13s: %d\n" % (type, foundDevices[type]))
    421
    422
    423  if __name__=='__main__':
    424      batchDumper = BatchDeviceDumper()
    425      batchDumper.run()
    426