Package Products :: Package ZenModel :: Module BatchDeviceLoader |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.BatchDeviceLoader

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2009, 2011, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  __doc__ = """zenbatchload
     12
     13  zenbatchload loads a list of devices read from a file.
     14  """
     15
     16  import sys
     17  import re
     18  from traceback import format_exc
     19  import socket
     20
     21  import Globals
     22  from DateTime import DateTime
     23  from ZODB.POSException import ConflictError
     24  from ZODB.transact import transact
     25  from zope.component import getUtility
     26
     27  from zExceptions import BadRequest
     28
     29  from ZPublisher.Converters import type_converters
     30  from Products.ZenModel.interfaces import IDeviceLoader
     31  from Products.ZenUtils.ZCmdBase import ZCmdBase
     32  from Products.ZenModel.Device import Device
     33  from Products.ZenRelations.ZenPropertyManager import iszprop
     34  from Products.ZenModel.ZenModelBase import iscustprop
     35  from Products.ZenEvents.ZenEventClasses import Change_Add
     36
     37  from zenoss.protocols.protobufs.zep_pb2 import SEVERITY_INFO, SEVERITY_ERROR
     38
     39
     40 -class BatchDeviceLoader(ZCmdBase):
     41      """
     42      Base class wrapping around dmd.DeviceLoader
     43      """
     44
     45      sample_configs = """#
     46  # Example zenbatchloader device file
     47  #
     48  # This file is formatted with one entry per line, like this:
     49  #
     50  #  /Devices/device_class_name Python-expression
     51  #  hostname Python-expression
     52  #
     53  # For organizers (ie the /Devices path), the Python-expression
     54  # is used to define defaults to be used for devices listed
     55  # after the organizer. The defaults that can be specified are:
     56  #
     57  #   * loader arguments (use the --show_options flag to show these)
     58  #   * zPropertie (from a device, use the More -> zProperties
     59  #      menu option to see the available ones.)
     60  #
     61  #      NOTE: new zProperties *cannot* be created through this file
     62  #
     63  #  The Python-expression is used to create a dictionary of settings.
     64  #  device_settings = eval( 'dict(' + python-expression + ')' )
     65  #
     66
     67
     68  # Setting locations
     69  /Locations/Canada address="Canada"
     70
     71  /Locations/Canada/Alberta address="Alberta, Canada"
     72
     73  /Locations/Canada/Alberta/Calgary address="Calgary, Alberta, Canada"
     74
     75
     76  # If no organizer is specified at the beginning of the file,
     77  # defaults to the /Devices/Discovered device class.
     78  device0 comments="A simple device"
     79  # All settings must be seperated by a comma.
     80  device1 comments="A simple device", zSnmpCommunity='blue', zSnmpVer='v1'
     81
     82  # Notes for this file:
     83  #  * Oraganizer names *must* start with '/'
     84  #
     85  /Devices/Server/Linux zSnmpPort=1543
     86  # Python strings can use either ' or " -- there's no difference.
     87  # As a special case, it is also possible to specify the IP address
     88  linux_device1 setManageIp='10.10.10.77', zSnmpCommunity='blue', zSnmpVer="v2c"
     89  # A '\' at the end of the line allows you to place more
     90  # expressions on a new line. Don't forget the comma...
     91  linux_device2 zLinks="<a href='http://example.org'>Support site</a>",  \
     92  zTelnetEnable=True, \
     93  zTelnetPromptTimeout=15.3
     94
     95  # A new organizer drops all previous settings, and allows
     96  # for new ones to be used.  Settings do not span files.
     97  /Devices/Server/Windows zWinUser="administrator", zWinPassword='fred'
     98  # Bind templates
     99  windows_device1 zDeviceTemplates=[ 'Device', 'myTemplate' ]
    100  # Override the default from the organizer setting.
    101  windows_device2 zWinUser="administrator", zWinPassword='thomas'
    102
    103  # Apply other settings to the device
    104  settingsDevice setManageIp='10.10.10.77', setLocation="123 Elm Street", \
    105    setSystems=['/mySystems'], setPerformanceMonitor='remoteCollector1', \
    106    setHWSerialNumber="abc123456789", setGroups=['/myGroup'], \
    107    setHWProduct=('myproductName','manufacturer'), setOSProduct=('OS Name','manufacturer')
    108
    109  # If the device or device class contains a space, then it must be quoted (either ' or ")
    110  "/Server/Windows/WMI/Active Directory/2008"
    111
    112  # Now, what if we have a device that isn't really a device, and requires
    113  # a special loader?
    114  # The 'loader' setting requires a registered utility, and 'loader_arg_keys' is
    115  # a list from which any other settings will be passed into the loader callable.
    116  #
    117  # Here is a commmented-out example of how a VMware endpoint might be added:
    118  #
    119  #/Devices/VMware loader='vmware', loader_arg_keys=['host', 'username', 'password', 'useSsl', 'id']
    120  #esxwin2 id='esxwin2', host='esxwin2.zenoss.loc', username='testuser', password='password', useSsl=True
    121
    122  # Apply custom schema properties (c-properties) to a device
    123  windows_device7 cDateTest='2010/02/28'
    124
    125  """
    126
    127 -    def __init__(self, *args, **kwargs):
    128          ZCmdBase.__init__(self, *args, **kwargs)
    129          self.defaults = {}
    130
    131          self.loader = self.dmd.DeviceLoader.loadDevice
    132
    133          self.fqdn = socket.getfqdn()
    134          self.baseEvent = dict(
    135              device=self.fqdn,
    136              component='',
    137              agent='zenbatchload',
    138              monitor='localhost',
    139              manager=self.fqdn,
    140              severity=SEVERITY_ERROR,
    141              # Note: Change_Add is probably a better event class, but these
    142              #       events get sent to history by the default Zen property stuff
    143              #       on the event class. zendisc uses Status_Snmp, so so will we.
    144              eventClass=Change_Add,
    145          )
    146
    147          # Create the list of options we want people to know about
    148          self.loader_args = dict.fromkeys( self.loader.func_code.co_varnames )
    149          unsupportable_args = [
    150              'REQUEST', 'device', 'self', 'xmlrpc', 'e', 'handler',
    151          ]
    152          for opt in unsupportable_args:
    153              if opt in self.loader_args:
    154                  del self.loader_args[opt]
    155
    156 -    def loadDeviceList(self, args=None):
    157          """
    158          Read through all of the files listed as arguments and
    159          return a list of device entries.
    160
    161          @parameter args: list of filenames (uses self.args is this is None)
    162          @type args: list of strings
    163          @return: list of device specifications
    164          @rtype: list of dictionaries
    165          """
    166          if args is None:
    167              args = self.args
    168
    169          device_list = []
    170          for filename in args:
    171              if filename.strip() != '':
    172                  try:
    173                      data = open(filename,'r').readlines()
    174                  except IOError:
    175                      msg = "Unable to open the file '%s'" % filename
    176                      self.reportException(msg)
    177                      continue
    178
    179                  temp_dev_list = self.parseDevices(data)
    180                  if temp_dev_list:
    181                      device_list += temp_dev_list
    182
    183          return device_list
    184
    185 -    def applyZProps(self, device, device_specs):
    186          """
    187          Apply zProperty settings (if any) to the device.
    188
    189          @parameter device: device to modify
    190          @type device: DMD device object
    191          @parameter device_specs: device creation dictionary
    192          @type device_specs: dictionary
    193          """
    194          self.log.debug( "Applying zProperties..." )
    195          # Returns a list of (key, value) pairs.
    196          # Convert it to a dictionary.
    197          dev_zprops = dict( device.zenPropertyItems() )
    198
    199          for zprop, value in device_specs.items():
    200              self.log.debug( "Evaluating zProperty <%s -> %s> on %s" % (zprop, value, device.id) )
    201              if not iszprop(zprop):
    202                  self.log.debug( "Evaluating zProperty <%s -> %s> on %s: not iszprop()" % (zprop, value, device.id) )
    203                  continue
    204
    205              if zprop in dev_zprops:
    206                  try:
    207                      self.log.debug( "Setting zProperty <%s -> %s> on %s (currently set to %s)" % (
    208                         zprop, value, device.id, getattr(device, zprop, 'notset')) )
    209                      device.setZenProperty(zprop, value)
    210                  except BadRequest:
    211                      self.log.warn( "Object %s zproperty %s is invalid or duplicate" % (
    212                         device.titleOrId(), zprop) )
    213                  except Exception, ex:
    214                      self.log.warn( "Object %s zproperty %s not set (%s)" % (
    215                         device.titleOrId(), zprop, ex) )
    216                  self.log.debug( "Set zProperty <%s -> %s> on %s (now set to %s)" % (
    217                     zprop, value, device.id, getattr(device, zprop, 'notset')) )
    218              else:
    219                  self.log.warn( "The zproperty %s doesn't exist in %s" % (
    220                     zprop, device_specs.get('deviceName', device.id)))
    221
    222 -    def applyCustProps(self, device, device_specs):
    223          """
    224          Custom schema properties
    225          """
    226          self.log.debug( "Applying custom schema properties..." )
    227          dev_cprops = device.custPropertyMap()
    228
    229          for cprop, value in device_specs.items():
    230              if not iscustprop(cprop):
    231                 continue
    232
    233              matchProps = [prop for prop in dev_cprops if prop['id'] == cprop]
    234              if matchProps:
    235                  ctype = matchProps[0]['type']
    236                  if ctype == 'password':
    237                      ctype = 'string'
    238                  if ctype in type_converters and value:
    239                      value = type_converters[ctype](value)
    240                  device.setZenProperty( cprop, value)
    241              else:
    242                  self.log.warn( "The cproperty %s doesn't exist in %s" % (
    243                         cprop, device_specs.get('deviceName', device.id)))
    244
    245 -    def addAllLGSOrganizers(self, device_specs):
    246          location = device_specs.get('setLocation')
    247          if location:
    248              self.addLGSOrganizer('Locations', (location,) )
    249
    250          systems = device_specs.get('setSystems')
    251          if systems:
    252              if not isinstance(systems, list) and not isinstance(systems, tuple):
    253                  systems = (systems,)
    254              self.addLGSOrganizer('Systems', systems)
    255
    256          groups = device_specs.get('setGroups')
    257          if groups:
    258              if not isinstance(groups, list) and not isinstance(groups, tuple):
    259                  groups = (groups,)
    260              self.addLGSOrganizer('Groups', groups)
    261
    262 -    def addLGSOrganizer(self, lgsType, paths=[]):
    263          """
    264          Add any new locations, groups or organizers
    265          """
    266          prefix = '/zport/dmd/' + lgsType
    267          base = getattr(self.dmd, lgsType)
    268          if hasattr(base, 'sync'):
    269              base.sync()
    270          existing = [x.getPrimaryUrlPath().replace(prefix, '') \
    271                                        for x in base.getSubOrganizers()]
    272          for path in paths:
    273              if path in existing:
    274                  continue
    275              try:
    276                  base.manage_addOrganizer(path)
    277              except BadRequest:
    278                  pass
    279
    280      @transact
    281 -    def addOrganizer(self, device_specs):
    282          """
    283          Add any organizers as required, and apply zproperties to them.
    284          """
    285          path = device_specs.get('devicePath')
    286          baseOrg = path.split('/', 2)[1]
    287          base = getattr(self.dmd, baseOrg, None)
    288          if base is None:
    289              self.log.error("The base of path %s (%s) does not exist -- skipping",
    290                             baseOrg, path)
    291              return
    292
    293          try:
    294              org = base.getDmdObj(path)
    295          except KeyError:
    296              try:
    297                  self.log.info("Creating organizer %s", path)
    298                  @transact
    299                  def inner():
    300                      base.manage_addOrganizer(path)
    301                  inner()
    302                  org = base.getDmdObj(path)
    303              except IOError, ex:
    304                  self.log.error("Unable to create organizer! Is Rabbit up and configured correctly?")
    305                  sys.exit(1)
    306          self.applyZProps(org, device_specs)
    307          self.applyOtherProps(org, device_specs)
    308
    309 -    def applyOtherProps(self, device, device_specs):
    310          """
    311          Apply non-zProperty settings (if any) to the device.
    312
    313          @parameter device: device to modify
    314          @type device: DMD device object
    315          @parameter device_specs: device creation dictionary
    316          @type device_specs: dictionary
    317          """
    318          self.log.debug( "Applying other properties..." )
    319          internalVars = [
    320             'deviceName', 'devicePath', 'comments', 'loader', 'loader_arg_keys',
    321          ]
    322          @transact
    323          def setNamedProp(org, name, description):
    324              setattr(org, name, description)
    325
    326          for functor, value in device_specs.items():
    327              if iszprop(functor) or iscustprop(functor) or functor in internalVars:
    328                 continue
    329
    330              # Special case for organizers which can take a description
    331              if functor in ('description', 'address'):
    332                  if hasattr(device, functor):
    333                      setNamedProp(device, functor, value)
    334                  continue
    335
    336              try:
    337                  self.log.debug("For %s, calling device.%s(%s)",
    338                                device.id, functor, value)
    339                  func = getattr(device, functor, None)
    340                  if func is None or not callable(func):
    341                      self.log.warn("The function '%s' for device %s is not found.",
    342                                    functor, device.id)
    343                  elif isinstance(value, (list, tuple)):
    344                      # The function either expects a list or arguments
    345                      try: # arguments
    346                          func(*value)
    347                      except TypeError: # Try as a list
    348                          func(value)
    349                  else:
    350                      func(value)
    351              except ConflictError:
    352                  raise
    353              except Exception:
    354                  msg = "Device %s device.%s(%s) failed" % (device.id, functor, value)
    355                  self.reportException(msg, device.id)
    356
    357 -    def runLoader(self, loader, device_specs):
    358          """
    359          It's up to the loader now to figure out what's going on.
    360
    361          @parameter loader: device loader
    362          @type loader: callable
    363          @parameter device_specs: device entries
    364          @type device_specs: dictionary
    365          """
    366          argKeys = device_specs.get('loader_arg_keys', [])
    367          loader_args = {}
    368          for key in argKeys:
    369              if key in device_specs:
    370                  loader_args[key] = device_specs[key]
    371
    372          result = loader().load_device(self.dmd, **loader_args)
    373
    374          # If the loader returns back a device object, carry
    375          # on processing
    376          if isinstance(result, Device):
    377              return result
    378          return None
    379
    380 -    def processDevices(self, device_list):
    381          """
    382          Read the input and process the devices
    383            * create the device entry
    384            * set zproperties
    385            * set custom schema properties
    386            * model the device
    387
    388          @parameter device_list: list of device entries
    389          @type device_list: list of dictionaries
    390          """
    391
    392          def transactional(f):
    393              return f if self.options.nocommit else transact(f)
    394
    395          processed = {'total':0, 'errors':0}
    396
    397          @transactional
    398          def _process(device_specs):
    399              # Get the latest bits
    400              self.dmd.zport._p_jar.sync()
    401
    402              loaderName = device_specs.get('loader')
    403              if loaderName is not None:
    404                  try:
    405                      orgName = device_specs['devicePath']
    406                      organizer = self.dmd.getObjByPath('dmd' + orgName)
    407                      deviceLoader = getUtility(IDeviceLoader, loaderName, organizer)
    408                      devobj = self.runLoader(deviceLoader, device_specs)
    409                  except ConflictError:
    410                      raise
    411                  except Exception:
    412                      devName = device_specs.get('device_specs', 'Unkown Device')
    413                      msg = "Ignoring device loader issue for %s" % devName
    414                      self.reportException(msg, devName, specs=str(device_specs))
    415                      processed['errors'] += 1
    416                      return
    417              else:
    418                  devobj = self.getDevice(device_specs)
    419                  deviceLoader = None
    420
    421              if devobj is None:
    422                  if deviceLoader is not None:
    423                      processed['total'] += 1
    424              else:
    425                  self.addAllLGSOrganizers(device_specs)
    426                  self.applyZProps(devobj, device_specs)
    427                  self.applyCustProps(devobj, device_specs)
    428                  self.applyOtherProps(devobj, device_specs)
    429
    430              return devobj
    431
    432          @transactional
    433          def _snmp_community(device_specs, devobj):
    434              # Discover the SNMP community if it isn't explicitly set.
    435              if 'zSnmpCommunity' not in device_specs:
    436                  self.log.debug('Discovering SNMP version and community')
    437                  devobj.manage_snmpCommunity()
    438
    439          @transactional
    440          def _model(devobj):
    441              try:
    442                  devobj.collectDevice(setlog=self.options.showModelOutput)
    443              except ConflictError:
    444                  raise
    445              except Exception, ex:
    446                  msg = "Modeling error for %s" % devobj.id
    447                  self.reportException(msg, devobj.id, exception=str(ex))
    448                  processed['errors'] += 1
    449              processed['total'] += 1
    450
    451          for device_specs in device_list:
    452              devobj = _process(device_specs)
    453
    454              # We need to commit in order to model, so don't bother
    455              # trying to model unless we can do both
    456              if devobj and not self.options.nocommit and not self.options.nomodel:
    457                  _snmp_community(device_specs, devobj)
    458                  _model(devobj)
    459
    460          self.reportResults(processed, len(device_list))
    461
    462 -    def reportException(self, msg, devName='', **kwargs):
    463          """
    464          Report exceptions back to the the event console
    465          """
    466          self.log.exception(msg)
    467          if not self.options.nocommit:
    468              evt = self.baseEvent.copy()
    469              evt.update(dict(
    470                  summary=msg,
    471                  traceback=format_exc()
    472              ))
    473              evt.update(kwargs)
    474              if devName:
    475                  evt['device'] = devName
    476              self.dmd.ZenEventManager.sendEvent(evt)
    477
    478 -    def reportResults(self, processed, totalDevices):
    479          """
    480          Report the success + total counts from loading devices.
    481          """
    482          msg = "Modeled %d of %d devices, with %d errors" % (
    483                    processed['total'], totalDevices, processed['errors'] )
    484          self.log.info(msg)
    485
    486          if not self.options.nocommit:
    487              evt = self.baseEvent.copy()
    488              evt.update(dict(
    489                  severity=SEVERITY_INFO,
    490                  summary=msg,
    491                  modeled=processed['total'],
    492                  errors=processed['errors'],
    493                  total=totalDevices,
    494              ))
    495              self.dmd.ZenEventManager.sendEvent(evt)
    496
    497 -    def notifyNewDeviceCreated(self, deviceName):
    498          """
    499          Report that we added a new device.
    500          """
    501          if not self.options.nocommit:
    502              evt = self.baseEvent.copy()
    503              evt.update(dict(
    504                  severity=SEVERITY_INFO,
    505                  summary= "Added new device %s" % deviceName
    506              ))
    507              self.dmd.ZenEventManager.sendEvent(evt)
    508
    509 -    def getDevice(self, device_specs):
    510          """
    511          Find or create the specified device
    512
    513          @parameter device_specs: device creation dictionary
    514          @type device_specs: dictionary
    515          @return: device or None
    516          @rtype: DMD device object
    517          """
    518          if 'deviceName' not in device_specs:
    519              return None
    520          name = device_specs['deviceName']
    521          devobj  = self.dmd.Devices.findDevice(name)
    522          if devobj is not None:
    523              self.log.info("Found existing device %s" % name)
    524              return devobj
    525
    526          specs = {}
    527          for key in self.loader_args:
    528              if key in device_specs:
    529                  specs[key] = device_specs[key]
    530
    531          try:
    532              self.log.info("Creating device %s" % name)
    533
    534              # Do NOT model at this time
    535              specs['discoverProto'] = 'none'
    536
    537              self.loader(**specs)
    538              devobj  = self.dmd.Devices.findDevice(name)
    539              if devobj is None:
    540                  self.log.error("Unable to find newly created device %s -- skipping" \
    541                                % name)
    542              else:
    543                  self.notifyNewDeviceCreated(name)
    544
    545          except Exception:
    546              msg = "Unable to load %s -- skipping" % name
    547              self.reportException(msg, name)
    548
    549          return devobj
    550
    551 -    def buildOptions(self):
    552          """
    553          Add our command-line options to the basics
    554          """
    555          ZCmdBase.buildOptions(self)
    556
    557          self.parser.add_option('--show_options',
    558              dest="show_options", default=False,
    559              action="store_true",
    560              help="Show the various options understood by the loader")
    561
    562          self.parser.add_option('--sample_configs',
    563              dest="sample_configs", default=False,
    564              action="store_true",
    565              help="Show an example configuration file.")
    566
    567          self.parser.add_option('--showModelOutput',
    568              dest="showModelOutput", default=True,
    569              action="store_false",
    570              help="Show modelling activity")
    571
    572          self.parser.add_option('--nocommit',
    573              dest="nocommit", default=False,
    574              action="store_true",
    575              help="Don't commit changes to the ZODB. Use for verifying config file.")
    576
    577          self.parser.add_option('--nomodel',
    578              dest="nomodel", default=False,
    579              action="store_true",
    580              help="Don't model the remote devices. Must be able to commit changes.")
    581
    582 -    def parseDevices(self, data):
    583          """
    584          From the list of strings in rawDevices, construct a list
    585          of device dictionaries, ready to load into Zenoss.
    586
    587          @parameter data: list of strings representing device entries
    588          @type data: list of strings
    589          @return: list of parsed device entries
    590          @rtype: list of dictionaries
    591          """
    592          if not data:
    593              return []
    594
    595          comment = re.compile(r'^\s*#.*')
    596
    597          defaults = {'devicePath':"/Discovered" }
    598          finalList = []
    599          i = 0
    600          while i < len(data):
    601              line = data[i]
    602              line = re.sub(comment, '', line).strip()
    603              if line == '':
    604                  i += 1
    605                  continue
    606
    607              # Check for line continuation character '\'
    608              while line[-1] == '\\' and i < len(data):
    609                  i += 1
    610                  line = line[:-1] + data[i]
    611                  line = re.sub(comment, '', line).strip()
    612
    613              if line[0] == '/' or line[1] == '/': # Found an organizer
    614                  defaults = self.parseDeviceEntry(line, {})
    615                  if defaults is None:
    616                      defaults = {'devicePath':"/Discovered" }
    617                  else:
    618                      defaults['devicePath'] = defaults['deviceName']
    619                      del defaults['deviceName']
    620                  self.addOrganizer(defaults)
    621
    622              else:
    623                  configs = self.parseDeviceEntry(line, defaults)
    624                  if configs:
    625                      finalList.append(configs)
    626              i += 1
    627
    628          return finalList
    629
    630 -    def parseDeviceEntry(self, line, defaults):
    631          """
    632          Build a dictionary of properties from one line's input
    633
    634          @parameter line: string containing one device's info
    635          @type line: string
    636          @parameter defaults: dictionary of default settings
    637          @type defaults: dictionary
    638          @return: parsed device entry
    639          @rtype: dictionary
    640          """
    641          options = []
    642          # Note: organizers and device names can have spaces in them
    643          if line[0] in ["'", '"']:
    644              delim = line[0]
    645              eom = line.find(delim, 1)
    646              if eom == -1:
    647                  self.log.error("While reading name, unable to parse" \
    648                                 " the entry for %s -- skipping", line )
    649                  return None
    650              name = line[1:eom]
    651              options = line[eom+1:]
    652
    653          else:
    654              options = line.split(None, 1)
    655              name = options.pop(0)
    656              if options:
    657                  options = options.pop(0)
    658
    659          configs = defaults.copy()
    660          configs['deviceName'] = name
    661
    662          if options:
    663              try:
    664                  # Add a newline to allow for trailing comments
    665                  evalString = 'dict(' + options + '\n)'
    666                  configs.update(eval(evalString))
    667              except:
    668                  self.log.error( "Unable to parse the entry for %s -- skipping" % name )
    669                  self.log.error( "Raw string: %s" % options )
    670                  return None
    671
    672          return configs
    673
    674
    675  if __name__=='__main__':
    676      batchLoader = BatchDeviceLoader()
    677      if batchLoader.options.show_options:
    678          print "Options = %s" % sorted( batchLoader.loader_args.keys() )
    679          help(batchLoader.loader)
    680          sys.exit(0)
    681
    682      if batchLoader.options.sample_configs:
    683          print batchLoader.sample_configs
    684          sys.exit(0)
    685
    686      device_list = batchLoader.loadDeviceList()
    687      if not device_list:
    688          batchLoader.log.warn("No device entries found to load.")
    689          sys.exit(1)
    690
    691      batchLoader.processDevices(device_list)
    692      sys.exit(0)
    693