Package Products :: Package ZenModel :: Module BatchDeviceLoader |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.BatchDeviceLoader

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2009, 2011 Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 or (at your
      8  # option) any later version as published by the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13  __doc__ = """zenbatchload
     14
     15  zenbatchload loads a list of devices read from a file.
     16  """
     17
     18  import sys
     19  import re
     20  from traceback import format_exc
     21  import socket
     22
     23  import Globals
     24  from ZODB.POSException import ConflictError
     25  from ZODB.transact import transact
     26  from zope.component import getUtility
     27
     28  from zExceptions import BadRequest
     29
     30  from Products.ZenModel.interfaces import IDeviceLoader
     31  from Products.ZenUtils.ZCmdBase import ZCmdBase
     32  from Products.ZenModel.Device import Device
     33  from Products.ZenRelations.ZenPropertyManager import iszprop
     34  from Products.ZenModel.ZenModelBase import iscustprop
     35  from Products.ZenEvents.ZenEventClasses import Change_Add
     36
     37  from zenoss.protocols.protobufs.zep_pb2 import SEVERITY_INFO, SEVERITY_ERROR
     38
     39
     40 -class BatchDeviceLoader(ZCmdBase):
     41      """
     42      Base class wrapping around dmd.DeviceLoader
     43      """
     44
     45      sample_configs = """#
     46  # Example zenbatchloader device file
     47  #
     48  # This file is formatted with one entry per line, like this:
     49  #
     50  #  /Devices/device_class_name Python-expression
     51  #  hostname Python-expression
     52  #
     53  # For organizers (ie the /Devices path), the Python-expression
     54  # is used to define defaults to be used for devices listed
     55  # after the organizer. The defaults that can be specified are:
     56  #
     57  #   * loader arguments (use the --show_options flag to show these)
     58  #   * zPropertie (from a device, use the More -> zProperties
     59  #      menu option to see the available ones.)
     60  #
     61  #      NOTE: new zProperties *cannot* be created through this file
     62  #
     63  #  The Python-expression is used to create a dictionary of settings.
     64  #  device_settings = eval( 'dict(' + python-expression + ')' )
     65  #
     66
     67
     68  # If no organizer is specified at the beginning of the file,
     69  # defaults to the /Devices/Discovered device class.
     70  device0 comments="A simple device"
     71  # All settings must be seperated by a comma.
     72  device1 comments="A simple device", zSnmpCommunity='blue', zSnmpVer='v1'
     73
     74  # Notes for this file:
     75  #  * Oraganizer names *must* start with '/'
     76  #
     77  /Devices/Server/Linux zSnmpPort=1543
     78  # Python strings can use either ' or " -- there's no difference.
     79  # As a special case, it is also possible to specify the IP address
     80  linux_device1 setManageIp='10.10.10.77', zSnmpCommunity='blue', zSnmpVer="v2c"
     81  # A '\' at the end of the line allows you to place more
     82  # expressions on a new line. Don't forget the comma...
     83  linux_device2 zLinks="<a href='http://example.org'>Support site</a>",  \
     84  zTelnetEnable=True, \
     85  zTelnetPromptTimeout=15.3
     86
     87  # A new organizer drops all previous settings, and allows
     88  # for new ones to be used.  Settings do not span files.
     89  /Devices/Server/Windows zWinUser="administrator", zWinPassword='fred'
     90  # Bind templates
     91  windows_device1 zDeviceTemplates=[ 'Device', 'myTemplate' ]
     92  # Override the default from the organizer setting.
     93  windows_device2 zWinUser="administrator", zWinPassword='thomas'
     94
     95  # Apply other settings to the device
     96  settingsDevice setManageIp='10.10.10.77', setLocation="123 Elm Street", \
     97    setSystems=['/mySystems'], setPerformanceMonitor='remoteCollector1', \
     98    setHWSerialNumber="abc123456789", setGroups=['/myGroup'], \
     99    setHWProduct=('myproductName','manufacturer'), setOSProduct=('OS Name','manufacturer')
    100
    101  # If the device or device class contains a space, then it must be quoted (either ' or ")
    102  "/Server/Windows/WMI/Active Directory/2008"
    103
    104  # Now, what if we have a device that isn't really a device, and requires
    105  # a special loader?
    106  # The 'loader' setting requires a registered utility, and 'loader_arg_keys' is
    107  # a list from which any other settings will be passed into the loader callable.
    108  #
    109  # Here is a commmented-out example of how a VMware endpoint might be added:
    110  #
    111  #/Devices/VMware loader='vmware', loader_arg_keys=['host', 'username', 'password', 'useSsl', 'id']
    112  #esxwin2 id='esxwin2', host='esxwin2.zenoss.loc', username='testuser', password='password', useSsl=True
    113
    114  # Apply custom schema properties (c-properties) to a device
    115  windows_device7 cDateTest='2010/02/28'
    116
    117  """
    118
    119 -    def __init__(self, *args, **kwargs):
    120          ZCmdBase.__init__(self, *args, **kwargs)
    121          self.defaults = {}
    122
    123          self.loader = self.dmd.DeviceLoader.loadDevice
    124
    125          self.fqdn = socket.getfqdn()
    126          self.baseEvent = dict(
    127              device=self.fqdn,
    128              component='',
    129              agent='zenbatchload',
    130              monitor='localhost',
    131              manager=self.fqdn,
    132              severity=SEVERITY_ERROR,
    133              # Note: Change_Add is probably a better event class, but these
    134              #       events get sent to history by the default Zen property stuff
    135              #       on the event class. zendisc uses Status_Snmp, so so will we.
    136              eventClass=Change_Add,
    137          )
    138
    139          # Create the list of options we want people to know about
    140          self.loader_args = dict.fromkeys( self.loader.func_code.co_varnames )
    141          unsupportable_args = [
    142              'REQUEST', 'device', 'self', 'xmlrpc', 'e', 'handler',
    143          ]
    144          for opt in unsupportable_args:
    145              if opt in self.loader_args:
    146                  del self.loader_args[opt]
    147
    148 -    def loadDeviceList(self, args=None):
    149          """
    150          Read through all of the files listed as arguments and
    151          return a list of device entries.
    152
    153          @parameter args: list of filenames (uses self.args is this is None)
    154          @type args: list of strings
    155          @return: list of device specifications
    156          @rtype: list of dictionaries
    157          """
    158          if args is None:
    159              args = self.args
    160
    161          device_list = []
    162          for filename in args:
    163              try:
    164                  data = open(filename,'r').readlines()
    165              except IOError:
    166                  msg = "Unable to open the file '%s'" % filename
    167                  self.reportException(msg)
    168                  continue
    169
    170              temp_dev_list = self.parseDevices(data)
    171              if temp_dev_list:
    172                  device_list += temp_dev_list
    173
    174          return device_list
    175
    176 -    def applyZProps(self, device, device_specs):
    177          """
    178          Apply zProperty settings (if any) to the device.
    179
    180          @parameter device: device to modify
    181          @type device: DMD device object
    182          @parameter device_specs: device creation dictionary
    183          @type device_specs: dictionary
    184          """
    185          self.log.debug( "Applying zProperties..." )
    186          # Returns a list of (key, value) pairs.
    187          # Convert it to a dictionary.
    188          dev_zprops = dict( device.zenPropertyItems() )
    189
    190          for zprop, value in device_specs.items():
    191              if not iszprop(zprop):
    192                 continue
    193
    194              if zprop in dev_zprops:
    195                  try:
    196                      device.setZenProperty(zprop, value)
    197                  except BadRequest:
    198                      self.log.warn( "Object %s zproperty %s is invalid or duplicate" % (
    199                         device.titleOrId(), zprop) )
    200              else:
    201                  self.log.warn( "The zproperty %s doesn't exist in %s" % (
    202                         zprop, device_specs.get('deviceName', device.id)))
    203
    204 -    def applyCustProps(self, device, device_specs):
    205          """
    206          Custom schema properties
    207          """
    208          self.log.debug( "Applying custom schema properties..." )
    209          dev_cprops = device.custPropertyIds()
    210
    211          for cprop, value in device_specs.items():
    212              if not iscustprop(cprop):
    213                 continue
    214
    215              if cprop in dev_cprops:
    216                  setattr(device, cprop, value)
    217              else:
    218                  self.log.warn( "The cproperty %s doesn't exist in %s" % (
    219                         cprop, device_specs.get('deviceName', device.id)))
    220
    221 -    def addAllLGSOrganizers(self, device_specs):
    222          location = device_specs.get('setLocation')
    223          if location:
    224              self.addLGSOrganizer('Locations', (location,) )
    225
    226          systems = device_specs.get('setSystems')
    227          if systems:
    228              if not isinstance(systems, list) and not isinstance(systems, tuple):
    229                  systems = (systems,)
    230              self.addLGSOrganizer('Systems', systems)
    231
    232          groups = device_specs.get('setGroups')
    233          if groups:
    234              if not isinstance(groups, list) and not isinstance(groups, tuple):
    235                  groups = (groups,)
    236              self.addLGSOrganizer('Groups', groups)
    237
    238 -    def addLGSOrganizer(self, lgsType, paths=[]):
    239          """
    240          Add any new locations, groups or organizers
    241          """
    242          prefix = '/zport/dmd/' + lgsType
    243          base = getattr(self.dmd, lgsType)
    244          if hasattr(base, 'sync'):
    245              base.sync()
    246          existing = [x.getPrimaryUrlPath().replace(prefix, '') \
    247                                        for x in base.getSubOrganizers()]
    248          for path in paths:
    249              if path in existing:
    250                  continue
    251              try:
    252                  base.manage_addOrganizer(path)
    253              except BadRequest:
    254                  pass
    255
    256 -    def addOrganizer(self, device_specs):
    257          """
    258          Add any organizers as required, and apply zproperties to them.
    259          """
    260          path = device_specs.get('devicePath')
    261          baseOrg = path.split('/', 2)[1]
    262          base = getattr(self.dmd, baseOrg, None)
    263          if base is None:
    264              self.log.error("The base of path %s (%s) does not exist -- skipping",
    265                             baseOrg, path)
    266              return
    267
    268          try:
    269              org = base.getDmdObj(path)
    270          except KeyError:
    271              self.log.info("Creating organizer %s", path)
    272              @transact
    273              def inner():
    274                  base.manage_addOrganizer(path)
    275              inner()
    276              org = base.getDmdObj(path)
    277          self.applyZProps(org, device_specs)
    278          self.applyOtherProps(org, device_specs)
    279
    280 -    def applyOtherProps(self, device, device_specs):
    281          """
    282          Apply non-zProperty settings (if any) to the device.
    283
    284          @parameter device: device to modify
    285          @type device: DMD device object
    286          @parameter device_specs: device creation dictionary
    287          @type device_specs: dictionary
    288          """
    289          self.log.debug( "Applying other properties..." )
    290          internalVars = [
    291             'deviceName', 'devicePath', 'comments', 'loader', 'loader_arg_keys',
    292          ]
    293          @transact
    294          def setDescription(org, description):
    295              setattr(org, 'description', description)
    296
    297          for functor, value in device_specs.items():
    298              if iszprop(functor) or iscustprop(functor) or functor in internalVars:
    299                 continue
    300
    301              # Special case for organizers which can take a description
    302              if functor == 'description':
    303                  if hasattr(device, functor):
    304                      setDescription(device, value)
    305                  continue
    306
    307              try:
    308                  self.log.debug("For %s, calling device.%s(%s)",
    309                                device.id, functor, value)
    310                  func = getattr(device, functor, None)
    311                  if func is None:
    312                      self.log.warn("The function '%s' for device %s is not found.",
    313                                    functor, device.id)
    314                  elif isinstance(value, (list, tuple)):
    315                      # The function either expects a list or arguments
    316                      try: # arguments
    317                          func(*value)
    318                      except TypeError: # Try as a list
    319                          func(value)
    320                  else:
    321                      func(value)
    322              except ConflictError:
    323                  raise
    324              except Exception:
    325                  msg = "Device %s device.%s(%s) failed" % (device.id, functor, value)
    326                  self.reportException(msg, device.id)
    327
    328 -    def runLoader(self, loader, device_specs):
    329          """
    330          It's up to the loader now to figure out what's going on.
    331
    332          @parameter loader: device loader
    333          @type loader: callable
    334          @parameter device_specs: device entries
    335          @type device_specs: dictionary
    336          """
    337          argKeys = device_specs.get('loader_arg_keys', [])
    338          loader_args = {}
    339          for key in argKeys:
    340              if key in device_specs:
    341                  loader_args[key] = device_specs[key]
    342
    343          result = loader().load_device(self.dmd, **loader_args)
    344
    345          # If the loader returns back a device object, carry
    346          # on processing
    347          if isinstance(result, Device):
    348              return result
    349          return None
    350
    351 -    def processDevices(self, device_list):
    352          """
    353          Read the input and process the devices
    354            * create the device entry
    355            * set zproperties
    356            * set custom schema properties
    357            * model the device
    358
    359          @parameter device_list: list of device entries
    360          @type device_list: list of dictionaries
    361          """
    362
    363          def transactional(f):
    364              return f if self.options.nocommit else transact(f)
    365
    366          processed = {'total':0, 'errors':0}
    367
    368          @transactional
    369          def _process(device_specs):
    370              # Get the latest bits
    371              self.dmd.zport._p_jar.sync()
    372
    373              loaderName = device_specs.get('loader')
    374              if loaderName is not None:
    375                  try:
    376                      orgName = device_specs['devicePath']
    377                      organizer = self.dmd.getObjByPath('dmd' + orgName)
    378                      deviceLoader = getUtility(IDeviceLoader, loaderName, organizer)
    379                      devobj = self.runLoader(deviceLoader, device_specs)
    380                  except ConflictError:
    381                      raise
    382                  except Exception:
    383                      devName = device_specs.get('device_specs', 'Unkown Device')
    384                      msg = "Ignoring device loader issue for %s" % devName
    385                      self.reportException(msg, devName, {'specs':str(device_specs)})
    386                      processed['errors'] += 1
    387                      return
    388              else:
    389                  devobj = self.getDevice(device_specs)
    390                  deviceLoader = None
    391
    392              if devobj is None:
    393                  if deviceLoader is not None:
    394                      processed['total'] += 1
    395              else:
    396                  self.addAllLGSOrganizers(device_specs)
    397                  self.applyZProps(devobj, device_specs)
    398                  self.applyCustProps(devobj, device_specs)
    399                  self.applyOtherProps(devobj, device_specs)
    400
    401              return devobj
    402
    403          @transactional
    404          def _snmp_community(device_specs, devobj):
    405              # Discover the SNMP community if it isn't explicitly set.
    406              if 'zSnmpCommunity' not in device_specs:
    407                  self.log.debug('Discovering SNMP version and community')
    408                  devobj.manage_snmpCommunity()
    409
    410          @transactional
    411          def _model(devobj):
    412              try:
    413                  devobj.collectDevice(setlog=self.options.showModelOutput)
    414              except ConflictError:
    415                  raise
    416              except Exception, ex:
    417                  msg = "Modeling error for %s" % devobj.id
    418                  self.reportException(msg, devobj.id, ex)
    419                  processed['errors'] += 1
    420              processed['total'] += 1
    421
    422          for device_specs in device_list:
    423              devobj = _process(device_specs)
    424
    425              # We need to commit in order to model, so don't bother
    426              # trying to model unless we can do both
    427              if devobj and not self.options.nocommit and not self.options.nomodel:
    428                  _snmp_community(device_specs, devobj)
    429                  _model(devobj)
    430
    431          self.reportResults(processed, len(device_list))
    432
    433 -    def reportException(self, msg, devName='', **kwargs):
    434          """
    435          Report exceptions back to the the event console
    436          """
    437          self.log.exception(msg)
    438          if not self.options.nocommit:
    439              evt = self.baseEvent.copy()
    440              evt.update(dict(
    441                  summary=msg,
    442                  traceback=format_exc()
    443              ))
    444              evt.update(kwargs)
    445              if devName:
    446                  evt['device'] = devName
    447              self.dmd.ZenEventManager.sendEvent(evt)
    448
    449 -    def reportResults(self, processed, totalDevices):
    450          """
    451          Report the success + total counts from loading devices.
    452          """
    453          msg = "Modeled %d of %d devices, with %d errors" % (
    454                    processed['total'], totalDevices, processed['errors'] )
    455          self.log.info(msg)
    456
    457          if not self.options.nocommit:
    458              evt = self.baseEvent.copy()
    459              evt.update(dict(
    460                  severity=SEVERITY_INFO,
    461                  summary=msg,
    462                  modeled=processed['total'],
    463                  errors=processed['errors'],
    464                  total=totalDevices,
    465              ))
    466              self.dmd.ZenEventManager.sendEvent(evt)
    467
    468 -    def notifyNewDeviceCreated(self, deviceName):
    469          """
    470          Report that we added a new device.
    471          """
    472          if not self.options.nocommit:
    473              evt = self.baseEvent.copy()
    474              evt.update(dict(
    475                  severity=SEVERITY_INFO,
    476                  summary= "Added new device %s" % deviceName
    477              ))
    478              self.dmd.ZenEventManager.sendEvent(evt)
    479
    480 -    def getDevice(self, device_specs):
    481          """
    482          Find or create the specified device
    483
    484          @parameter device_specs: device creation dictionary
    485          @type device_specs: dictionary
    486          @return: device or None
    487          @rtype: DMD device object
    488          """
    489          if 'deviceName' not in device_specs:
    490              return None
    491          name = device_specs['deviceName']
    492          devobj  = self.dmd.Devices.findDevice(name)
    493          if devobj is not None:
    494              self.log.info("Found existing device %s" % name)
    495              return devobj
    496
    497          specs = {}
    498          for key in self.loader_args:
    499              if key in device_specs:
    500                  specs[key] = device_specs[key]
    501
    502          try:
    503              self.log.info("Creating device %s" % name)
    504
    505              # Do NOT model at this time
    506              specs['discoverProto'] = 'none'
    507
    508              self.loader(**specs)
    509              devobj  = self.dmd.Devices.findDevice(name)
    510              if devobj is None:
    511                  self.log.error("Unable to find newly created device %s -- skipping" \
    512                                % name)
    513              else:
    514                  self.notifyNewDeviceCreated(name)
    515
    516          except Exception:
    517              msg = "Unable to load %s -- skipping" % name
    518              self.reportException(msg, name)
    519
    520          return devobj
    521
    522 -    def buildOptions(self):
    523          """
    524          Add our command-line options to the basics
    525          """
    526          ZCmdBase.buildOptions(self)
    527
    528          self.parser.add_option('--show_options',
    529              dest="show_options", default=False,
    530              action="store_true",
    531              help="Show the various options understood by the loader")
    532
    533          self.parser.add_option('--sample_configs',
    534              dest="sample_configs", default=False,
    535              action="store_true",
    536              help="Show an example configuration file.")
    537
    538          self.parser.add_option('--showModelOutput',
    539              dest="showModelOutput", default=True,
    540              action="store_false",
    541              help="Show modelling activity")
    542
    543          self.parser.add_option('--nocommit',
    544              dest="nocommit", default=False,
    545              action="store_true",
    546              help="Don't commit changes to the ZODB. Use for verifying config file.")
    547
    548          self.parser.add_option('--nomodel',
    549              dest="nomodel", default=False,
    550              action="store_true",
    551              help="Don't model the remote devices. Must be able to commit changes.")
    552
    553 -    def parseDevices(self, data):
    554          """
    555          From the list of strings in rawDevices, construct a list
    556          of device dictionaries, ready to load into Zenoss.
    557
    558          @parameter data: list of strings representing device entries
    559          @type data: list of strings
    560          @return: list of parsed device entries
    561          @rtype: list of dictionaries
    562          """
    563          if not data:
    564              return []
    565
    566          comment = re.compile(r'^\s*#.*')
    567
    568          defaults = {'devicePath':"/Discovered" }
    569          finalList = []
    570          i = 0
    571          while i < len(data):
    572              line = data[i]
    573              line = re.sub(comment, '', line).strip()
    574              if line == '':
    575                  i += 1
    576                  continue
    577
    578              # Check for line continuation character '\'
    579              while line[-1] == '\\' and i < len(data):
    580                  i += 1
    581                  line = line[:-1] + data[i]
    582                  line = re.sub(comment, '', line).strip()
    583
    584              if line[0] == '/' or line[1] == '/': # Found an organizer
    585                  defaults = self.parseDeviceEntry(line, {})
    586                  if defaults is None:
    587                      defaults = {'devicePath':"/Discovered" }
    588                  else:
    589                      defaults['devicePath'] = defaults['deviceName']
    590                      del defaults['deviceName']
    591                  self.addOrganizer(defaults)
    592
    593              else:
    594                  configs = self.parseDeviceEntry(line, defaults)
    595                  if configs:
    596                      finalList.append(configs)
    597              i += 1
    598
    599          return finalList
    600
    601 -    def parseDeviceEntry(self, line, defaults):
    602          """
    603          Build a dictionary of properties from one line's input
    604
    605          @parameter line: string containing one device's info
    606          @type line: string
    607          @parameter defaults: dictionary of default settings
    608          @type defaults: dictionary
    609          @return: parsed device entry
    610          @rtype: dictionary
    611          """
    612          options = []
    613          # Note: organizers and device names can have spaces in them
    614          if line[0] in ["'", '"']:
    615              delim = line[0]
    616              eom = line.find(delim, 1)
    617              if eom == -1:
    618                  self.log.error("While reading name, unable to parse" \
    619                                 " the entry for %s -- skipping", line )
    620                  return None
    621              name = line[1:eom]
    622              options = line[eom+1:]
    623
    624          else:
    625              options = line.split(None, 1)
    626              name = options.pop(0)
    627              if options:
    628                  options = options.pop(0)
    629
    630          configs = defaults.copy()
    631          configs['deviceName'] = name
    632
    633          if options:
    634              try:
    635                  # Add a newline to allow for trailing comments
    636                  evalString = 'dict(' + options + '\n)'
    637                  configs.update(eval(evalString))
    638              except:
    639                  self.log.error( "Unable to parse the entry for %s -- skipping" % name )
    640                  self.log.error( "Raw string: %s" % options )
    641                  return None
    642
    643          return configs
    644
    645
    646  if __name__=='__main__':
    647      batchLoader = BatchDeviceLoader()
    648
    649      if batchLoader.options.show_options:
    650          print "Options = %s" % sorted( batchLoader.loader_args.keys() )
    651          help(batchLoader.loader)
    652          sys.exit(0)
    653
    654      if batchLoader.options.sample_configs:
    655          print batchLoader.sample_configs
    656          sys.exit(0)
    657
    658      device_list = batchLoader.loadDeviceList()
    659      if not device_list:
    660          batchLoader.log.warn("No device entries found to load.")
    661          sys.exit(1)
    662
    663      batchLoader.processDevices(device_list)
    664      sys.exit(0)
    665