Package Products :: Package ZenModel :: Module BatchDeviceLoader |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.BatchDeviceLoader

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2009, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 as published by
      8  # the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13  __doc__ = """zenbatchload
     14
     15  zenbatchload loads a list of devices read from a file.
     16  """
     17
     18  import sys
     19  import re
     20
     21  import Globals
     22  from transaction import commit
     23
     24  from Products.ZenUtils.ZCmdBase import ZCmdBase
     25  from Products.ZenRelations.ZenPropertyManager import iszprop
     26  from zExceptions import BadRequest
     27
     28
     29 -class BatchDeviceLoader(ZCmdBase):
     30      """
     31      Base class wrapping around dmd.DeviceLoader
     32      """
     33
     34      sample_configs = """#
     35  # Example zenbatchloader device file
     36  #
     37  # This file is formatted with one entry per line, like this:
     38  #
     39  #  /Devices/device_class_name Python-expression
     40  #  hostname Python-expression
     41  #
     42  # For organizers (ie the /Devices path), the Python-expression
     43  # is used to define defaults to be used for devices listed
     44  # after the organizer. The defaults that can be specified are:
     45  #
     46  #   * loader arguments (use the --show_options flag to show these)
     47  #   * zPropertie (from a device, use the More -> zProperties
     48  #      menu option to see the available ones.)
     49  #
     50  #      NOTE: new zProperties *cannot* be created through this file
     51  #
     52  #  The Python-expression is used to create a dictionary of settings.
     53  #  device_settings = eval( 'dict(' + python-expression + ')' )
     54  #
     55
     56
     57  # If no organizer is specified at the beginning of the file,
     58  # defaults to the /Devices/Discovered device class.
     59  device0 comments="A simple device"
     60  # All settings must be seperated by a comma.
     61  device1 comments="A simple device", zSnmpCommunity='blue', zSnmpVer='v1'
     62
     63  # Notes for this file:
     64  #  * Oraganizer names *must* start with '/'
     65  #
     66  /Devices/Server/Linux zSnmpPort=1543
     67  # Python strings can use either ' or " -- there's no difference.
     68  # As a special case, it is also possible to specify the IP address
     69  linux_device1 setManageIp='10.10.10.77', zSnmpCommunity='blue', zSnmpVer="v2c"
     70  # A '\' at the end of the line allows you to place more
     71  # expressions on a new line. Don't forget the comma...
     72  linux_device2 zLinks="<a href='http://example.org'>Support site</a>",  \
     73  zTelnetEnable=True, \
     74  zTelnetPromptTimeout=15.3
     75
     76  # A new organizer drops all previous settings, and allows
     77  # for new ones to be used.  Settings do not span files.
     78  /Devices/Server/Windows zWinUser="administrator", zWinPassword='fred'
     79  # Bind templates
     80  windows_device1 zDeviceTemplates=[ 'Device', 'myTemplate' ]
     81  # Override the default from the organizer setting.
     82  windows_device2 zWinUser="administrator", zWinPassword='thomas'
     83
     84  # Apply other settings to the device
     85  settingsDevice setManageIp='10.10.10.77', setLocation="123 Elm Street", \
     86    setSystems='/mySystems', setPerformanceMonitor='remoteCollector1', \
     87    setHWSerialNumber="abc123456789", setGroups='/myGroup', \
     88    setHWProduct=('myproductName','manufacturer'), setOSProduct=('OS Name','manufacturer')
     89
     90  # If the device or device class contains a space, then it must be quoted (either ' or ")
     91  "/Server/Windows/WMI/Active Directory/2008"
     92  """
     93
     94 -    def __init__(self):
     95          ZCmdBase.__init__(self)
     96          self.defaults = {}
     97
     98          self.loader = self.dmd.DeviceLoader.loadDevice
     99
    100          # Create the list of options we want people to know about
    101          self.loader_args = dict.fromkeys( self.loader.func_code.co_varnames )
    102          unsupportable_args = [
    103              'REQUEST', 'device', 'self', 'xmlrpc', 'e', 'handler',
    104          ]
    105          for opt in unsupportable_args:
    106              if opt in self.loader_args:
    107                  del self.loader_args[opt]
    108
    109 -    def loadDeviceList(self, args=None):
    110          """
    111          Read through all of the files listed as arguments and
    112          return a list of device entries.
    113
    114          @parameter args: list of filenames (uses self.args is this is None)
    115          @type args: list of strings
    116          @return: list of device specifications
    117          @rtype: list of dictionaries
    118          """
    119          if args is None:
    120              args = self.args
    121
    122          device_list = []
    123          for filename in args:
    124              try:
    125                  data = open(filename,'r').readlines()
    126              except IOError:
    127                  self.log.critical("Unable to open the file '%s'" % filename)
    128                  continue
    129
    130              temp_dev_list = self.parseDevices(data)
    131              if temp_dev_list:
    132                  device_list += temp_dev_list
    133
    134          return device_list
    135
    136 -    def applyZProps(self, device, device_specs):
    137          """
    138          Apply zProperty settings (if any) to the device.
    139
    140          @parameter device: device to modify
    141          @type device: DMD device object
    142          @parameter device_specs: device creation dictionary
    143          @type device_specs: dictionary
    144          """
    145          self.log.debug( "Applying zProperties..." )
    146          # Returns a list of (key, value) pairs.
    147          # Convert it to a dictionary.
    148          dev_zprops = dict( device.zenPropertyItems() )
    149
    150          for zprop, value in device_specs.items():
    151              if not iszprop(zprop):
    152                 continue
    153
    154              if zprop in dev_zprops:
    155                  try:
    156                      device.setZenProperty(zprop, value)
    157                  except BadRequest:
    158                      self.log.warn( "Device %s zproperty %s is invalid or duplicate" % (
    159                         device_specs['deviceName'], zprop) )
    160              else:
    161                  self.log.warn( "The zproperty %s doesn't exist in %s" % (
    162                         zprop, device_specs['deviceName']))
    163
    164 -    def applyOtherProps(self, device, device_specs):
    165          """
    166          Apply non-zProperty settings (if any) to the device.
    167
    168          @parameter device: device to modify
    169          @type device: DMD device object
    170          @parameter device_specs: device creation dictionary
    171          @type device_specs: dictionary
    172          """
    173          self.log.debug( "Applying other properties..." )
    174          internalVars = [
    175             'deviceName', 'devicePath', 'comments',
    176          ]
    177          for functor, value in device_specs.items():
    178              if iszprop(functor) or functor in internalVars:
    179                 continue
    180
    181              try:
    182                  self.log.debug("For %s, calling device.%s(%s)",
    183                                device.id, functor, value)
    184                  func = getattr(device, functor, None)
    185                  if func is None:
    186                      self.log.warn("The function '%s' for device %s is not found.",
    187                                    functor, device.id)
    188                  elif isinstance(value, type([])) or isinstance(value, type(())):
    189                      func(*value)
    190                  else:
    191                      func(value)
    192              except (SystemExit, KeyboardInterrupt): raise
    193              except:
    194                  self.log.exception("Device %s device.%s(%s) failed" % (
    195                                     device.id, functor, value))
    196
    197 -    def processDevices(self, device_list):
    198          """
    199          Read the input and process the devices
    200            * create the device entry
    201            * set zproperties
    202            * model the device
    203
    204          @parameter device_list: list of device entries
    205          @type device_list: list of dictionaries
    206          """
    207          processed = 0
    208          for device_specs in device_list:
    209              devobj = self.getDevice(device_specs)
    210              if devobj is None:
    211                  continue
    212
    213              self.applyZProps(devobj, device_specs)
    214              self.applyOtherProps(devobj, device_specs)
    215
    216              # We need to commit in order to model, so don't bother
    217              # trying to model unless we can do both
    218              if not self.options.nocommit and not self.options.nomodel:
    219                  # What if zSnmpCommunity isn't set in the file?
    220                  devobj.manage_snmpCommunity()
    221
    222                  # Make sure that ZODB has changes before modeling
    223                  commit()
    224                  try:
    225                      devobj.collectDevice(setlog=self.options.showModelOutput)
    226                  except (SystemExit, KeyboardInterrupt):
    227                      self.log.info("User interrupted modeling")
    228                      break
    229                  except Exception, ex:
    230                      self.log.exception("Modeling error for %s", devobj.id)
    231
    232              if not self.options.nocommit:
    233                  commit()
    234              processed += 1
    235
    236          self.log.info( "Processed %d of %d devices" % (processed, len(device_list)))
    237
    238 -    def getDevice(self, device_specs):
    239          """
    240          Find or create the specified device
    241
    242          @parameter device_specs: device creation dictionary
    243          @type device_specs: dictionary
    244          @return: device or None
    245          @rtype: DMD device object
    246          """
    247          if 'deviceName' not in device_specs:
    248              return None
    249          name = device_specs['deviceName']
    250          devobj  = self.dmd.Devices.findDevice(name)
    251          if devobj is not None:
    252              self.log.info("Found existing device %s" % name)
    253              return devobj
    254
    255          specs = {}
    256          for key in self.loader_args:
    257              if key in device_specs:
    258                  specs[key] = device_specs[key]
    259
    260          try:
    261              self.log.info("Creating device %s" % name)
    262
    263              # Do NOT model at this time
    264              specs['discoverProto'] = 'none'
    265
    266              self.loader(**specs)
    267              devobj  = self.dmd.Devices.findDevice(name)
    268              if devobj is None:
    269                  self.log.error("Unable to find newly created device %s -- skipping" \
    270                                % name)
    271          except Exception, ex:
    272              self.log.exception("Unable to load %s -- skipping" % name )
    273
    274          return devobj
    275
    276 -    def buildOptions(self):
    277          """
    278          Add our command-line options to the basics
    279          """
    280          ZCmdBase.buildOptions(self)
    281
    282          self.parser.add_option('--show_options',
    283              dest="show_options", default=False,
    284              action="store_true",
    285              help="Show the various options understood by the loader")
    286
    287          self.parser.add_option('--sample_configs',
    288              dest="sample_configs", default=False,
    289              action="store_true",
    290              help="Show an example configuration file.")
    291
    292          self.parser.add_option('--showModelOutput',
    293              dest="showModelOutput", default=True,
    294              action="store_false",
    295              help="Show modelling activity")
    296
    297          self.parser.add_option('--nocommit',
    298              dest="nocommit", default=False,
    299              action="store_true",
    300              help="Don't commit changes to the ZODB. Use for verifying config file.")
    301
    302          self.parser.add_option('--nomodel',
    303              dest="nomodel", default=False,
    304              action="store_true",
    305              help="Don't model the remote devices. Must be able to commit changes.")
    306
    307 -    def parseDevices(self, data):
    308          """
    309          From the list of strings in rawDevices, construct a list
    310          of device dictionaries, ready to load into Zenoss.
    311
    312          @parameter data: list of strings representing device entries
    313          @type data: list of strings
    314          @return: list of parsed device entries
    315          @rtype: list of dictionaries
    316          """
    317          if not data:
    318              return []
    319
    320          comment = re.compile(r'^\s*#.*')
    321
    322          defaults = {'devicePath':"/Discovered" }
    323          finalList = []
    324          i = 0
    325          while i < len(data):
    326              line = data[i]
    327              line = re.sub(comment, '', line).strip()
    328              if line == '':
    329                  i += 1
    330                  continue
    331
    332              # Check for line continuation character '\'
    333              while line[-1] == '\\' and i < len(data):
    334                  i += 1
    335                  line = line[:-1] + data[i]
    336                  line = re.sub(comment, '', line).strip()
    337
    338              if line[0] == '/' or line[1] == '/': # Found an organizer
    339                  defaults = self.parseDeviceEntry(line, {})
    340                  if defaults is None:
    341                      defaults = {'devicePath':"/Discovered" }
    342                  else:
    343                      defaults['devicePath'] = defaults['deviceName']
    344                      del defaults['deviceName']
    345
    346              else:
    347                  configs = self.parseDeviceEntry(line, defaults)
    348                  if configs:
    349                      finalList.append(configs)
    350              i += 1
    351
    352          return finalList
    353
    354 -    def parseDeviceEntry(self, line, defaults):
    355          """
    356          Build a dictionary of properties from one line's input
    357
    358          @parameter line: string containing one device's info
    359          @type line: string
    360          @parameter defaults: dictionary of default settings
    361          @type defaults: dictionary
    362          @return: parsed device entry
    363          @rtype: dictionary
    364          """
    365          options = []
    366          # Note: organizers and device names can have spaces in them
    367          if line[0] in ["'", '"']:
    368              delim = line[0]
    369              eom = line.find(delim, 1)
    370              if eom == -1:
    371                  self.log.error( "Unable to parse the entry for %s -- skipping" % name )
    372                  self.log.error( "Raw string: %s" % options )
    373                  return None
    374              name = line[1:eom]
    375              options = line[eom+1:]
    376
    377          else:
    378              options = line.split(None, 1)
    379              name = options.pop(0)
    380              if options:
    381                  options = options.pop(0)
    382
    383          configs = defaults.copy()
    384          configs['deviceName'] = name
    385
    386          if options:
    387              try:
    388                  # Add a newline to allow for trailing comments
    389                  evalString = 'dict(' + options + '\n)'
    390                  configs.update(eval(evalString))
    391              except:
    392                  self.log.error( "Unable to parse the entry for %s -- skipping" % name )
    393                  self.log.error( "Raw string: %s" % options )
    394                  return None
    395
    396          return configs
    397
    398
    399  if __name__=='__main__':
    400      batchLoader = BatchDeviceLoader()
    401
    402      if batchLoader.options.show_options:
    403          print "Options = %s" % sorted( batchLoader.loader_args.keys() )
    404          help(batchLoader.loader)
    405          sys.exit(0)
    406
    407      if batchLoader.options.sample_configs:
    408          print batchLoader.sample_configs
    409          sys.exit(0)
    410
    411      device_list = batchLoader.loadDeviceList()
    412      batchLoader.processDevices(device_list)
    413