Package Products :: Package ZenModel :: Module zenmib |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.zenmib

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2007, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 as published by
      8  # the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13  __doc__= """zenmib
     14     The zenmib program converts MIBs into python data structures and then
     15  (by default) adds the data to the Zenoss DMD.  Essentially, zenmib is a
     16  wrapper program around the smidump program, whose output (python code) is
     17  then executed "inside" the Zope database.
     18
     19   Overview of terms:
     20     SNMP Simple Network Management Protocol
     21        A network protocol originally based on UDP which allows a management
     22        application (ie SNMP manager) to contact and get information from a
     23        device (ie router, computer, network-capable toaster).  The software
     24        on the device that is capable of understanding SNMP and responding
     25        appropriately is called an SNMP agent.
     26
     27     MIB Management of Information Base
     28        A description of what a particular SNMP agent provides and what
     29        traps it sends.  A MIB is a part of a tree structure based on a root
     30        MIB.  Since a MIB is a rooted tree, it allows for delegation of areas
     31        under the tree to different organizations.
     32
     33     ASN Abstract Syntax Notation
     34        The notation used to construct a MIB.
     35
     36     OID Object IDentifier
     37        A MIB is constructed of unique identifiers
     38
     39
     40   Background information:
     41     http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol
     42         Overview of SNMP.
     43
     44     http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en
     45         The libsmi project is the creator of smidump.  There are several
     46         interesting sub-projects available.
     47
     48     http://net-snmp.sourceforge.net/
     49         Homepage for Net-SNMP which is used by Zenoss for SNMP management.
     50
     51     http://www.et.put.poznan.pl/snmp/asn1/asn1.html
     52         An overview of Abstract Syntax Notation (ASN), the language in
     53         which MIBs are written.
     54  """
     55
     56  import os
     57  import os.path
     58  import sys
     59  import glob
     60  import re
     61  from subprocess import Popen, PIPE
     62
     63  import Globals
     64  import transaction
     65
     66  from Products.ZenUtils.ZCmdBase import ZCmdBase
     67  from Products.ZenUtils.Utils import zenPath
     68  from zExceptions import BadRequest
     69
     70
     71 -def walk(*dirs):
     72      """
     73      Generator function to create a list of absolute paths
     74      for all files in a directory tree starting from the list
     75      of directories given as arguments to this function.
     76
     77      @param *dirs: list of directories to investigate
     78      @type *dirs: list of strings
     79      @return: directory to investigate
     80      @rtype: string
     81      """
     82      for dir in dirs:
     83          for dirname, _, filenames in os.walk(dir):
     84              for filename in filenames:
     85                  yield os.path.join(dirname, filename)
     86
     87
     88
     89 -class DependencyMap:
     90      """
     91      A dependency is a reference to another part of the MIB tree.
     92      All MIB definitions start from the base of the tree (ie .1).
     93      Generally dependencies are from MIB definitions external to
     94      the MIB under inspection.
     95      """
     96
     97 -    def __init__(self):
     98          self.fileMap = {}
     99          self.depMap = {}
    100
    101
    102 -    def add(self, filename, name, dependencies):
    103          """
    104          Add a dependency to the dependency tree if it's not already there.
    105
    106          @param filename: name of MIB file to import
    107          @type filename: string
    108          @param name: name of MIB
    109          @type name: string
    110          @param dependencies: dependency
    111          @type dependencies: dependency object
    112          """
    113          if name not in self.depMap:
    114              self.fileMap[filename] = name
    115              self.depMap[name] = (filename, dependencies)
    116
    117
    118 -    def getName(self, filename):
    119          """
    120          Given a filename, return the name of the MIB tree defined in it.
    121          Makes the assumption that there's only one MIB tree per file.
    122
    123          @param filename: MIB filename
    124          @type filename: string
    125          @return: MIB name
    126          @rtype: string
    127          @todo: to allow for multiple MIBs in a file, should return a list
    128          """
    129          return self.fileMap.get(filename, None)
    130
    131
    132 -    def getDependencies(self, name):
    133          """
    134          Given a name of the MIB tree, return the filename that it's from
    135          and its dependencies.
    136
    137          @param name: name of MIB
    138          @type name: string
    139          @return: dependency tree
    140          @rtype: tuple of ( name, dependency object)
    141          """
    142          return self.depMap.get(name, None)
    143
    144
    145
    146 -class zenmib(ZCmdBase):
    147      """
    148      Wrapper around the smidump utilities to load MIB files into
    149      the DMD tree.
    150      """
    151 -    def map_file_to_dependents(self, mibfile):
    152          """
    153          Scan the MIB file to determine what MIB trees the file is dependent on.
    154
    155          @param mibfile: MIB filename
    156          @type mibfile: string
    157          @return: dependency tree
    158          @rtype: tuple of ( name, dependency object)
    159          """
    160          # Slurp in the whole file
    161          fp = open(mibfile)
    162          mib = fp.read()
    163          fp.close()
    164          return self.map_mib_to_dependents(mib)
    165
    166 -    def map_mib_to_dependents(self, mib):
    167          # ASN.1 syntax regular expressions for declaring MIBs
    168          #
    169          # An example from http://www.faqs.org/rfcs/rfc1212.html
    170          #
    171          # RFC1213-MIB DEFINITIONS ::= BEGIN
    172          #
    173          # IMPORTS
    174          #        experimental, OBJECT-TYPE, Counter
    175          #             FROM RFC1155-SMI;
    176          #
    177          #     -- contact IANA for actual number
    178          #
    179          # root    OBJECT IDENTIFIER ::= { experimental xx }
    180          #
    181          # END
    182
    183          DEFINITIONS = re.compile(r'(?P<mib_name>[A-Za-z-0-9]+) +DEFINITIONS *::= *BEGIN')
    184          DEPENDS = re.compile(r'FROM *(?P<mib_dependency>[A-Za-z-0-9]+)')
    185
    186          # Split up the file and determine what OIDs need what other OIDs
    187          #
    188          # TODO: Fix the code so that it takes care of the case where there are multiple
    189          #       OBJECT IDENTIFIER sections in the MIB.
    190          parts = mib.split('OBJECT IDENTIFIER', 1)
    191          mib_prologue = parts[0]
    192          match = DEFINITIONS.search(mib_prologue)
    193          if not match:
    194               # Special case: bootstrap MIB for the root of the MIB tree
    195               return None, []
    196
    197          # Search through the prologue to find all of the IMPORTS.
    198          # An example from a real MIB
    199          #
    200          # IMPORTS
    201          #        MODULE-IDENTITY, OBJECT-TYPE,  enterprises, Integer32,
    202          #        TimeTicks,NOTIFICATION-TYPE             FROM SNMPv2-SMI
    203          #        DisplayString                           FROM RFC1213-MIB
    204          #        MODULE-COMPLIANCE, OBJECT-GROUP,
    205          #        NOTIFICATION-GROUP                      FROM SNMPv2-CONF;
    206          depends = []
    207          name = match.group('mib_name')
    208          start = match.end(0)  # Jump to past the first FROM token
    209          while 1:
    210              match = DEPENDS.search(mib_prologue, start)
    211              if not match:
    212                  break
    213
    214              depends.append(match.group('mib_dependency'))
    215
    216              # Skip to just past the next FROM token
    217              start = match.end(0)
    218
    219          return name, depends
    220
    221
    222
    223 -    def dependencies(self, filenames):
    224          """
    225          Create a dependency map for all MIB files.
    226          Exit the program if we're missing any files.
    227
    228          @param filenames: names of MIB files to import
    229          @type filenames: list of strings
    230          @return: dependency tree
    231          @rtype: DependencyMap
    232          """
    233          missing_files = 0
    234          result = DependencyMap()
    235          for filename in filenames:
    236              try:
    237                  defines, depends = self.map_file_to_dependents(filename)
    238
    239              except IOError:
    240                  self.log.error( "Couldn't open file %s", filename)
    241                  missing_files += 1
    242                  continue
    243
    244              if defines == None:
    245                  self.log.debug( "Unable to parse information from %s -- skipping", filename)
    246              else:
    247                  result.add(filename, defines, depends)
    248
    249          if missing_files > 0:
    250              self.log.error( "Missing %s files", missing_files )
    251              sys.exit(1)
    252
    253          return result
    254
    255
    256
    257 -    def getDependencies(self, filename, depMap):
    258          """
    259          smidump needs to know the list of dependent files for a MIB file in
    260          order to properly resolve external references.
    261
    262          @param filename: name of MIB file to import
    263          @type filename: string
    264          @param depMap: dependency tree
    265          @type depMap: DependencyMap
    266          @return: list of dependencies
    267          @rtype: list of strings
    268          """
    269          # Sanity check: if a file doesn't need anything else, it
    270          # has no dependencies.  Avoid further work.
    271          name = depMap.getName(filename)
    272          if not name:
    273              return []
    274
    275          # Find the files required by the OID tree in the file.
    276          deps = []
    277          def dependency_search(name):
    278              """
    279              Create a list of files required by an OID.
    280
    281              @param name: name of OID
    282              @type name: string
    283              """
    284              fileAndDeps = depMap.getDependencies(name)
    285              if not fileAndDeps:
    286                  self.log.warn( "Unable to find a file providing the OID %s", name)
    287                  return
    288
    289              mib_file, dependent_oids = fileAndDeps
    290              if mib_file and mib_file not in deps:
    291                  deps.append(mib_file)
    292
    293              for unresolved_oid in dependent_oids:
    294                  dependency_search(unresolved_oid)
    295
    296          # Search the dependency map
    297          dependency_search(name)
    298          if deps[1:]:
    299              return deps[1:]
    300
    301          return []
    302
    303
    304
    305 -    def generate_python_from_mib( self, mibname, dependencies ):
    306          """
    307          Use the smidump program to convert a MIB into Python code"
    308
    309          @param mibname: name of the MIB
    310          @type mibname: string
    311          @param dependencies: list of dependent files
    312          @type dependencies: list of strings
    313          @return: the newly created MIB
    314          @rtype: MIB object
    315          """
    316          dump_command =  [ 'smidump', '-k', '-fpython' ]
    317          for dep in dependencies[1:]:
    318              #  Add command-line flag for our dependent files
    319              dump_command.append( '-p')
    320              dump_command.append( dep )
    321
    322          dump_command.append( mibname )
    323          self.log.debug('Running %s', ' '.join( dump_command))
    324          proc = Popen( dump_command, stdout=PIPE, stderr=PIPE )
    325
    326          python_code, warnings = proc.communicate()
    327          proc.wait()
    328          if proc.returncode:
    329              self.log.error(warnings)
    330              return None
    331
    332          if len(warnings) > 0:
    333              self.log.debug("Found warnings while trying to import MIB:\n%s" \
    334                   % warnings)
    335
    336          # Now we'll be brave and try to execute the MIB-to-python code
    337          # and store the resulting dictionary in 'result'
    338          result = {}
    339          try:
    340              exec python_code in result
    341
    342          except (SystemExit, KeyboardInterrupt): raise
    343          except:
    344              self.log.exception("Unable to import Pythonized-MIB: %s", mibname)
    345              return None
    346
    347          # Now look for the start of the MIB
    348          mib = result.get( 'MIB', None)
    349          return mib
    350
    351
    352
    353      # Standard MIB attributes that we expect in all MIBs
    354      MIB_MOD_ATTS = ('language', 'contact', 'description')
    355
    356 -    def load_mib(self, mibs, mibname, depmap):
    357          """
    358          Attempt to load a MIB after generating its dependency tree
    359
    360          @param mibs: filenames of the MIBs to load
    361          @type mibs: list of strings
    362          @param mibname: name of the MIB
    363          @type mibname: string
    364          @param dependencies: list of dependent files
    365          @type dependencies: string
    366          @return: whether the MIB load was successful or not
    367          @rtype: boolean
    368          """
    369          dependencies = self.getDependencies(mibname, depmap)
    370
    371          mib = self.generate_python_from_mib( mibname, dependencies )
    372          if not mib:
    373              return False
    374
    375          # Check for duplicates -- or maybe not...
    376          modname = mib['moduleName']
    377          # TODO: Find out Why this is commented out
    378          #mod = mibs.findMibModule(modname)
    379          mod = None
    380          if mod:
    381              self.log.warn( "Skipping %s as it is already loaded", modname)
    382              return False
    383
    384          # Create the container for the MIBs and define meta-data
    385          # In the DMD this creates another container class which
    386          # contains mibnodes.  These data types are found in
    387          # Products.ZenModel.MibModule and Products.ZenModel.MibNode
    388          mod = mibs.createMibModule(modname, self.options.path)
    389          for key, val in mib[modname].items():
    390              if key in self.MIB_MOD_ATTS:
    391                  setattr(mod, key, val)
    392
    393          # Add regular OIDs to the mibmodule + mibnode relationship tree
    394          nodes_added = 0
    395          if 'nodes' in mib:
    396              for name, values in mib['nodes'].items():
    397                  try:
    398                      mod.createMibNode(name, **values)
    399                      nodes_added += 1
    400                  except BadRequest:
    401                      try:
    402                          self.log.warn("Unable to add node id '%s' as this"
    403                                        " name is reserved for use by Zope",
    404                                        name)
    405                          newName = '_'.join([name, modname])
    406                          self.log.warn("Trying to add node '%s' as '%s'",
    407                                    name, newName)
    408                          mod.createMibNode(newName, **values)
    409                          self.log.warn("Renamed '%s' to '%s' and added to MIB"
    410                                    " nodes", name, newName)
    411                      except:
    412                          self.log.warn("Unable to add '%s' -- skipping",
    413                                        name)
    414
    415          # Put SNMP trap information into Products.ZenModel.MibNotification
    416          traps_added = 0
    417          if 'notifications' in mib:
    418              for name, values in mib['notifications'].items():
    419                  try:
    420                      mod.createMibNotification(name, **values)
    421                      traps_added += 1
    422                  except BadRequest:
    423                      try:
    424                          self.log.warn("Unable to add trap id '%s' as this"
    425                                        " name is reserved for use by Zope",
    426                                        name)
    427                          newName = '_'.join([name, modname])
    428                          self.log.warn("Trying to add trap '%s' as '%s'",
    429                                    name, newName)
    430                          mod.createMibNotification(newName, **values)
    431                          self.log.warn("Renamed '%s' to '%s' and added to MIB"
    432                                    " traps", name, newName)
    433                      except:
    434                          self.log.warn("Unable to add '%s' -- skipping",
    435                                        name)
    436
    437          self.log.info("Parsed %d nodes and %d notifications", nodes_added,
    438                        traps_added)
    439
    440          # Add the MIB tree permanently to the DMD except if we get the
    441          # --nocommit flag.
    442          if not self.options.nocommit:
    443              trans = transaction.get()
    444              trans.setUser( "zenmib" )
    445              trans.note("Loaded MIB %s into the DMD" % modname)
    446              trans.commit()
    447              self.log.info("Loaded MIB %s into the DMD", modname)
    448
    449          return True
    450
    451
    452
    453 -    def main(self):
    454          """
    455          Main loop of the program
    456          """
    457          # Prepare to load the default MIBs
    458          smimibdir = self.options.mibsdir
    459          if not os.path.exists( smimibdir ):
    460              self.log.error("The directory %s doesn't exist!" % smimibdir )
    461              sys.exit(1)
    462
    463          ietf, iana, irtf, tubs, site = \
    464                map(lambda x: os.path.join(smimibdir, x),
    465                    'ietf iana irtf tubs site'.split())
    466
    467          # Either load MIBs from the command-line or from the default
    468          # location where MIBs are stored by Zenoss.
    469          if len(self.args) > 0:
    470              mibnames = self.args
    471              depMap = self.dependencies(list(walk(ietf, iana, irtf, tubs))
    472                                         + mibnames)
    473          else:
    474              mibnames = glob.glob(os.path.join(smimibdir, 'site', '*'))
    475              depMap = self.dependencies(walk(ietf, iana, irtf, tubs, site))
    476
    477          # Make connection to the DMD at the start of the MIB tree
    478          mibs = self.dmd.Mibs
    479
    480          # Process all of the MIBs that we've found
    481          loaded_mib_files = 0
    482          for mibname in mibnames:
    483              try:
    484                  if self.load_mib( mibs, mibname, depMap):
    485                      loaded_mib_files += 1
    486
    487              except (SystemExit, KeyboardInterrupt): raise
    488              except Exception, ex:
    489                  self.log.exception("Failed to load MIB: %s" % mibname)
    490
    491          action = "Loaded"
    492          if self.options.nocommit:
    493              action = "Processed"
    494          self.log.info( "%s %d MIB file(s)" % ( action, loaded_mib_files))
    495          sys.exit(0)
    496
    497
    498 -    def buildOptions(self):
    499          """
    500          Command-line options
    501          """
    502          ZCmdBase.buildOptions(self)
    503          self.parser.add_option('--mibsdir',
    504                                 dest='mibsdir', default=zenPath('share/mibs'),
    505                                 help="Directory of input MIB files [ default: %default ]")
    506          self.parser.add_option('--path',
    507                                 dest='path', default="/",
    508                                 help="Path to load MIB into the DMD")
    509          self.parser.add_option('--nocommit', action='store_true',
    510                                 dest='nocommit', default=False,
    511                                 help="Don't commit the MIB to the DMD after loading")
    512
    513
    514  if __name__ == '__main__':
    515      zm = zenmib()
    516      zm.main()
    517