Package Products :: Package ZenModel :: Module zenmib |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.zenmib

       1  ##############################################################################
       2  #
       3  # Copyright (C) Zenoss, Inc. 2007, 2009, all rights reserved.
       4  #
       5  # This content is made available according to terms specified in
       6  # License.zenoss under the directory where your Zenoss product is installed.
       7  #
       8  ##############################################################################
       9
      10
      11  __doc__ = """zenmib
      12     The zenmib program converts MIBs into python data structures and then
      13  (by default) adds the data to the Zenoss DMD.  Essentially, zenmib is a
      14  wrapper program around the smidump program, whose output (python code) is
      15  then executed "inside" the Zope database.
      16
      17   Overview of terms:
      18     SNMP Simple Network Management Protocol
      19        A network protocol originally based on UDP which allows a management
      20        application (ie SNMP manager) to contact and get information from a
      21        device (ie router, computer, network-capable toaster).  The software
      22        on the device that is capable of understanding SNMP and responding
      23        appropriately is called an SNMP agent.
      24
      25     MIB Management of Information Base
      26        A description of what a particular SNMP agent provides and what
      27        traps it sends.  A MIB is a part of a tree structure based on a root
      28        MIB.  Since a MIB is a rooted tree, it allows for delegation of areas
      29        under the tree to different organizations.
      30
      31     ASN Abstract Syntax Notation
      32        The notation used to construct a MIB.
      33
      34     OID Object IDentifier
      35        A MIB is constructed of unique identifiers
      36
      37
      38   Background information:
      39     http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol
      40         Overview of SNMP.
      41
      42     http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en
      43         The libsmi project is the creator of smidump.  There are several
      44         interesting sub-projects available.
      45
      46     http://net-snmp.sourceforge.net/
      47         Homepage for Net-SNMP which is used by Zenoss for SNMP management.
      48
      49     http://www.et.put.poznan.pl/snmp/asn1/asn1.html
      50         An overview of Abstract Syntax Notation (ASN), the language in
      51         which MIBs are written.
      52  """
      53
      54  import os
      55  import os.path
      56  import sys
      57  import re
      58  from subprocess import Popen, PIPE
      59  import tempfile
      60  import urllib
      61  import tarfile
      62  import zipfile
      63  from urlparse import urlsplit
      64  from collections import OrderedDict
      65
      66  import Globals
      67  import transaction
      68
      69  from Products.ZenUtils.ZCmdBase import ZCmdBase
      70  from Products.ZenUtils.Utils import zenPath
      71  from zExceptions import BadRequest
      72
      73
      74  CHUNK_SIZE = 50
      75
      76
      77 -class MibFile(object):
      78      """
      79      A MIB file has the meta-data for a MIB inside of it.
      80      """
      81 -    def __init__(self, fileName, fileContents=""):
      82          self.fileName = fileName
      83          self.mibs = [] # Order of MIBs defined in the file
      84          self.mibToDeps = {} # Dependency defintions for each MIB
      85          self.fileContents = self.removeMibComments(fileContents)
      86          self.mibDefinitions = self.splitFileToMIBs(self.fileContents)
      87          self.mapMibToDependents(self.mibDefinitions)
      88
      89          # Reclaim some memory
      90          self.fileContents = ""
      91          self.mibDefinitions = ""
      92
      93 -    def removeMibComments(self, fileContents):
      94          """
      95          Parses the string provided as an argument and extracts
      96          all of the ASN.1 comments from the string.
      97
      98          Assumes that fileContents contains the contents of a well-formed
      99          (no errors) MIB file.
     100
     101          @param fileContents: entire contents of a MIB file
     102          @type fileContents: string
     103          @return: text without any comments
     104          @rtype: string
     105          """
     106          if not fileContents:
     107              return fileContents
     108
     109          cursor = 0
     110          fc = []
     111          while cursor < len(fileContents):
     112              searchParty = [ fileContents.find('"',cursor), fileContents.find('/*',cursor), fileContents.find('--',cursor) ]
     113              try:
     114                  cursor = min( x for x in searchParty if x >= 0 )
     115              except ValueError:
     116                  break
     117
     118              if searchParty[0] == cursor:
     119                  '''
     120                  MIB block quote rules
     121                  1.  Begins with '"'
     122                  2.  Ends with '"'
     123                  3.  Quotes can be nested with '""'
     124                  4.  Any characters between the beginning and the end of the string
     125                      are considered part of the string.  All comment characters like
     126                      ('--') and ('/*...*/') are not ignored.  Newlines are included
     127                      in the blockquote
     128                  '''
     129                  cursor = fileContents.find('"',cursor+1) + 1
     130                  while 0 < cursor < len(fileContents) and fileContents[cursor] == '"':
     131                      cursor = fileContents.find('"', cursor + 1) + 1
     132
     133                  # Syntax error if no close quote found
     134                  if not cursor:
     135                      raise ValueError("Syntax Error: missing close (\")")
     136
     137                  continue
     138              elif searchParty[1] == cursor:
     139                  '''
     140                  MIB block comment rules:
     141                  1.  Begins with '/*'
     142                  2.  Ends with '*/'
     143                  3.  Block comments can be nested
     144                  4.  Any characters between the beginning and end of the comment
     145                      are ignored as part of the comment, including quotes and
     146                      start/end delimiters for single line comments ('--'). Newlines
     147                      are included as part of the block comment.
     148                  '''
     149                  endComment = fileContents.find("*/",cursor+2)
     150                  if endComment < 0:
     151                      raise ValueError("Syntax Error: missing close (*/)")
     152
     153                  nestCount = 1
     154                  # Check for a nested block
     155                  while nestCount != fileContents.count("/*", cursor, endComment):
     156
     157                      # Corner case for /*/
     158                      if fileContents[endComment-1] == '/':
     159                          fc_list = list(fileContents)
     160                          fc_list[endComment-1] = ' '
     161                          fileContents = ''.join(fc_list)
     162                      endComment = fileContents.find("*/", endComment+2)
     163                      if endComment < 0:
     164                          raise ValueError("Syntax Error: missing close (*/)")
     165
     166                      nestCount += 1
     167
     168                  endComment += 2
     169              elif searchParty[2] == cursor:
     170                  '''
     171                  MIB single line comment rules:
     172                  1.  Begins with '--'
     173                  2.  Ends with '--' or newline
     174                  3.  Any characters between the beginning and end of the comment
     175                      are ignored as part of the comment, including quotes and
     176                      start/end delimiters for block comments ( '/*' and '*/')
     177                  '''
     178                  endComment = fileContents.find('\n',cursor+2)
     179
     180                  #EOF reached
     181                  if endComment < 0:
     182                      endComment = len(fileContents)
     183
     184                  # Ignore dash lines
     185                  dashline = fileContents[cursor:endComment].strip()
     186                  if dashline != '-' * len(dashline):
     187                      ec = fileContents.find('--', cursor+2, endComment)
     188                      if ec > -1:
     189                          endComment = ec+2
     190
     191              fc.append(fileContents[:cursor])
     192              fileContents = fileContents[endComment:]
     193              cursor = 0
     194
     195          fc.append(fileContents)
     196          return ''.join(fc)
     197
     198 -    def splitFileToMIBs(self, fileContents):
     199          """
     200          Isolates each MIB definition in fileContents into a separate string
     201
     202          @param fileContents: the complete contents of a MIB file
     203          @type fileContents: string
     204          @return: MIB definition strings
     205          @rtype: list of strings
     206          """
     207          if fileContents is None:
     208              return []
     209
     210          DEFINITIONS = re.compile(r'([A-Za-z-0-9]+\s+DEFINITIONS'
     211              '(\s+EXPLICIT TAGS|\s+IMPLICIT TAGS|\s+AUTOMATIC TAGS|\s*)'
     212              '(\s+EXTENSIBILITY IMPLIED|\s*)\s*::=\s*BEGIN)')
     213
     214          definitionSpans = []
     215          for definitionMatch in DEFINITIONS.finditer(fileContents):
     216              definitionSpans.append(list(definitionMatch.span()))
     217              # If more than one definiton in the file, set the end of the
     218              # last span to the beginning of the current span
     219              if len(definitionSpans) > 1:
     220                  definitionSpans[-2][1] = definitionSpans[-1][0]
     221
     222          # Use the start and end positions to create a string for each
     223          # MIB definition
     224          mibDefinitions = []
     225          if definitionSpans:
     226              # Set the end of the last span to the end of the file
     227              definitionSpans[-1][1] = len(fileContents)
     228              for definitionSpan in definitionSpans:
     229                  mibDefinitions.append(
     230                      fileContents[definitionSpan[0]:definitionSpan[1]])
     231
     232          return mibDefinitions
     233
     234 -    def mapMibToDependents(self, mibDefinitions):
     235          # ASN.1 syntax regular expressions for declaring MIBs
     236          #
     237          # An example from http://www.faqs.org/rfcs/rfc1212.html
     238          #
     239          # RFC1213-MIB DEFINITIONS ::= BEGIN
     240          #
     241          # IMPORTS
     242          #        experimental, OBJECT-TYPE, Counter
     243          #             FROM RFC1155-SMI;
     244          #
     245          #     -- a MIB may or may not have an IMPORTS section
     246          #
     247          # root    OBJECT IDENTIFIER ::= { experimental xx }
     248          #
     249          # END
     250          IMPORTS = re.compile(r'\sIMPORTS\s.+;', re.DOTALL)
     251          DEPENDENCIES = re.compile(
     252              r'\sFROM\s+(?P<dependency>[A-Za-z-0-9]+)')
     253
     254          for definition in mibDefinitions:
     255              mibName = re.split(r'([A-Za-z0-9-]+)', definition)[1]
     256              dependencies = set()
     257              importsMatch = IMPORTS.search(definition)
     258              if importsMatch:
     259                  imports = importsMatch.group()
     260                  for dependencyMatch in DEPENDENCIES.finditer(imports):
     261                      dependencies.add(dependencyMatch.group('dependency'))
     262              self.mibs.append(mibName)
     263              self.mibToDeps[mibName] = dependencies
     264
     265 -class PackageManager:
     266      """
     267      Given an URL, filename or archive (eg zip, tar), extract the files from
     268      the package and return a list of filenames.
     269      """
     270 -    def __init__(self, log, downloaddir, extractdir):
     271          """
     272          Initialize the packagae manager.
     273
     274          @parameter log: logging object
     275          @type log: logging class object
     276          @parameter downloaddir: directory name to store downloads
     277          @type downloaddir: string
     278          @parameter extractdir: directory name to store downloads
     279          @type extractdir: string
     280          """
     281          self.log = log
     282          self.downloaddir = downloaddir
     283          if self.downloaddir[-1] != '/':
     284              self.downloaddir += '/'
     285          self.extractdir = extractdir
     286          if self.extractdir[-1] != '/':
     287              self.extractdir += '/'
     288
     289 -    def downloadExtract(self, url):
     290          """
     291          Download and extract the list of filenames.
     292          """
     293          try:
     294              localFile = self.download(url)
     295          except Exception:
     296               self.log.error("Problems downloading the file from %s: %s" % (
     297                  url, sys.exc_info()[1] ) )
     298               return []
     299          self.log.debug("Will attempt to load %s", localFile)
     300          try:
     301              return self.processPackage(localFile)
     302          except IOError, ex:
     303              self.log.error("Unable to process '%s' because: %s",
     304                             localFile, str(ex))
     305              sys.exit(1)
     306
     307 -    def download(self, url):
     308          """
     309          Download the package from the given URL, or if it's a filename,
     310          return the filename.
     311          """
     312          urlParts = urlsplit(url)
     313          schema = urlParts[0]
     314          path = urlParts[2]
     315          if not schema:
     316              return os.path.abspath(url)
     317          file = path.split(os.sep)[-1]
     318          os.makedirs(self.downloaddir)
     319          downloadFile = os.path.join(self.downloaddir, file)
     320          self.log.debug("Downloading to file '%s'", downloadFile)
     321          filename, _ = urllib.urlretrieve(url, downloadFile)
     322          return filename
     323
     324 -    def processPackage(self, pkgFileName):
     325          """
     326          Figure out what type of file we have and extract out any
     327          files and then enumerate the file names.
     328          """
     329          self.log.debug("Determining file type of %s" % pkgFileName)
     330          if zipfile.is_zipfile(pkgFileName):
     331              return self.unbundlePackage(pkgFileName, self.unzip)
     332
     333          elif tarfile.is_tarfile(pkgFileName):
     334              return self.unbundlePackage(pkgFileName, self.untar)
     335
     336          elif os.path.isdir(pkgFileName):
     337              return self.processDir(pkgFileName)
     338
     339          else:
     340              return [ os.path.abspath(pkgFileName) ]
     341
     342 -    def unzip(self, file):
     343          """
     344          Unzip the given file into the current directory and return
     345          the directory in which files can be loaded.
     346          """
     347          pkgZip= zipfile.ZipFile(file, 'r')
     348          if pkgZip.testzip() != None:
     349              self.log.error("File %s is corrupted -- please download again", file)
     350              return
     351
     352          for file in pkgZip.namelist():
     353              self.log.debug("Unzipping file/dir %s..." % file)
     354              try:
     355                  if re.search(r'/$', file) != None:
     356                      os.makedirs(file)
     357                  else:
     358                      contents = pkgZip.read(file)
     359                      try:
     360                          unzipped = open(file, "w")
     361                      except IOError: # Directory missing?
     362                          os.makedirs(os.path.dirname(file))
     363                          unzipped = open(file, "w")
     364                      unzipped.write(contents)
     365                      unzipped.close()
     366              except (SystemExit, KeyboardInterrupt): raise
     367              except:
     368                  self.log.error("Error in extracting %s because %s" % (
     369                      file, sys.exc_info()[1] ) )
     370                  return
     371
     372          return os.getcwd()
     373
     374 -    def untar(self, file):
     375          """
     376          Given a tar file, extract from the tar into the current directory.
     377          """
     378          try:
     379              self.log.debug("Extracting files from tar...")
     380              pkgTar = tarfile.open(file, 'r')
     381              for tarInfo in pkgTar:
     382                  pkgTar.extract(tarInfo)
     383              pkgTar.close()
     384          except (SystemExit, KeyboardInterrupt): raise
     385          except:
     386              self.log.error("Error in un-tarring %s because %s" % ( file,
     387                                                          sys.exc_info()[1] ) )
     388              return
     389          return os.getcwd()
     390
     391 -    def processDir(self, dir):
     392          """
     393          Note all of the files in a directory.
     394          """
     395          fileList = []
     396          self.log.debug("Enumerating files in %s", dir)
     397          if not os.path.isdir(dir):
     398              self.log.debug("%s is not a directory", dir)
     399              return []
     400          for directoryName, _, fileNames in os.walk(dir):
     401              for fileName in fileNames:
     402                  fileList.append(os.path.join(directoryName, fileName))
     403          return fileList
     404
     405 -    def unbundlePackage(self, package, unpackageMethod):
     406          """
     407          Extract the files and then add to the list of files.
     408          """
     409          self.makeExtractionDir()
     410          baseDir = unpackageMethod(package)
     411          if baseDir is not None:
     412              return self.processDir(baseDir)
     413          return []
     414
     415 -    def makeExtractionDir(self):
     416          """
     417          Create an uniquely named extraction directory starting from a base
     418          extraction directory.
     419          """
     420          try:
     421              if not os.path.isdir(self.extractdir):
     422                  os.makedirs(self.extractdir)
     423              extractDir = tempfile.mkdtemp(prefix=self.extractdir)
     424          except (SystemExit, KeyboardInterrupt): raise
     425          except:
     426              self.log.error("Error in creating temp dir because %s",
     427                                                      sys.exc_info()[1] )
     428              sys.exit(1)
     429          os.chdir(extractDir)
     430
     431 -    def cleanup(self):
     432          """
     433          Remove any clutter left over from the installation.
     434          """
     435          self.cleanupDir(self.downloaddir)
     436          self.cleanupDir(self.extractdir)
     437
     438 -    def cleanupDir(self, dirName):
     439          for root, dirs, files in os.walk(dirName, topdown=False):
     440              if root == os.sep: # Should *never* get here
     441                  break
     442              for name in files:
     443                  os.remove(os.path.join(root, name))
     444              for name in dirs:
     445                  try:
     446                      os.removedirs(os.path.join(root, name))
     447                  except OSError:
     448                      pass
     449          try:
     450              os.removedirs(dirName)
     451          except OSError:
     452              pass
     453
     454
     455 -class ZenMib(ZCmdBase):
     456      """
     457      Wrapper around the smidump utilities used to convert MIB definitions into
     458      python code which is in turn loaded into the DMD tree.
     459      """
     460 -    def makeMibFileObj(self, fileName):
     461          """
     462          Scan the MIB file to determine what MIBs are defined in the file and
     463          what their dependencies are.
     464
     465          @param fileName: MIB fileName
     466          @type fileName: string
     467          @return: dependencyDict, indexDict
     468              dependencyDict - a dictionary that associates MIB definitions
     469                  found in fileName with their dependencies
     470              indexDict - a dictionary that associates MIB definitions with their
     471                  ordinal position within fileName}
     472          @rtype:
     473              dependencyDict = {mibName: [string list of MIB definition names
     474                  that mibName is dependant on]}
     475              indexDict = {mibname: number}
     476          """
     477          # Retrieve the entire contents of the MIB file
     478          self.log.debug("Processing %s", fileName)
     479          file = open(fileName)
     480          fileContents = file.read()
     481          file.close()
     482          return MibFile(fileName, fileContents)
     483
     484 -    def populateDependencyMap(self, importFileNames, depFileNames):
     485          """
     486          Populates the self.mibToMibFile instance object with data.
     487          Exit the program if we're missing any files.
     488
     489          @param importFileNames: fully qualified file names of MIB files to import
     490          @type importFileNames: list of strings
     491          @param depFileNames: fully qualified file names of all MIB files
     492          @type depFileNames: list of strings
     493          @return: mibFileObjects of files to import
     494          @rtype: MibFile
     495          """
     496          self.log.debug("Collecting MIB meta-data and creating depedency map.")
     497          toImportMibFileObjs = []
     498          for fileName in depFileNames.union(importFileNames):
     499              try:
     500                  mibFileObj = self.makeMibFileObj(fileName)
     501              except IOError:
     502                  self.log.error("Couldn't open file %s", fileName)
     503                  continue
     504
     505              mibDependencies = mibFileObj.mibToDeps
     506              if not mibDependencies:
     507                  self.log.warn("Unable to parse information from "
     508                      "%s -- skipping", fileName)
     509                  continue
     510
     511              if fileName in importFileNames:
     512                  toImportMibFileObjs.append(mibFileObj)
     513
     514              for mibName, dependencies in mibDependencies.items():
     515                  self.mibToMibFile[mibName] = mibFileObj
     516          return toImportMibFileObjs
     517
     518 -    def getDependencyFileNames(self, mibFileObj):
     519          """
     520          smidump needs to know the list of dependent files for a MIB file in
     521          order to properly resolve external references.
     522
     523          @param mibFileObj: MibFile object
     524          @type mibFileObj: MibFile
     525          @return: list of dependency fileNames
     526          @rtype: list of strings
     527          """
     528          dependencies = []
     529          dependencyFileNames = OrderedDict()
     530
     531          def dependencySearch(mibName):
     532              """
     533              Create a list of files required by a MIB definition.
     534
     535              @param mibName: name of MIB definition
     536              @type mibName: string
     537              """
     538              dependencies.append(mibName)
     539              mibFileObj = self.mibToMibFile.get(mibName)
     540              if not mibFileObj:
     541                  self.log.warn("Unable to find a file that defines %s", mibName)
     542                  return
     543
     544              dependencyFileNames[mibFileObj.fileName] = None
     545              for dependency in mibFileObj.mibToDeps[mibName]:
     546                  if dependency not in dependencies:
     547                      dependencySearch(dependency)
     548
     549          for mibName in mibFileObj.mibs:
     550              dependencySearch(mibName)
     551
     552          if mibFileObj.fileName in dependencyFileNames:
     553              del dependencyFileNames[mibFileObj.fileName]
     554          names = dependencyFileNames.keys()
     555          names.reverse()
     556          return names
     557
     558 -    def savePythonCode(self, pythonCode, fileName):
     559          """
     560          Stores the smidump-generated Python code to a file.
     561          """
     562          if not os.path.exists(self.options.pythoncodedir):
     563              self.options.keeppythoncode = False
     564              self.log.warn('The directory %s to store converted MIB file code '
     565                  'does not exist.' % self.options.pythoncodedir)
     566              return
     567          try:
     568              pythonFileName = os.path.join(self.options.pythoncodedir,
     569              os.path.basename(fileName) ) + '.py'
     570              pythonFile = open(pythonFileName, 'w')
     571              pythonFile.write(pythonCode)
     572              pythonFile.close()
     573          except (SystemExit, KeyboardInterrupt): raise
     574          except:
     575              self.log.warn('Could not output converted MIB to %s' %
     576                  pythonFileName)
     577
     578 -    def generatePythonFromMib(self, fileName, dependencyFileNames,
     579              mibNamesInFile):
     580          """
     581          Use the smidump program to convert a MIB into Python code.
     582
     583          One major caveat: smidump by default only outputs the last MIB
     584          definition in a file. For that matter, it always outputs the last MIB
     585          definition in a file whether it is requested or not. Therefore, if
     586          there are multiple MIB definitions in a file, all but the last must be
     587          explicitly named on the command line. If you name the last, it will
     588          come out twice. We don't want that.
     589
     590          OK, so we need to determine if there are multiple MIB definitions
     591          in fileName and then add all but the last to the command line. That
     592          works except the resulting python code will create a dictionary
     593          for each MIB definition, all of them named MIB. Executing the code is
     594          equivalent to running a=1; a=2; a=3. You only wind up with a=3.
     595          Therefore, we separate each dictionary definition into its own string
     596          and return a list of strings so each one can be executed individually.
     597
     598          @param fileName: name of the file containing MIB definitions
     599          @type fileName: string
     600          @param dependencyFileNames: list of fileNames that fileName is
     601              dependent on
     602          @type dependencyFileNames: list of strings
     603          @param mibNamesInFile: names of MIB definitions in file
     604          @type mibNamesInFile: list of strings
     605          @return: list of dictionaries. Each dictionary containing the contents
     606              of a MIB definition. [ {'mibName': MIB data} ]
     607          @rtype: list
     608          """
     609          dumpCommand =  ['smidump', '--keep-going', '--format', 'python']
     610          for dependencyFileName in dependencyFileNames:
     611              #  Add command-line flag for our dependent files
     612              dumpCommand.append('--preload')
     613              dumpCommand.append(dependencyFileName)
     614          dumpCommand.append(fileName)
     615
     616          # If more than one MIB definition exists in the file, name all but the
     617          # last on the command line. (See method description for reasons.)
     618          if len(mibNamesInFile) > 1:
     619              dumpCommand += mibNamesInFile[:-1]
     620
     621          self.log.debug('Running %s', ' '.join(dumpCommand))
     622          sys.stdout.flush()
     623          proc = Popen(dumpCommand, stdout=PIPE, stderr=PIPE, close_fds=True)
     624
     625          # If the child process generates enough output to the PIPE,
     626          # then proc.wait() blocks waiting for the OS pipe buffer to accept more data
     627          # (ie forever).  Poll from time to time to avoid the problem with large MIBs.
     628          pythonCode = ''
     629          warnings = ''
     630          while proc.poll() is None:
     631              output, err = proc.communicate()
     632              pythonCode += output
     633              warnings += err
     634
     635          if proc.poll() != 0 or proc.returncode:
     636              if warnings.strip():
     637                  self.log.error(warnings)
     638              return None
     639
     640          if warnings:
     641              self.log.debug("Found warnings while trying to import MIB:\n%s" \
     642                   % warnings)
     643
     644          if self.options.keeppythoncode:
     645              self.savePythonCode(pythonCode, fileName)
     646
     647          return self.evalPythonToMibs(pythonCode, fileName)
     648
     649 -    def evalPythonToMibs(self, pythonCode, name):
     650          """
     651          Evaluate the code and return an array of MIB dictionaries.
     652          """
     653          def executePythonCode(pythonCode, name):
     654              """
     655              Executes the python code generated smidump
     656
     657              @param pythonCode: Code generated by smidump
     658              @type pythonCode: string
     659              @return: a dictionary which contains one key: MIB
     660              @rtype: dictionary
     661              """
     662              result = {}
     663              try:
     664                  exec pythonCode in result
     665              except (SystemExit, KeyboardInterrupt): raise
     666              except:
     667                  self.log.exception("Unable to import Pythonized-MIB: %s",
     668                      name)
     669              return result.get('MIB', None)
     670
     671          # If more than one MIB definition exists in a file, pythonCode will
     672          # contain a 'MIB = {...}' section for each MIB definition. We must
     673          # split each section into its own string and return a string list.
     674          smiMibDelim = 'MIB = {'
     675          mibCodeParts = pythonCode.split(smiMibDelim)
     676          mibDicts = []
     677          if len(mibCodeParts) > 1:
     678              for mibCodePart in mibCodeParts[1:]:
     679                  mibDict = executePythonCode(smiMibDelim + mibCodePart, name)
     680                  if mibDict is not None:
     681                      mibDicts.append(mibDict)
     682          else:
     683              mibDict = executePythonCode(pythonCode, name)
     684              if mibDict is not None:
     685                  mibDicts = [mibDict]
     686
     687          return mibDicts
     688
     689 -    def evalAddSavedPythonCode(self):
     690          """
     691          Read the file named in the command-line option, evaluate it, and add
     692          it to our list of MIBs.
     693          """
     694          pythonMibs = []
     695          for fileName in set(self.options.evalSavedPython):
     696              try:
     697                  pythonCode = open(fileName).read()
     698              except IOError:
     699                  self.log.warn("Unable to open '%s' -- skipping",
     700                                fileName)
     701                  continue
     702              pythonMibs += self.evalPythonToMibs(pythonCode, fileName)
     703
     704          self.loadPythonMibs(pythonMibs)
     705
     706
     707 -    def getDmdMibDict(self, dmdMibDict, mibOrganizer):
     708          """
     709          Populate a dictionary containing the MIB definitions that have been
     710          loaded into the DMD Mibs directory
     711
     712          @param dmdMibDict: maps a MIB definition to the path where
     713                  it is located with in the DMD.
     714              Format:
     715                  {'mibName': 'DMD path where mibName is stored'}
     716              Example: MIB-Dell-10892 is located in the DMD tree at
     717                  Mibs/SITE/Dell, Directory entry is
     718                  {'MIB-Dell-10892': '/SITE/Dell'] }
     719          @param mibOrganizer: the DMD directory to be searched
     720          @type mibOrganizer: MibOrganizer
     721          """
     722          organizerPath = mibOrganizer.getOrganizerName()
     723
     724          # Record each module from this organizer as a dictionary entry.
     725          # mibOrganizer.mibs.objectItems() returns tuple:
     726          # ('mibName', <MibModule at /zport/dmd/Mibs/...>)
     727          for mibModule in mibOrganizer.mibs.objectItems():
     728              mibName = mibModule[0]
     729              if mibName not in dmdMibDict:
     730                  dmdMibDict[mibName] = organizerPath
     731              else:
     732                  self.log.warn('\nFound two copies of %s:'
     733                      '  %s and %s' %
     734                      (mibName, dmdMibDict[mibName],
     735                      mibOrganizer.getOrganizerName()))
     736
     737          # If there are suborganizers, recurse into them
     738          for childOrganizer in mibOrganizer.children():
     739              self.getDmdMibDict(dmdMibDict, childOrganizer)
     740
     741 -    def addMibEntries(self, leafType, pythonMib, mibModule):
     742          """
     743          Add the different MIB leaves (ie nodes, notifications) into the DMD.
     744
     745          @paramater leafType: 'nodes', 'notifications'
     746          @type leafType: string
     747          @paramater pythonMib: dictionary of nodes and notifications
     748          @type pythonMib: dictionary
     749          @paramater mibModule: class containing functions to load leaves
     750          @type mibModule: class
     751          @return: number of leaves added
     752          @rtype: int
     753          """
     754          entriesAdded = 0
     755          functor = { 'nodes':mibModule.createMibNode,
     756                      'notifications':mibModule.createMibNotification,
     757                     }.get(leafType, None)
     758          if not functor or leafType not in pythonMib:
     759              return entriesAdded
     760
     761          mibName = pythonMib['moduleName']
     762
     763          for name, values in pythonMib[leafType].items():
     764              try:
     765                  functor(name, **values)
     766                  entriesAdded += 1
     767              except BadRequest:
     768                  try:
     769                      self.log.warn("Unable to add %s id '%s' as this"
     770                                  " name is reserved for use by Zope",
     771                                  leafType, name)
     772                      newName = '_'.join([name, mibName])
     773                      self.log.warn("Trying to add %s '%s' as '%s'",
     774                                  leafType, name, newName)
     775                      functor(newName, **values)
     776                      entriesAdded += 1
     777                      self.log.warn("Renamed '%s' to '%s' and added to"
     778                                  " MIB %s", name, newName, leafType)
     779                  except (SystemExit, KeyboardInterrupt): raise
     780                  except:
     781                      self.log.warn("Unable to add %s id '%s' -- skipping",
     782                                  leafType, name)
     783              else:
     784                  if not entriesAdded % CHUNK_SIZE:
     785                      self.commit("Loaded MIB %s into the DMD" % mibName)
     786          self.commit("Loaded MIB %s into the DMD" % mibName)
     787          return entriesAdded
     788
     789 -    def loadMibFile(self, mibFileObj, dmdMibDict):
     790          """
     791          Attempt to load the MIB definitions in fileName into DMD
     792
     793          @param fileName: name of the MIB file to be loaded
     794          @type fileName: string
     795          @return: whether the MIB load was successful or not
     796          @rtype: boolean
     797          """
     798          fileName = mibFileObj.fileName
     799          self.log.debug('Attempting to load %s' % fileName)
     800
     801          # Check to see if any MIB definitions in fileName have already
     802          # been loaded into Zenoss. If so, warn but don't fail
     803          mibNamesInFile = mibFileObj.mibs
     804          for mibName in mibNamesInFile:
     805              if mibName in dmdMibDict:
     806                  dmdMibPath = dmdMibDict[mibName]
     807                  self.log.warn('MIB definition %s found in %s is already '
     808                      'loaded at %s.' % (mibName, fileName, dmdMibPath))
     809
     810          # Retrieve a list of all the files containing MIB definitions that are
     811          # required by the MIB definitions in fileName
     812          dependencyFileNames = self.getDependencyFileNames(mibFileObj)
     813
     814          # Convert the MIB file data into python dictionaries. pythonMibs
     815          # contains a list of dictionaries, one for each MIB definition in
     816          # fileName.
     817          pythonMibs = self.generatePythonFromMib(fileName, dependencyFileNames,
     818              mibNamesInFile)
     819          if not pythonMibs:
     820              return False
     821
     822          self.loadPythonMibs(pythonMibs)
     823          return True
     824
     825 -    def commit(self, message):
     826          if not self.options.nocommit:
     827              self.log.debug('Committing a batch of objects')
     828              trans = transaction.get()
     829              trans.setUser('zenmib')
     830              trans.note(message)
     831              trans.commit()
     832              self.syncdb()
     833
     834 -    def loadPythonMibs(self, pythonMibs):
     835          """
     836          Walk through the MIB dictionaries and add the MIBs to the DMD.
     837          """
     838          # Standard MIB attributes that we expect in all MIBs
     839          MIB_MOD_ATTS = ('language', 'contact', 'description')
     840
     841          self.syncdb()
     842
     843          # Add the MIB data for each MIB into Zenoss
     844          for pythonMib in pythonMibs:
     845              mibName = pythonMib['moduleName']
     846
     847              # Create the container for the MIBs and define meta-data.
     848              # In the DMD this creates another container class which
     849              # contains mibnodes.  These data types are found in
     850              # Products.ZenModel.MibModule and Products.ZenModel.MibNode
     851              mibModule = self.dmd.Mibs.createMibModule(
     852                  mibName, self.options.path)
     853
     854              def gen():
     855                  for key, val in pythonMib[mibName].iteritems():
     856                      if key in MIB_MOD_ATTS:
     857                          yield key, val
     858
     859              for i, attr in enumerate(gen()):
     860                  setattr(mibModule, *attr)
     861                  if not i % CHUNK_SIZE:
     862                      self.commit("Loaded MIB %s into the DMD" % mibName)
     863              self.commit("Loaded MIB %s into the DMD" % mibName)
     864
     865              nodesAdded = self.addMibEntries('nodes', pythonMib, mibModule)
     866              trapsAdded = self.addMibEntries('notifications', pythonMib, mibModule)
     867              self.log.info("Parsed %d nodes and %d notifications from %s",
     868                            nodesAdded, trapsAdded, mibName)
     869
     870              # Add the MIB tree permanently to the DMD unless --nocommit flag.
     871              msg = "Loaded MIB %s into the DMD" % mibName
     872              self.commit(msg)
     873              if not self.options.nocommit:
     874                  self.log.info(msg)
     875
     876
     877 -    def getAllMibDepFileNames(self):
     878          """
     879          Use command line parameters to create a list of files containing MIB
     880          definitions that will be used as a reference list for the files being
     881          loaded into the DMD
     882
     883          @return: set of file names
     884          @rtype: set
     885          """
     886          defaultMibDepDirs = [ 'ietf', 'iana', 'irtf', 'tubs', 'site' ]
     887          mibDepFileNames = set()
     888          for subdir in defaultMibDepDirs:
     889              depDir = os.path.join(self.options.mibdepsdir, subdir)
     890              mibDepFileNames.update(self.pkgMgr.processDir(depDir))
     891          return mibDepFileNames
     892
     893 -    def getMibsToImport(self):
     894          """
     895          Uses command-line parameters to create a list of files containing MIB
     896          definitions that are to be loaded into the DMD
     897
     898          @return: list of file names that are to be loaded into the DMD
     899          @rtype: list
     900          """
     901          loadFileNames = []
     902          if self.args:
     903              for fileName in self.args:
     904                  loadFileNames.extend(self.pkgMgr.downloadExtract(fileName))
     905          else:
     906              loadFileNames = self.pkgMgr.processDir(self.options.mibsdir)
     907
     908          if loadFileNames:
     909              self.log.debug("Will attempt to load the following files: %s",
     910                         loadFileNames)
     911          else:
     912              self.log.error("No MIB files to load!")
     913              sys.exit(1)
     914
     915          return set(loadFileNames)
     916
     917 -    def main(self):
     918          """
     919          Main loop of the program
     920          """
     921          if self.options.evalSavedPython:
     922              self.evalAddSavedPythonCode()
     923              return
     924
     925          # Verify MIBs search directory is valid. Fail if not
     926          if not os.path.exists(self.options.mibsdir):
     927              self.log.error("The directory %s doesn't exist!" %
     928                  self.options.mibsdir )
     929              sys.exit(1)
     930
     931          self.pkgMgr = PackageManager(self.log, self.options.downloaddir,
     932                                       self.options.extractdir)
     933          self.mibToMibFile = {}
     934
     935          requestedFiles = self.getMibsToImport()
     936          mibDepFileNames = self.getAllMibDepFileNames()
     937          mibFileObjs = self.populateDependencyMap(requestedFiles, mibDepFileNames)
     938
     939          # dmdMibDict = {'mibName': 'DMD path to MIB'}
     940          dmdMibDict = {}
     941          self.getDmdMibDict(dmdMibDict, self.dmd.Mibs)
     942
     943          # Load the MIB files
     944          self.log.info("Found %d MIBs to import.", len(mibFileObjs))
     945          loadedMibFiles = 0
     946          for mibFileObj in mibFileObjs:
     947              try:
     948                  if self.loadMibFile(mibFileObj, dmdMibDict):
     949                      loadedMibFiles += 1
     950              except Exception:
     951                  self.log.exception("Failed to load MIB: %s", mibFileObj.fileName)
     952
     953          action = "Loaded"
     954          if self.options.nocommit:
     955              action = "Processed"
     956
     957          self.log.info("%s %d MIB file(s)" % (action, loadedMibFiles))
     958          self.pkgMgr.cleanup()
     959
     960          sys.exit(0)
     961
     962 -    def buildOptions(self):
     963          """
     964          Command-line options
     965          """
     966          ZCmdBase.buildOptions(self)
     967          self.parser.add_option('--mibsdir',
     968                                 dest='mibsdir', default=zenPath('share/mibs/site'),
     969                         help="Directory of input MIB files [ default: %default ]")
     970          self.parser.add_option('--mibdepsdir',
     971                          dest='mibdepsdir', default=zenPath('share/mibs'),
     972                         help="Directory of input MIB files [ default: %default ]")
     973          self.parser.add_option('--path',
     974                                 dest='path', default="/",
     975                                 help="Path to load MIB into the DMD")
     976          self.parser.add_option('--nocommit', action='store_true',
     977                                 dest='nocommit', default=False,
     978                             help="Don't commit the MIB to the DMD after loading")
     979          self.parser.add_option('--keeppythoncode', action='store_true',
     980                                 dest='keeppythoncode', default=False,
     981                             help="Don't commit the MIB to the DMD after loading")
     982          self.parser.add_option('--pythoncodedir', dest='pythoncodedir',
     983              default=tempfile.gettempdir() + "/mib_pythoncode/",
     984              help="This is the directory where the converted MIB will be output. " \
     985                  "[ default: %default ]")
     986          self.parser.add_option('--downloaddir', dest='downloaddir',
     987              default=tempfile.gettempdir() + "/mib_downloads/",
     988              help="This is the directory where the MIB will be downloaded. " \
     989                  "[ default: %default ]")
     990          self.parser.add_option('--extractdir', dest='extractdir',
     991              default=tempfile.gettempdir() + "/mib_extract/",
     992              help="This is the directory where unzipped MIB files will be stored. " \
     993                  "[ default: %default ]")
     994          self.parser.add_option('--evalSavedPython', dest='evalSavedPython',
     995                             default=[], action='append',
     996                             help="Execute the Python code previously generated" \
     997                                  " and saved.")
     998
     999
    1000  if __name__ == '__main__':
    1001      zm = ZenMib()
    1002      zm.main()
    1003