Package Products :: Package ZenModel :: Module zenmib |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.zenmib

       1  ###########################################################################
       2  #
       3  # This program is part of Zenoss Core, an open source monitoring platform.
       4  # Copyright (C) 2007, 2009 Zenoss Inc.
       5  #
       6  # This program is free software; you can redistribute it and/or modify it
       7  # under the terms of the GNU General Public License version 2 or (at your
       8  # option) any later version as published by the Free Software Foundation.
       9  #
      10  # For complete information please visit: http://www.zenoss.com/oss/
      11  #
      12  ###########################################################################
      13  __doc__ = """zenmib
      14     The zenmib program converts MIBs into python data structures and then
      15  (by default) adds the data to the Zenoss DMD.  Essentially, zenmib is a
      16  wrapper program around the smidump program, whose output (python code) is
      17  then executed "inside" the Zope database.
      18
      19   Overview of terms:
      20     SNMP Simple Network Management Protocol
      21        A network protocol originally based on UDP which allows a management
      22        application (ie SNMP manager) to contact and get information from a
      23        device (ie router, computer, network-capable toaster).  The software
      24        on the device that is capable of understanding SNMP and responding
      25        appropriately is called an SNMP agent.
      26
      27     MIB Management of Information Base
      28        A description of what a particular SNMP agent provides and what
      29        traps it sends.  A MIB is a part of a tree structure based on a root
      30        MIB.  Since a MIB is a rooted tree, it allows for delegation of areas
      31        under the tree to different organizations.
      32
      33     ASN Abstract Syntax Notation
      34        The notation used to construct a MIB.
      35
      36     OID Object IDentifier
      37        A MIB is constructed of unique identifiers
      38
      39
      40   Background information:
      41     http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol
      42         Overview of SNMP.
      43
      44     http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en
      45         The libsmi project is the creator of smidump.  There are several
      46         interesting sub-projects available.
      47
      48     http://net-snmp.sourceforge.net/
      49         Homepage for Net-SNMP which is used by Zenoss for SNMP management.
      50
      51     http://www.et.put.poznan.pl/snmp/asn1/asn1.html
      52         An overview of Abstract Syntax Notation (ASN), the language in
      53         which MIBs are written.
      54  """
      55
      56  import os
      57  import os.path
      58  import sys
      59  import re
      60  from subprocess import Popen, PIPE
      61  import tempfile
      62  import urllib
      63  import tarfile
      64  import zipfile
      65  from urlparse import urlsplit
      66
      67  import Globals
      68  import transaction
      69
      70  from Products.ZenUtils.ZCmdBase import ZCmdBase
      71  from Products.ZenUtils.Utils import zenPath
      72  from zExceptions import BadRequest
      73
      74
      75  CHUNK_SIZE = 50
      76
      77
      78 -class MibFile:
      79      """
      80      A MIB file has the meta-data for a MIB inside of it.
      81      """
      82 -    def __init__(self, fileName, fileContents=""):
      83          self.fileName = fileName
      84          self.mibs = [] # Order of MIBs defined in the file
      85          self.mibToDeps = {} # Dependency defintions for each MIB
      86          self.fileContents = self.removeMibComments(fileContents)
      87          self.mibDefinitions = self.splitFileToMIBs(self.fileContents)
      88          self.mapMibToDependents(self.mibDefinitions)
      89
      90          # Reclaim some memory
      91          self.fileContents = ""
      92          self.mibDefinitions = ""
      93
      94 -    def removeMibComments(self, fileContents):
      95          """
      96          Parses the string provided as an argument and extracts
      97          all of the ASN.1 comments from the string.
      98
      99          Assumes that fileContents contains the contents of a well-formed
     100          (no errors) MIB file.
     101
     102          @param fileContents: entire contents of a MIB file
     103          @type fileContents: string
     104          @return: text without any comments
     105          @rtype: string
     106          """
     107          def findSingleLineCommentEndPos(startPos):
     108              """
     109              Beginning at startPos + 2, searches fileContents for the end of
     110              a single line comment. If comment ends with a newline
     111              character, the newline is not included in the comment.
     112
     113              MIB single line comment rules:
     114              1. Begins with '--'
     115              2. Ends with '--' or newline
     116              3. Any characters between the beginning and end of the comment
     117                  are ignored as part of the comment, including quotes and
     118                  start/end delimiters for block comments ( '/*' and '*/')
     119
     120              @param startPos: character position of the beginning of the single
     121                      line comment within fileContents
     122              @type fileContents: string
     123              @return: startPos, endPos (character position of the last character
     124                      in the comment + 1)
     125              @rtype: tuple (integer, integer)
     126              """
     127              commentEndPosDash = fileContents.find('--', startPos + 2)
     128              commentEndPosNewline = fileContents.find('\n', startPos + 2)
     129              if commentEndPosDash != -1:
     130                  if commentEndPosNewline != -1:
     131                      if commentEndPosDash < commentEndPosNewline:
     132                          endPos = commentEndPosDash + 2
     133                      else:
     134                          endPos = commentEndPosNewline
     135                  else:
     136                      endPos = commentEndPosDash + 2
     137              else:
     138                  if commentEndPosNewline != -1:
     139                      endPos = commentEndPosNewline
     140                  else:
     141                      endPos = len(fileContents)
     142
     143              return startPos, endPos
     144
     145          def findBlockCommentEndPos(searchStartPos):
     146              """
     147              Beginning at startPos + 2, searches fileContents for the end of
     148              a block comment. If block comments are nested, the
     149              function interates into each block comment by calling itself.
     150
     151              MIB block comment rules:
     152              1. Begins with '/*'
     153              2. Ends with '*/'
     154              3. Block comments can be nested
     155              3. Any characters between the beginning and end of the comment
     156                  are ignored as part of the comment, including quotes and
     157                  start/end delimiters for single line comments ('--'). Newlines
     158                  are included as part of the block comment.
     159
     160              @param startPos: character position of the beginning of the block
     161                      comment within fileContents
     162              @type fileContents: string
     163              @return: startPos, endPos (character position of the last character
     164                      in the comment + 1)
     165              @rtype: tuple (integer, integer)
     166              """
     167              # Locate the next start and end markers
     168              nextBlockStartPos = fileContents.find('/*', searchStartPos + 2)
     169              nextBlockEndPos = fileContents.find('*/', searchStartPos + 2)
     170
     171              # If a nested comment exists, find the end
     172              if nextBlockStartPos != -1 and \
     173                      nextBlockStartPos < nextBlockEndPos:
     174                  nestedComment = findBlockCommentEndPos(nextBlockStartPos)
     175                  nextBlockEndPos = fileContents.find('*/', nestedComment[1])
     176
     177              return searchStartPos, nextBlockEndPos + 2
     178
     179          # START removeMibComments
     180          if not fileContents:
     181              return fileContents
     182
     183          # Get rid of any lines that are completely made up of hyphens
     184          fileContents = re.sub(r'[ \t]*-{2}[ \t]*$', '', fileContents)
     185
     186          # commentRanges holds a list of tuples in the form (startPos, endPos)
     187          # that define the beginning and end of comments within fileContents
     188          commentRanges = []
     189          searchStartPos = 0   # character position within fileContents
     190          functions = {'SINGLE': findSingleLineCommentEndPos,
     191                            'BLOCK': findBlockCommentEndPos}
     192
     193          # Parse fileContents, looking for single line comments, block comments
     194          # and string literals
     195          while searchStartPos < len(fileContents):
     196              # Find the beginning of the next occurrance of each item
     197              singleLineStartPos = fileContents.find('--', searchStartPos)
     198              blockStartPos = fileContents.find('/*', searchStartPos)
     199              stringStartPos = fileContents.find('\"', searchStartPos)
     200
     201              nextItemPos = sys.maxint
     202              nextItemType = ''
     203
     204              # Compare the next starting point for each item type.
     205              if singleLineStartPos != -1 and \
     206                      singleLineStartPos < nextItemPos:
     207                  nextItemPos = singleLineStartPos
     208                  nextItemType = 'SINGLE'
     209
     210              if blockStartPos != -1 and \
     211                      blockStartPos < nextItemPos:
     212                  nextItemPos = blockStartPos
     213                  nextItemType = 'BLOCK'
     214
     215              # If the next item type is a string literal, just search for the
     216              # next double quote and continue from there. This works because
     217              # all double quotes (that are not part of a comment) appear in
     218              # pairs. Even double-double quotes (escaped quotes) will work
     219              # with this method since the first double quote will look like a
     220              # string literal close quote and the second double quote will look
     221              # like the beginning of a string literal.
     222              if stringStartPos != -1 and \
     223                      stringStartPos < nextItemPos:
     224                  newSearchStartPos = \
     225                      fileContents.find('\"', stringStartPos + 1) + 1
     226                  if newSearchStartPos > searchStartPos:
     227                      searchStartPos = newSearchStartPos
     228                  else: # Weird error case
     229                      break
     230
     231              # If the next item is a comment, use the functions dictionary
     232              # to call the appropriate function
     233              elif nextItemPos != sys.maxint:
     234                  commentRange = functions[nextItemType](nextItemPos)
     235                  commentRanges.append(commentRange)
     236                  #searchStartPos = commentRange[1]
     237                  if commentRange[1] > 0:
     238                      searchStartPos = commentRange[1]
     239
     240              else: # No other items are found!
     241                  break
     242
     243          startPos = 0
     244          mibParts = []
     245
     246          # Iterate through each comment, adding the non-comment parts
     247          # to mibParts. Finally, return the text without comments.
     248          for commentRange in commentRanges:
     249              mibParts.append(fileContents[startPos:(commentRange[0])])
     250              startPos = commentRange[1]
     251          if startPos != len(fileContents):
     252              mibParts.append(fileContents[startPos:(len(fileContents))])
     253          return ''.join(mibParts)
     254
     255 -    def splitFileToMIBs(self, fileContents):
     256          """
     257          Isolates each MIB definition in fileContents into a separate string
     258
     259          @param fileContents: the complete contents of a MIB file
     260          @type fileContents: string
     261          @return: MIB definition strings
     262          @rtype: list of strings
     263          """
     264          if fileContents is None:
     265              return []
     266
     267          DEFINITIONS = re.compile(r'([A-Za-z-0-9]+\s+DEFINITIONS'
     268              '(\s+EXPLICIT TAGS|\s+IMPLICIT TAGS|\s+AUTOMATIC TAGS|\s*)'
     269              '(\s+EXTENSIBILITY IMPLIED|\s*)\s*::=\s*BEGIN)')
     270
     271          definitionSpans = []
     272          for definitionMatch in DEFINITIONS.finditer(fileContents):
     273              definitionSpans.append(list(definitionMatch.span()))
     274              # If more than one definiton in the file, set the end of the
     275              # last span to the beginning of the current span
     276              if len(definitionSpans) > 1:
     277                  definitionSpans[-2][1] = definitionSpans[-1][0]
     278
     279          # Use the start and end positions to create a string for each
     280          # MIB definition
     281          mibDefinitions = []
     282          if definitionSpans:
     283              # Set the end of the last span to the end of the file
     284              definitionSpans[-1][1] = len(fileContents)
     285              for definitionSpan in definitionSpans:
     286                  mibDefinitions.append(
     287                      fileContents[definitionSpan[0]:definitionSpan[1]])
     288
     289          return mibDefinitions
     290
     291 -    def mapMibToDependents(self, mibDefinitions):
     292          # ASN.1 syntax regular expressions for declaring MIBs
     293          #
     294          # An example from http://www.faqs.org/rfcs/rfc1212.html
     295          #
     296          # RFC1213-MIB DEFINITIONS ::= BEGIN
     297          #
     298          # IMPORTS
     299          #        experimental, OBJECT-TYPE, Counter
     300          #             FROM RFC1155-SMI;
     301          #
     302          #     -- a MIB may or may not have an IMPORTS section
     303          #
     304          # root    OBJECT IDENTIFIER ::= { experimental xx }
     305          #
     306          # END
     307          IMPORTS = re.compile(r'\sIMPORTS\s.+;', re.DOTALL)
     308          DEPENDENCIES = re.compile(
     309              r'\sFROM\s+(?P<dependency>[A-Za-z-0-9]+)')
     310
     311          for definition in mibDefinitions:
     312              mibName = re.split(r'([A-Za-z0-9-]+)', definition)[1]
     313              dependencies = set()
     314              importsMatch = IMPORTS.search(definition)
     315              if importsMatch:
     316                  imports = importsMatch.group()
     317                  for dependencyMatch in DEPENDENCIES.finditer(imports):
     318                      dependencies.add(dependencyMatch.group('dependency'))
     319              self.mibs.append(mibName)
     320              self.mibToDeps[mibName] = dependencies
     321
     322 -class PackageManager:
     323      """
     324      Given an URL, filename or archive (eg zip, tar), extract the files from
     325      the package and return a list of filenames.
     326      """
     327 -    def __init__(self, log, downloaddir, extractdir):
     328          """
     329          Initialize the packagae manager.
     330
     331          @parameter log: logging object
     332          @type log: logging class object
     333          @parameter downloaddir: directory name to store downloads
     334          @type downloaddir: string
     335          @parameter extractdir: directory name to store downloads
     336          @type extractdir: string
     337          """
     338          self.log = log
     339          self.downloaddir = downloaddir
     340          if self.downloaddir[-1] != '/':
     341              self.downloaddir += '/'
     342          self.extractdir = extractdir
     343          if self.extractdir[-1] != '/':
     344              self.extractdir += '/'
     345
     346 -    def downloadExtract(self, url):
     347          """
     348          Download and extract the list of filenames.
     349          """
     350          try:
     351              localFile = self.download(url)
     352          except Exception:
     353               self.log.error("Problems downloading the file from %s: %s" % (
     354                  url, sys.exc_info()[1] ) )
     355               return []
     356          self.log.debug("Will attempt to load %s", localFile)
     357          try:
     358              return self.processPackage(localFile)
     359          except IOError, ex:
     360              self.log.error("Unable to process '%s' because: %s",
     361                             localFile, str(ex))
     362              sys.exit(1)
     363
     364 -    def download(self, url):
     365          """
     366          Download the package from the given URL, or if it's a filename,
     367          return the filename.
     368          """
     369          urlParts = urlsplit(url)
     370          schema = urlParts[0]
     371          path = urlParts[2]
     372          if not schema:
     373              return os.path.abspath(url)
     374          file = path.split(os.sep)[-1]
     375          os.makedirs(self.downloaddir)
     376          downloadFile = os.path.join(self.downloaddir, file)
     377          self.log.debug("Downloading to file '%s'", downloadFile)
     378          filename, _ = urllib.urlretrieve(url, downloadFile)
     379          return filename
     380
     381 -    def processPackage(self, pkgFileName):
     382          """
     383          Figure out what type of file we have and extract out any
     384          files and then enumerate the file names.
     385          """
     386          self.log.debug("Determining file type of %s" % pkgFileName)
     387          if zipfile.is_zipfile(pkgFileName):
     388              return self.unbundlePackage(pkgFileName, self.unzip)
     389
     390          elif tarfile.is_tarfile(pkgFileName):
     391              return self.unbundlePackage(pkgFileName, self.untar)
     392
     393          elif os.path.isdir(pkgFileName):
     394              return self.processDir(pkgFileName)
     395
     396          else:
     397              return [ os.path.abspath(pkgFileName) ]
     398
     399 -    def unzip(self, file):
     400          """
     401          Unzip the given file into the current directory and return
     402          the directory in which files can be loaded.
     403          """
     404          pkgZip= zipfile.ZipFile(file, 'r')
     405          if pkgZip.testzip() != None:
     406              self.log.error("File %s is corrupted -- please download again", file)
     407              return
     408
     409          for file in pkgZip.namelist():
     410              self.log.debug("Unzipping file/dir %s..." % file)
     411              try:
     412                  if re.search(r'/$', file) != None:
     413                      os.makedirs(file)
     414                  else:
     415                      contents = pkgZip.read(file)
     416                      try:
     417                          unzipped = open(file, "w")
     418                      except IOError: # Directory missing?
     419                          os.makedirs(os.path.dirname(file))
     420                          unzipped = open(file, "w")
     421                      unzipped.write(contents)
     422                      unzipped.close()
     423              except (SystemExit, KeyboardInterrupt): raise
     424              except:
     425                  self.log.error("Error in extracting %s because %s" % (
     426                      file, sys.exc_info()[1] ) )
     427                  return
     428
     429          return os.getcwd()
     430
     431 -    def untar(self, file):
     432          """
     433          Given a tar file, extract from the tar into the current directory.
     434          """
     435          try:
     436              self.log.debug("Extracting files from tar...")
     437              pkgTar = tarfile.open(file, 'r')
     438              for tarInfo in pkgTar:
     439                  pkgTar.extract(tarInfo)
     440              pkgTar.close()
     441          except (SystemExit, KeyboardInterrupt): raise
     442          except:
     443              self.log.error("Error in un-tarring %s because %s" % ( file,
     444                                                          sys.exc_info()[1] ) )
     445              return
     446          return os.getcwd()
     447
     448 -    def processDir(self, dir):
     449          """
     450          Note all of the files in a directory.
     451          """
     452          fileList = []
     453          self.log.debug("Enumerating files in %s", dir)
     454          if not os.path.isdir(dir):
     455              self.log.debug("%s is not a directory", dir)
     456              return []
     457          for directoryName, _, fileNames in os.walk(dir):
     458              for fileName in fileNames:
     459                  fileList.append(os.path.join(directoryName, fileName))
     460          return fileList
     461
     462 -    def unbundlePackage(self, package, unpackageMethod):
     463          """
     464          Extract the files and then add to the list of files.
     465          """
     466          self.makeExtractionDir()
     467          baseDir = unpackageMethod(package)
     468          if baseDir is not None:
     469              return self.processDir(baseDir)
     470          return []
     471
     472 -    def makeExtractionDir(self):
     473          """
     474          Create an uniquely named extraction directory starting from a base
     475          extraction directory.
     476          """
     477          try:
     478              if not os.path.isdir(self.extractdir):
     479                  os.makedirs(self.extractdir)
     480              extractDir = tempfile.mkdtemp(prefix=self.extractdir)
     481          except (SystemExit, KeyboardInterrupt): raise
     482          except:
     483              self.log.error("Error in creating temp dir because %s",
     484                                                      sys.exc_info()[1] )
     485              sys.exit(1)
     486          os.chdir(extractDir)
     487
     488 -    def cleanup(self):
     489          """
     490          Remove any clutter left over from the installation.
     491          """
     492          self.cleanupDir(self.downloaddir)
     493          self.cleanupDir(self.extractdir)
     494
     495 -    def cleanupDir(self, dirName):
     496          for root, dirs, files in os.walk(dirName, topdown=False):
     497              if root == os.sep: # Should *never* get here
     498                  break
     499              for name in files:
     500                  os.remove(os.path.join(root, name))
     501              for name in dirs:
     502                  try:
     503                      os.removedirs(os.path.join(root, name))
     504                  except OSError:
     505                      pass
     506          try:
     507              os.removedirs(dirName)
     508          except OSError:
     509              pass
     510
     511
     512 -class ZenMib(ZCmdBase):
     513      """
     514      Wrapper around the smidump utilities used to convert MIB definitions into
     515      python code which is in turn loaded into the DMD tree.
     516      """
     517 -    def makeMibFileObj(self, fileName):
     518          """
     519          Scan the MIB file to determine what MIBs are defined in the file and
     520          what their dependencies are.
     521
     522          @param fileName: MIB fileName
     523          @type fileName: string
     524          @return: dependencyDict, indexDict
     525              dependencyDict - a dictionary that associates MIB definitions
     526                  found in fileName with their dependencies
     527              indexDict - a dictionary that associates MIB definitions with their
     528                  ordinal position within fileName}
     529          @rtype:
     530              dependencyDict = {mibName: [string list of MIB definition names
     531                  that mibName is dependant on]}
     532              indexDict = {mibname: number}
     533          """
     534          # Retrieve the entire contents of the MIB file
     535          self.log.debug("Processing %s", fileName)
     536          file = open(fileName)
     537          fileContents = file.read()
     538          file.close()
     539          return MibFile(fileName, fileContents)
     540
     541 -    def populateDependencyMap(self, importFileNames, depFileNames):
     542          """
     543          Populates the self.mibToMibFile instance object with data.
     544          Exit the program if we're missing any files.
     545
     546          @param importFileNames: fully qualified file names of MIB files to import
     547          @type importFileNames: list of strings
     548          @param depFileNames: fully qualified file names of all MIB files
     549          @type depFileNames: list of strings
     550          @return: mibFileObjects of files to import
     551          @rtype: MibFile
     552          """
     553          self.log.debug("Collecting MIB meta-data and creating depedency map.")
     554          toImportMibFileObjs = []
     555          for fileName in depFileNames.union(importFileNames):
     556              try:
     557                  mibFileObj = self.makeMibFileObj(fileName)
     558              except IOError:
     559                  self.log.error("Couldn't open file %s", fileName)
     560                  continue
     561
     562              mibDependencies = mibFileObj.mibToDeps
     563              if not mibDependencies:
     564                  self.log.warn("Unable to parse information from "
     565                      "%s -- skipping", fileName)
     566                  continue
     567
     568              if fileName in importFileNames:
     569                  toImportMibFileObjs.append(mibFileObj)
     570
     571              for mibName, dependencies in mibDependencies.items():
     572                  self.mibToMibFile[mibName] = mibFileObj
     573          return toImportMibFileObjs
     574
     575 -    def getDependencyFileNames(self, mibFileObj):
     576          """
     577          smidump needs to know the list of dependent files for a MIB file in
     578          order to properly resolve external references.
     579
     580          @param mibFileObj: MibFile object
     581          @type mibFileObj: MibFile
     582          @return: list of dependency fileNames
     583          @rtype: list of strings
     584          """
     585          dependencies = []
     586          dependencyFileNames = set()
     587
     588          def dependencySearch(mibName):
     589              """
     590              Create a list of files required by a MIB definition.
     591
     592              @param mibName: name of MIB definition
     593              @type mibName: string
     594              """
     595              dependencies.append(mibName)
     596              mibFileObj = self.mibToMibFile.get(mibName)
     597              if not mibFileObj:
     598                  self.log.warn("Unable to find a file that defines %s", mibName)
     599                  return
     600
     601              dependencyFileNames.add(mibFileObj.fileName)
     602              for dependency in mibFileObj.mibToDeps[mibName]:
     603                  if dependency not in dependencies:
     604                      dependencySearch(dependency)
     605
     606          for mibName in mibFileObj.mibs:
     607              dependencySearch(mibName)
     608
     609          dependencyFileNames.discard(mibFileObj.fileName)
     610          return dependencyFileNames
     611
     612 -    def savePythonCode(self, pythonCode, fileName):
     613          """
     614          Stores the smidump-generated Python code to a file.
     615          """
     616          if not os.path.exists(self.options.pythoncodedir):
     617              self.options.keeppythoncode = False
     618              self.log.warn('The directory %s to store converted MIB file code '
     619                  'does not exist.' % self.options.pythoncodedir)
     620              return
     621          try:
     622              pythonFileName = os.path.join(self.options.pythoncodedir,
     623              os.path.basename(fileName) ) + '.py'
     624              pythonFile = open(pythonFileName, 'w')
     625              pythonFile.write(pythonCode)
     626              pythonFile.close()
     627          except (SystemExit, KeyboardInterrupt): raise
     628          except:
     629              self.log.warn('Could not output converted MIB to %s' %
     630                  pythonFileName)
     631
     632 -    def generatePythonFromMib(self, fileName, dependencyFileNames,
     633              mibNamesInFile):
     634          """
     635          Use the smidump program to convert a MIB into Python code.
     636
     637          One major caveat: smidump by default only outputs the last MIB
     638          definition in a file. For that matter, it always outputs the last MIB
     639          definition in a file whether it is requested or not. Therefore, if
     640          there are multiple MIB definitions in a file, all but the last must be
     641          explicitly named on the command line. If you name the last, it will
     642          come out twice. We don't want that.
     643
     644          OK, so we need to determine if there are multiple MIB definitions
     645          in fileName and then add all but the last to the command line. That
     646          works except the resulting python code will create a dictionary
     647          for each MIB definition, all of them named MIB. Executing the code is
     648          equivalent to running a=1; a=2; a=3. You only wind up with a=3.
     649          Therefore, we separate each dictionary definition into its own string
     650          and return a list of strings so each one can be executed individually.
     651
     652          @param fileName: name of the file containing MIB definitions
     653          @type fileName: string
     654          @param dependencyFileNames: list of fileNames that fileName is
     655              dependent on
     656          @type dependencyFileNames: list of strings
     657          @param mibNamesInFile: names of MIB definitions in file
     658          @type mibNamesInFile: list of strings
     659          @return: list of dictionaries. Each dictionary containing the contents
     660              of a MIB definition. [ {'mibName': MIB data} ]
     661          @rtype: list
     662          """
     663          dumpCommand =  ['smidump', '--keep-going', '--format', 'python']
     664          for dependencyFileName in dependencyFileNames:
     665              #  Add command-line flag for our dependent files
     666              dumpCommand.append('--preload')
     667              dumpCommand.append(dependencyFileName)
     668          dumpCommand.append(fileName)
     669
     670          # If more than one MIB definition exists in the file, name all but the
     671          # last on the command line. (See method description for reasons.)
     672          if len(mibNamesInFile) > 1:
     673              dumpCommand += mibNamesInFile[:-1]
     674
     675          self.log.debug('Running %s', ' '.join(dumpCommand))
     676          sys.stdout.flush()
     677          proc = Popen(dumpCommand, stdout=PIPE, stderr=PIPE, close_fds=True)
     678
     679          # If the child process generates enough output to the PIPE,
     680          # then proc.wait() blocks waiting for the OS pipe buffer to accept more data
     681          # (ie forever).  Poll from time to time to avoid the problem with large MIBs.
     682          pythonCode = ''
     683          warnings = ''
     684          while proc.poll() is None:
     685              output, err = proc.communicate()
     686              pythonCode += output
     687              warnings += err
     688
     689          if proc.poll() != 0 or proc.returncode:
     690              if warnings.strip():
     691                  self.log.error(warnings)
     692              return None
     693
     694          if warnings:
     695              self.log.debug("Found warnings while trying to import MIB:\n%s" \
     696                   % warnings)
     697
     698          if self.options.keeppythoncode:
     699              self.savePythonCode(pythonCode, fileName)
     700
     701          return self.evalPythonToMibs(pythonCode, fileName)
     702
     703 -    def evalPythonToMibs(self, pythonCode, name):
     704          """
     705          Evaluate the code and return an array of MIB dictionaries.
     706          """
     707          def executePythonCode(pythonCode, name):
     708              """
     709              Executes the python code generated smidump
     710
     711              @param pythonCode: Code generated by smidump
     712              @type pythonCode: string
     713              @return: a dictionary which contains one key: MIB
     714              @rtype: dictionary
     715              """
     716              result = {}
     717              try:
     718                  exec pythonCode in result
     719              except (SystemExit, KeyboardInterrupt): raise
     720              except:
     721                  self.log.exception("Unable to import Pythonized-MIB: %s",
     722                      name)
     723              return result.get('MIB', None)
     724
     725          # If more than one MIB definition exists in a file, pythonCode will
     726          # contain a 'MIB = {...}' section for each MIB definition. We must
     727          # split each section into its own string and return a string list.
     728          smiMibDelim = 'MIB = {'
     729          mibCodeParts = pythonCode.split(smiMibDelim)
     730          mibDicts = []
     731          if len(mibCodeParts) > 1:
     732              for mibCodePart in mibCodeParts[1:]:
     733                  mibDict = executePythonCode(smiMibDelim + mibCodePart, name)
     734                  if mibDict is not None:
     735                      mibDicts.append(mibDict)
     736          else:
     737              mibDict = executePythonCode(pythonCode, name)
     738              if mibDict is not None:
     739                  mibDicts = [mibDict]
     740
     741          return mibDicts
     742
     743 -    def evalAddSavedPythonCode(self):
     744          """
     745          Read the file named in the command-line option, evaluate it, and add
     746          it to our list of MIBs.
     747          """
     748          pythonMibs = []
     749          for fileName in set(self.options.evalSavedPython):
     750              try:
     751                  pythonCode = open(fileName).read()
     752              except IOError:
     753                  self.log.warn("Unable to open '%s' -- skipping",
     754                                fileName)
     755                  continue
     756              pythonMibs += self.evalPythonToMibs(pythonCode, fileName)
     757
     758          self.loadPythonMibs(pythonMibs)
     759
     760
     761 -    def getDmdMibDict(self, dmdMibDict, mibOrganizer):
     762          """
     763          Populate a dictionary containing the MIB definitions that have been
     764          loaded into the DMD Mibs directory
     765
     766          @param dmdMibDict: maps a MIB definition to the path where
     767                  it is located with in the DMD.
     768              Format:
     769                  {'mibName': 'DMD path where mibName is stored'}
     770              Example: MIB-Dell-10892 is located in the DMD tree at
     771                  Mibs/SITE/Dell, Directory entry is
     772                  {'MIB-Dell-10892': '/SITE/Dell'] }
     773          @param mibOrganizer: the DMD directory to be searched
     774          @type mibOrganizer: MibOrganizer
     775          """
     776          organizerPath = mibOrganizer.getOrganizerName()
     777
     778          # Record each module from this organizer as a dictionary entry.
     779          # mibOrganizer.mibs.objectItems() returns tuple:
     780          # ('mibName', <MibModule at /zport/dmd/Mibs/...>)
     781          for mibModule in mibOrganizer.mibs.objectItems():
     782              mibName = mibModule[0]
     783              if mibName not in dmdMibDict:
     784                  dmdMibDict[mibName] = organizerPath
     785              else:
     786                  self.log.warn('\nFound two copies of %s:'
     787                      '  %s and %s' %
     788                      (mibName, dmdMibDict[mibName],
     789                      mibOrganizer.getOrganizerName()))
     790
     791          # If there are suborganizers, recurse into them
     792          for childOrganizer in mibOrganizer.children():
     793              self.getDmdMibDict(dmdMibDict, childOrganizer)
     794
     795 -    def addMibEntries(self, leafType, pythonMib, mibModule):
     796          """
     797          Add the different MIB leaves (ie nodes, notifications) into the DMD.
     798
     799          @paramater leafType: 'nodes', 'notifications'
     800          @type leafType: string
     801          @paramater pythonMib: dictionary of nodes and notifications
     802          @type pythonMib: dictionary
     803          @paramater mibModule: class containing functions to load leaves
     804          @type mibModule: class
     805          @return: number of leaves added
     806          @rtype: int
     807          """
     808          entriesAdded = 0
     809          functor = { 'nodes':mibModule.createMibNode,
     810                      'notifications':mibModule.createMibNotification,
     811                     }.get(leafType, None)
     812          if not functor or leafType not in pythonMib:
     813              return entriesAdded
     814
     815          mibName = pythonMib['moduleName']
     816
     817          for name, values in pythonMib[leafType].items():
     818              try:
     819                  functor(name, **values)
     820                  entriesAdded += 1
     821              except BadRequest:
     822                  try:
     823                      self.log.warn("Unable to add %s id '%s' as this"
     824                                  " name is reserved for use by Zope",
     825                                  leafType, name)
     826                      newName = '_'.join([name, mibName])
     827                      self.log.warn("Trying to add %s '%s' as '%s'",
     828                                  leafType, name, newName)
     829                      functor(newName, **values)
     830                      entriesAdded += 1
     831                      self.log.warn("Renamed '%s' to '%s' and added to"
     832                                  " MIB %s", name, newName, leafType)
     833                  except (SystemExit, KeyboardInterrupt): raise
     834                  except:
     835                      self.log.warn("Unable to add %s id '%s' -- skipping",
     836                                  leafType, name)
     837              else:
     838                  if not entriesAdded % CHUNK_SIZE:
     839                      self.commit("Loaded MIB %s into the DMD" % mibName)
     840          self.commit("Loaded MIB %s into the DMD" % mibName)
     841          return entriesAdded
     842
     843 -    def loadMibFile(self, mibFileObj, dmdMibDict):
     844          """
     845          Attempt to load the MIB definitions in fileName into DMD
     846
     847          @param fileName: name of the MIB file to be loaded
     848          @type fileName: string
     849          @return: whether the MIB load was successful or not
     850          @rtype: boolean
     851          """
     852          fileName = mibFileObj.fileName
     853          self.log.debug('Attempting to load %s' % fileName)
     854
     855          # Check to see if any MIB definitions in fileName have already
     856          # been loaded into Zenoss. If so, warn but don't fail
     857          mibNamesInFile = mibFileObj.mibs
     858          for mibName in mibNamesInFile:
     859              if mibName in dmdMibDict:
     860                  dmdMibPath = dmdMibDict[mibName]
     861                  self.log.warn('MIB definition %s found in %s is already '
     862                      'loaded at %s.' % (mibName, fileName, dmdMibPath))
     863
     864          # Retrieve a list of all the files containing MIB definitions that are
     865          # required by the MIB definitions in fileName
     866          dependencyFileNames = self.getDependencyFileNames(mibFileObj)
     867
     868          # Convert the MIB file data into python dictionaries. pythonMibs
     869          # contains a list of dictionaries, one for each MIB definition in
     870          # fileName.
     871          pythonMibs = self.generatePythonFromMib(fileName, dependencyFileNames,
     872              mibNamesInFile)
     873          if not pythonMibs:
     874              return False
     875
     876          self.loadPythonMibs(pythonMibs)
     877          return True
     878
     879 -    def commit(self, message):
     880          if not self.options.nocommit:
     881              self.log.debug('Committing a batch of objects')
     882              trans = transaction.get()
     883              trans.setUser('zenmib')
     884              trans.note(message)
     885              trans.commit()
     886              self.syncdb()
     887
     888 -    def loadPythonMibs(self, pythonMibs):
     889          """
     890          Walk through the MIB dictionaries and add the MIBs to the DMD.
     891          """
     892          # Standard MIB attributes that we expect in all MIBs
     893          MIB_MOD_ATTS = ('language', 'contact', 'description')
     894
     895          self.syncdb()
     896
     897          # Add the MIB data for each MIB into Zenoss
     898          for pythonMib in pythonMibs:
     899              mibName = pythonMib['moduleName']
     900
     901              # Create the container for the MIBs and define meta-data.
     902              # In the DMD this creates another container class which
     903              # contains mibnodes.  These data types are found in
     904              # Products.ZenModel.MibModule and Products.ZenModel.MibNode
     905              mibModule = self.dmd.Mibs.createMibModule(
     906                  mibName, self.options.path)
     907
     908              def gen():
     909                  for key, val in pythonMib[mibName].iteritems():
     910                      if key in MIB_MOD_ATTS:
     911                          yield key, val
     912
     913              for i, attr in enumerate(gen()):
     914                  setattr(mibModule, *attr)
     915                  if not i % CHUNK_SIZE:
     916                      self.commit("Loaded MIB %s into the DMD" % mibName)
     917              self.commit("Loaded MIB %s into the DMD" % mibName)
     918
     919              nodesAdded = self.addMibEntries('nodes', pythonMib, mibModule)
     920              trapsAdded = self.addMibEntries('notifications', pythonMib, mibModule)
     921              self.log.info("Parsed %d nodes and %d notifications from %s",
     922                            nodesAdded, trapsAdded, mibName)
     923
     924              # Add the MIB tree permanently to the DMD unless --nocommit flag.
     925              msg = "Loaded MIB %s into the DMD" % mibName
     926              self.commit(msg)
     927              if not self.options.nocommit:
     928                  self.log.info(msg)
     929
     930
     931 -    def getAllMibDepFileNames(self):
     932          """
     933          Use command line parameters to create a list of files containing MIB
     934          definitions that will be used as a reference list for the files being
     935          loaded into the DMD
     936
     937          @return: set of file names
     938          @rtype: set
     939          """
     940          defaultMibDepDirs = [ 'ietf', 'iana', 'irtf', 'tubs', 'site' ]
     941          mibDepFileNames = set()
     942          for subdir in defaultMibDepDirs:
     943              depDir = os.path.join(self.options.mibdepsdir, subdir)
     944              mibDepFileNames.update(self.pkgMgr.processDir(depDir))
     945          return mibDepFileNames
     946
     947 -    def getMibsToImport(self):
     948          """
     949          Uses command-line parameters to create a list of files containing MIB
     950          definitions that are to be loaded into the DMD
     951
     952          @return: list of file names that are to be loaded into the DMD
     953          @rtype: list
     954          """
     955          loadFileNames = []
     956          if self.args:
     957              for fileName in self.args:
     958                  loadFileNames.extend(self.pkgMgr.downloadExtract(fileName))
     959          else:
     960              loadFileNames = self.pkgMgr.processDir(self.options.mibsdir)
     961
     962          if loadFileNames:
     963              self.log.debug("Will attempt to load the following files: %s",
     964                         loadFileNames)
     965          else:
     966              self.log.error("No MIB files to load!")
     967              sys.exit(1)
     968
     969          return set(loadFileNames)
     970
     971 -    def main(self):
     972          """
     973          Main loop of the program
     974          """
     975          if self.options.evalSavedPython:
     976              self.evalAddSavedPythonCode()
     977              return
     978
     979          # Verify MIBs search directory is valid. Fail if not
     980          if not os.path.exists(self.options.mibsdir):
     981              self.log.error("The directory %s doesn't exist!" %
     982                  self.options.mibsdir )
     983              sys.exit(1)
     984
     985          self.pkgMgr = PackageManager(self.log, self.options.downloaddir,
     986                                       self.options.extractdir)
     987          self.mibToMibFile = {}
     988
     989          requestedFiles = self.getMibsToImport()
     990          mibDepFileNames = self.getAllMibDepFileNames()
     991          mibFileObjs = self.populateDependencyMap(requestedFiles, mibDepFileNames)
     992
     993          # dmdMibDict = {'mibName': 'DMD path to MIB'}
     994          dmdMibDict = {}
     995          self.getDmdMibDict(dmdMibDict, self.dmd.Mibs)
     996
     997          # Load the MIB files
     998          self.log.info("Found %d MIBs to import.", len(mibFileObjs))
     999          loadedMibFiles = 0
    1000          for mibFileObj in mibFileObjs:
    1001              try:
    1002                  if self.loadMibFile(mibFileObj, dmdMibDict):
    1003                      loadedMibFiles += 1
    1004              except Exception:
    1005                  self.log.exception("Failed to load MIB: %s", mibFileObj.fileName)
    1006
    1007          action = "Loaded"
    1008          if self.options.nocommit:
    1009              action = "Processed"
    1010
    1011          self.log.info("%s %d MIB file(s)" % (action, loadedMibFiles))
    1012          self.pkgMgr.cleanup()
    1013
    1014          sys.exit(0)
    1015
    1016 -    def buildOptions(self):
    1017          """
    1018          Command-line options
    1019          """
    1020          ZCmdBase.buildOptions(self)
    1021          self.parser.add_option('--mibsdir',
    1022                                 dest='mibsdir', default=zenPath('share/mibs/site'),
    1023                         help="Directory of input MIB files [ default: %default ]")
    1024          self.parser.add_option('--mibdepsdir',
    1025                          dest='mibdepsdir', default=zenPath('share/mibs'),
    1026                         help="Directory of input MIB files [ default: %default ]")
    1027          self.parser.add_option('--path',
    1028                                 dest='path', default="/",
    1029                                 help="Path to load MIB into the DMD")
    1030          self.parser.add_option('--nocommit', action='store_true',
    1031                                 dest='nocommit', default=False,
    1032                             help="Don't commit the MIB to the DMD after loading")
    1033          self.parser.add_option('--keeppythoncode', action='store_true',
    1034                                 dest='keeppythoncode', default=False,
    1035                             help="Don't commit the MIB to the DMD after loading")
    1036          self.parser.add_option('--pythoncodedir', dest='pythoncodedir',
    1037              default=tempfile.gettempdir() + "/mib_pythoncode/",
    1038              help="This is the directory where the converted MIB will be output. " \
    1039                  "[ default: %default ]")
    1040          self.parser.add_option('--downloaddir', dest='downloaddir',
    1041              default=tempfile.gettempdir() + "/mib_downloads/",
    1042              help="This is the directory where the MIB will be downloaded. " \
    1043                  "[ default: %default ]")
    1044          self.parser.add_option('--extractdir', dest='extractdir',
    1045              default=tempfile.gettempdir() + "/mib_extract/",
    1046              help="This is the directory where unzipped MIB files will be stored. " \
    1047                  "[ default: %default ]")
    1048          self.parser.add_option('--evalSavedPython', dest='evalSavedPython',
    1049                             default=[], action='append',
    1050                             help="Execute the Python code previously generated" \
    1051                                  " and saved.")
    1052
    1053
    1054  if __name__ == '__main__':
    1055      zm = ZenMib()
    1056      zm.main()
    1057