Package Products :: Package ZenModel :: Module zenmib |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.zenmib

       1  ###########################################################################
       2  #
       3  # This program is part of Zenoss Core, an open source monitoring platform.
       4  # Copyright (C) 2007, 2009 Zenoss Inc.
       5  #
       6  # This program is free software; you can redistribute it and/or modify it
       7  # under the terms of the GNU General Public License version 2 as published by
       8  # the Free Software Foundation.
       9  #
      10  # For complete information please visit: http://www.zenoss.com/oss/
      11  #
      12  ###########################################################################
      13  __doc__ = """zenmib
      14     The zenmib program converts MIBs into python data structures and then
      15  (by default) adds the data to the Zenoss DMD.  Essentially, zenmib is a
      16  wrapper program around the smidump program, whose output (python code) is
      17  then executed "inside" the Zope database.
      18
      19   Overview of terms:
      20     SNMP Simple Network Management Protocol
      21        A network protocol originally based on UDP which allows a management
      22        application (ie SNMP manager) to contact and get information from a
      23        device (ie router, computer, network-capable toaster).  The software
      24        on the device that is capable of understanding SNMP and responding
      25        appropriately is called an SNMP agent.
      26
      27     MIB Management of Information Base
      28        A description of what a particular SNMP agent provides and what
      29        traps it sends.  A MIB is a part of a tree structure based on a root
      30        MIB.  Since a MIB is a rooted tree, it allows for delegation of areas
      31        under the tree to different organizations.
      32
      33     ASN Abstract Syntax Notation
      34        The notation used to construct a MIB.
      35
      36     OID Object IDentifier
      37        A MIB is constructed of unique identifiers
      38
      39
      40   Background information:
      41     http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol
      42         Overview of SNMP.
      43
      44     http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en
      45         The libsmi project is the creator of smidump.  There are several
      46         interesting sub-projects available.
      47
      48     http://net-snmp.sourceforge.net/
      49         Homepage for Net-SNMP which is used by Zenoss for SNMP management.
      50
      51     http://www.et.put.poznan.pl/snmp/asn1/asn1.html
      52         An overview of Abstract Syntax Notation (ASN), the language in
      53         which MIBs are written.
      54  """
      55
      56  import os
      57  import os.path
      58  import sys
      59  import glob
      60  import re
      61  import logging
      62  from subprocess import Popen, PIPE
      63  import tempfile
      64  import urllib
      65  import tarfile
      66  import zipfile
      67  import signal
      68  from urllib2 import urlopen
      69  from urlparse import urljoin, urlsplit
      70
      71  import Globals
      72  import transaction
      73
      74  from Products.ZenUtils.ZCmdBase import ZCmdBase
      75  from Products.ZenUtils.Utils import zenPath
      76  from zExceptions import BadRequest
      77
      78
      79  CHUNK_SIZE = 50
      80
      81
      82 -class MibFile:
      83      """
      84      A MIB file has the meta-data for a MIB inside of it.
      85      """
      86 -    def __init__(self, fileName, fileContents=""):
      87          self.fileName = fileName
      88          self.mibs = [] # Order of MIBs defined in the file
      89          self.mibToDeps = {} # Dependency defintions for each MIB
      90          self.fileContents = self.removeMibComments(fileContents)
      91          self.mibDefinitions = self.splitFileToMIBs(self.fileContents)
      92          self.mapMibToDependents(self.mibDefinitions)
      93
      94          # Reclaim some memory
      95          self.fileContents = ""
      96          self.mibDefinitions = ""
      97
      98 -    def removeMibComments(self, fileContents):
      99          """
     100          Parses the string provided as an argument and extracts
     101          all of the ASN.1 comments from the string.
     102
     103          Assumes that fileContents contains the contents of a well-formed
     104          (no errors) MIB file.
     105
     106          @param fileContents: entire contents of a MIB file
     107          @type fileContents: string
     108          @return: text without any comments
     109          @rtype: string
     110          """
     111          def findSingleLineCommentEndPos(startPos):
     112              """
     113              Beginning at startPos + 2, searches fileContents for the end of
     114              a single line comment. If comment ends with a newline
     115              character, the newline is not included in the comment.
     116
     117              MIB single line comment rules:
     118              1. Begins with '--'
     119              2. Ends with '--' or newline
     120              3. Any characters between the beginning and end of the comment
     121                  are ignored as part of the comment, including quotes and
     122                  start/end delimiters for block comments ( '/*' and '*/')
     123
     124              @param startPos: character position of the beginning of the single
     125                      line comment within fileContents
     126              @type fileContents: string
     127              @return: startPos, endPos (character position of the last character
     128                      in the comment + 1)
     129              @rtype: tuple (integer, integer)
     130              """
     131              commentEndPosDash = fileContents.find('--', startPos + 2)
     132              commentEndPosNewline = fileContents.find('\n', startPos + 2)
     133              if commentEndPosDash != -1:
     134                  if commentEndPosNewline != -1:
     135                      if commentEndPosDash < commentEndPosNewline:
     136                          endPos = commentEndPosDash + 2
     137                      else:
     138                          endPos = commentEndPosNewline
     139                  else:
     140                      endPos = commentEndPosDash + 2
     141              else:
     142                  if commentEndPosNewline != -1:
     143                      endPos = commentEndPosNewline
     144                  else:
     145                      endPos = len(fileContents)
     146
     147              return startPos, endPos
     148
     149          def findBlockCommentEndPos(searchStartPos):
     150              """
     151              Beginning at startPos + 2, searches fileContents for the end of
     152              a block comment. If block comments are nested, the
     153              function interates into each block comment by calling itself.
     154
     155              MIB block comment rules:
     156              1. Begins with '/*'
     157              2. Ends with '*/'
     158              3. Block comments can be nested
     159              3. Any characters between the beginning and end of the comment
     160                  are ignored as part of the comment, including quotes and
     161                  start/end delimiters for single line comments ('--'). Newlines
     162                  are included as part of the block comment.
     163
     164              @param startPos: character position of the beginning of the block
     165                      comment within fileContents
     166              @type fileContents: string
     167              @return: startPos, endPos (character position of the last character
     168                      in the comment + 1)
     169              @rtype: tuple (integer, integer)
     170              """
     171              # Locate the next start and end markers
     172              nextBlockStartPos = fileContents.find('/*', searchStartPos + 2)
     173              nextBlockEndPos = fileContents.find('*/', searchStartPos + 2)
     174
     175              # If a nested comment exists, find the end
     176              if nextBlockStartPos != -1 and \
     177                      nextBlockStartPos < nextBlockEndPos:
     178                  nestedComment = findBlockCommentEndPos(nextBlockStartPos)
     179                  nextBlockEndPos = fileContents.find('*/', nestedComment[1])
     180
     181              return searchStartPos, nextBlockEndPos + 2
     182
     183          # START removeMibComments
     184          if not fileContents:
     185              return fileContents
     186
     187          # Get rid of any lines that are completely made up of hyphens
     188          fileContents = re.sub(r'[ \t]*-{2}[ \t]*$', '', fileContents)
     189
     190          # commentRanges holds a list of tuples in the form (startPos, endPos)
     191          # that define the beginning and end of comments within fileContents
     192          commentRanges = []
     193          searchStartPos = 0   # character position within fileContents
     194          functions = {'SINGLE': findSingleLineCommentEndPos,
     195                            'BLOCK': findBlockCommentEndPos}
     196
     197          # Parse fileContents, looking for single line comments, block comments
     198          # and string literals
     199          while searchStartPos < len(fileContents):
     200              # Find the beginning of the next occurrance of each item
     201              singleLineStartPos = fileContents.find('--', searchStartPos)
     202              blockStartPos = fileContents.find('/*', searchStartPos)
     203              stringStartPos = fileContents.find('\"', searchStartPos)
     204
     205              nextItemPos = sys.maxint
     206              nextItemType = ''
     207
     208              # Compare the next starting point for each item type.
     209              if singleLineStartPos != -1 and \
     210                      singleLineStartPos < nextItemPos:
     211                  nextItemPos = singleLineStartPos
     212                  nextItemType = 'SINGLE'
     213
     214              if blockStartPos != -1 and \
     215                      blockStartPos < nextItemPos:
     216                  nextItemPos = blockStartPos
     217                  nextItemType = 'BLOCK'
     218
     219              # If the next item type is a string literal, just search for the
     220              # next double quote and continue from there. This works because
     221              # all double quotes (that are not part of a comment) appear in
     222              # pairs. Even double-double quotes (escaped quotes) will work
     223              # with this method since the first double quote will look like a
     224              # string literal close quote and the second double quote will look
     225              # like the beginning of a string literal.
     226              if stringStartPos != -1 and \
     227                      stringStartPos < nextItemPos:
     228                  newSearchStartPos = \
     229                      fileContents.find('\"', stringStartPos + 1) + 1
     230                  if newSearchStartPos > searchStartPos:
     231                      searchStartPos = newSearchStartPos
     232                  else: # Weird error case
     233                      break
     234
     235              # If the next item is a comment, use the functions dictionary
     236              # to call the appropriate function
     237              elif nextItemPos != sys.maxint:
     238                  commentRange = functions[nextItemType](nextItemPos)
     239                  commentRanges.append(commentRange)
     240                  #searchStartPos = commentRange[1]
     241                  if commentRange[1] > 0:
     242                      searchStartPos = commentRange[1]
     243
     244              else: # No other items are found!
     245                  break
     246
     247          startPos = 0
     248          mibParts = []
     249
     250          # Iterate through each comment, adding the non-comment parts
     251          # to mibParts. Finally, return the text without comments.
     252          for commentRange in commentRanges:
     253              mibParts.append(fileContents[startPos:(commentRange[0])])
     254              startPos = commentRange[1]
     255          if startPos != len(fileContents):
     256              mibParts.append(fileContents[startPos:(len(fileContents))])
     257          return ''.join(mibParts)
     258
     259 -    def splitFileToMIBs(self, fileContents):
     260          """
     261          Isolates each MIB definition in fileContents into a separate string
     262
     263          @param fileContents: the complete contents of a MIB file
     264          @type fileContents: string
     265          @return: MIB definition strings
     266          @rtype: list of strings
     267          """
     268          if fileContents is None:
     269              return []
     270
     271          DEFINITIONS = re.compile(r'([A-Za-z-0-9]+\s+DEFINITIONS'
     272              '(\s+EXPLICIT TAGS|\s+IMPLICIT TAGS|\s+AUTOMATIC TAGS|\s*)'
     273              '(\s+EXTENSIBILITY IMPLIED|\s*)\s*::=\s*BEGIN)')
     274
     275          definitionSpans = []
     276          for definitionMatch in DEFINITIONS.finditer(fileContents):
     277              definitionSpans.append(list(definitionMatch.span()))
     278              # If more than one definiton in the file, set the end of the
     279              # last span to the beginning of the current span
     280              if len(definitionSpans) > 1:
     281                  definitionSpans[-2][1] = definitionSpans[-1][0]
     282
     283          # Use the start and end positions to create a string for each
     284          # MIB definition
     285          mibDefinitions = []
     286          if definitionSpans:
     287              # Set the end of the last span to the end of the file
     288              definitionSpans[-1][1] = len(fileContents)
     289              for definitionSpan in definitionSpans:
     290                  mibDefinitions.append(
     291                      fileContents[definitionSpan[0]:definitionSpan[1]])
     292
     293          return mibDefinitions
     294
     295 -    def mapMibToDependents(self, mibDefinitions):
     296          # ASN.1 syntax regular expressions for declaring MIBs
     297          #
     298          # An example from http://www.faqs.org/rfcs/rfc1212.html
     299          #
     300          # RFC1213-MIB DEFINITIONS ::= BEGIN
     301          #
     302          # IMPORTS
     303          #        experimental, OBJECT-TYPE, Counter
     304          #             FROM RFC1155-SMI;
     305          #
     306          #     -- a MIB may or may not have an IMPORTS section
     307          #
     308          # root    OBJECT IDENTIFIER ::= { experimental xx }
     309          #
     310          # END
     311          IMPORTS = re.compile(r'\sIMPORTS\s.+;', re.DOTALL)
     312          DEPENDENCIES = re.compile(
     313              r'\sFROM\s+(?P<dependency>[A-Za-z-0-9]+)')
     314
     315          mibDependencies = []
     316          for definition in mibDefinitions:
     317              mibName = re.split(r'([A-Za-z0-9-]+)', definition)[1]
     318              dependencies = set()
     319              importsMatch = IMPORTS.search(definition)
     320              if importsMatch:
     321                  imports = importsMatch.group()
     322                  for dependencyMatch in DEPENDENCIES.finditer(imports):
     323                      dependencies.add(dependencyMatch.group('dependency'))
     324              self.mibs.append(mibName)
     325              self.mibToDeps[mibName] = dependencies
     326
     327 -class PackageManager:
     328      """
     329      Given an URL, filename or archive (eg zip, tar), extract the files from
     330      the package and return a list of filenames.
     331      """
     332 -    def __init__(self, log, downloaddir, extractdir):
     333          """
     334          Initialize the packagae manager.
     335
     336          @parameter log: logging object
     337          @type log: logging class object
     338          @parameter downloaddir: directory name to store downloads
     339          @type downloaddir: string
     340          @parameter extractdir: directory name to store downloads
     341          @type extractdir: string
     342          """
     343          self.log = log
     344          self.downloaddir = downloaddir
     345          if self.downloaddir[-1] != '/':
     346              self.downloaddir += '/'
     347          self.extractdir = extractdir
     348          if self.extractdir[-1] != '/':
     349              self.extractdir += '/'
     350
     351 -    def downloadExtract(self, url):
     352          """
     353          Download and extract the list of filenames.
     354          """
     355          try:
     356              localFile = self.download(url)
     357          except (SystemExit, KeyboardInterrupt): raise
     358          except:
     359               self.log.error("Problems downloading the file from %s: %s" % (
     360                  url, sys.exc_info()[1] ) )
     361               return []
     362          self.log.debug("Will attempt to load %s", localFile)
     363          try:
     364              return self.processPackage(localFile)
     365          except IOError, ex:
     366              self.log.error("Unable to process '%s' because: %s",
     367                             localFile, str(ex))
     368              sys.exit(1)
     369
     370 -    def download(self, url):
     371          """
     372          Download the package from the given URL, or if it's a filename,
     373          return the filename.
     374          """
     375          urlParts = urlsplit(url)
     376          schema = urlParts[0]
     377          path = urlParts[2]
     378          if not schema:
     379              return os.path.abspath(url)
     380          file = path.split(os.sep)[-1]
     381          os.makedirs(self.downloaddir)
     382          downloadFile = os.path.join(self.downloaddir, file)
     383          self.log.debug("Downloading to file '%s'", downloadFile)
     384          filename, _ = urllib.urlretrieve(url, downloadFile)
     385          return filename
     386
     387 -    def processPackage(self, pkgFileName):
     388          """
     389          Figure out what type of file we have and extract out any
     390          files and then enumerate the file names.
     391          """
     392          self.log.debug("Determining file type of %s" % pkgFileName)
     393          if zipfile.is_zipfile(pkgFileName):
     394              return self.unbundlePackage(pkgFileName, self.unzip)
     395
     396          elif tarfile.is_tarfile(pkgFileName):
     397              return self.unbundlePackage(pkgFileName, self.untar)
     398
     399          elif os.path.isdir(pkgFileName):
     400              return self.processDir(pkgFileName)
     401
     402          else:
     403              return [ os.path.abspath(pkgFileName) ]
     404
     405 -    def unzip(self, file):
     406          """
     407          Unzip the given file into the current directory and return
     408          the directory in which files can be loaded.
     409          """
     410          pkgZip= zipfile.ZipFile(file, 'r')
     411          if pkgZip.testzip() != None:
     412              self.log.error("File %s is corrupted -- please download again", file)
     413              return
     414
     415          for file in pkgZip.namelist():
     416              self.log.debug("Unzipping file/dir %s..." % file)
     417              try:
     418                  if re.search(r'/$', file) != None:
     419                      os.makedirs(file)
     420                  else:
     421                      contents = pkgZip.read(file)
     422                      try:
     423                          unzipped = open(file, "w")
     424                      except IOError: # Directory missing?
     425                          os.makedirs(os.path.dirname(file))
     426                          unzipped = open(file, "w")
     427                      unzipped.write(contents)
     428                      unzipped.close()
     429              except (SystemExit, KeyboardInterrupt): raise
     430              except:
     431                  self.log.error("Error in extracting %s because %s" % (
     432                      file, sys.exc_info()[1] ) )
     433                  return
     434
     435          return os.getcwd()
     436
     437 -    def untar(self, file):
     438          """
     439          Given a tar file, extract from the tar into the current directory.
     440          """
     441          try:
     442              self.log.debug("Extracting files from tar...")
     443              pkgTar = tarfile.open(file, 'r')
     444              for tarInfo in pkgTar:
     445                  pkgTar.extract(tarInfo)
     446              pkgTar.close()
     447          except (SystemExit, KeyboardInterrupt): raise
     448          except:
     449              self.log.error("Error in un-tarring %s because %s" % ( file,
     450                                                          sys.exc_info()[1] ) )
     451              return
     452          return os.getcwd()
     453
     454 -    def processDir(self, dir):
     455          """
     456          Note all of the files in a directory.
     457          """
     458          fileList = []
     459          self.log.debug("Enumerating files in %s", dir)
     460          if not os.path.isdir(dir):
     461              self.log.debug("%s is not a directory", dir)
     462              return []
     463          for directoryName, _, fileNames in os.walk(dir):
     464              for fileName in fileNames:
     465                  fileList.append(os.path.join(directoryName, fileName))
     466          return fileList
     467
     468 -    def unbundlePackage(self, package, unpackageMethod):
     469          """
     470          Extract the files and then add to the list of files.
     471          """
     472          self.makeExtractionDir()
     473          baseDir = unpackageMethod(package)
     474          if baseDir is not None:
     475              return self.processDir(baseDir)
     476          return []
     477
     478 -    def makeExtractionDir(self):
     479          """
     480          Create an uniquely named extraction directory starting from a base
     481          extraction directory.
     482          """
     483          try:
     484              if not os.path.isdir(self.extractdir):
     485                  os.makedirs(self.extractdir)
     486              extractDir = tempfile.mkdtemp(prefix=self.extractdir)
     487          except (SystemExit, KeyboardInterrupt): raise
     488          except:
     489              self.log.error("Error in creating temp dir because %s",
     490                                                      sys.exc_info()[1] )
     491              sys.exit(1)
     492          os.chdir(extractDir)
     493
     494 -    def cleanup(self):
     495          """
     496          Remove any clutter left over from the installation.
     497          """
     498          self.cleanupDir(self.downloaddir)
     499          self.cleanupDir(self.extractdir)
     500
     501 -    def cleanupDir(self, dirName):
     502          for root, dirs, files in os.walk(dirName, topdown=False):
     503              if root == os.sep: # Should *never* get here
     504                  break
     505              for name in files:
     506                  os.remove(os.path.join(root, name))
     507              for name in dirs:
     508                  try:
     509                      os.removedirs(os.path.join(root, name))
     510                  except OSError:
     511                      pass
     512          try:
     513              os.removedirs(dirName)
     514          except OSError:
     515              pass
     516
     517
     518 -class ZenMib(ZCmdBase):
     519      """
     520      Wrapper around the smidump utilities used to convert MIB definitions into
     521      python code which is in turn loaded into the DMD tree.
     522      """
     523 -    def makeMibFileObj(self, fileName):
     524          """
     525          Scan the MIB file to determine what MIBs are defined in the file and
     526          what their dependencies are.
     527
     528          @param fileName: MIB fileName
     529          @type fileName: string
     530          @return: dependencyDict, indexDict
     531              dependencyDict - a dictionary that associates MIB definitions
     532                  found in fileName with their dependencies
     533              indexDict - a dictionary that associates MIB definitions with their
     534                  ordinal position within fileName}
     535          @rtype:
     536              dependencyDict = {mibName: [string list of MIB definition names
     537                  that mibName is dependant on]}
     538              indexDict = {mibname: number}
     539          """
     540          # Retrieve the entire contents of the MIB file
     541          self.log.debug("Processing %s", fileName)
     542          file = open(fileName)
     543          fileContents = file.read()
     544          file.close()
     545          return MibFile(fileName, fileContents)
     546
     547 -    def populateDependencyMap(self, importFileNames, depFileNames):
     548          """
     549          Populates the self.mibToMibFile instance object with data.
     550          Exit the program if we're missing any files.
     551
     552          @param importFileNames: fully qualified file names of MIB files to import
     553          @type importFileNames: list of strings
     554          @param depFileNames: fully qualified file names of all MIB files
     555          @type depFileNames: list of strings
     556          @return: mibFileObjects of files to import
     557          @rtype: MibFile
     558          """
     559          self.log.debug("Collecting MIB meta-data and creating depedency map.")
     560          toImportMibFileObjs = []
     561          for fileName in depFileNames.union(importFileNames):
     562              try:
     563                  mibFileObj = self.makeMibFileObj(fileName)
     564              except IOError:
     565                  self.log.error("Couldn't open file %s", fileName)
     566                  continue
     567
     568              mibDependencies = mibFileObj.mibToDeps
     569              if not mibDependencies:
     570                  self.log.warn("Unable to parse information from "
     571                      "%s -- skipping", fileName)
     572                  continue
     573
     574              if fileName in importFileNames:
     575                  toImportMibFileObjs.append(mibFileObj)
     576
     577              for mibName, dependencies in mibDependencies.items():
     578                  self.mibToMibFile[mibName] = mibFileObj
     579          return toImportMibFileObjs
     580
     581 -    def getDependencyFileNames(self, mibFileObj):
     582          """
     583          smidump needs to know the list of dependent files for a MIB file in
     584          order to properly resolve external references.
     585
     586          @param mibFileObj: MibFile object
     587          @type mibFileObj: MibFile
     588          @return: list of dependency fileNames
     589          @rtype: list of strings
     590          """
     591          dependencies = []
     592          dependencyFileNames = set()
     593
     594          def dependencySearch(mibName):
     595              """
     596              Create a list of files required by a MIB definition.
     597
     598              @param mibName: name of MIB definition
     599              @type mibName: string
     600              """
     601              dependencies.append(mibName)
     602              mibFileObj = self.mibToMibFile.get(mibName)
     603              if not mibFileObj:
     604                  self.log.warn("Unable to find a file that defines %s", mibName)
     605                  return
     606
     607              dependencyFileNames.add(mibFileObj.fileName)
     608              for dependency in mibFileObj.mibToDeps[mibName]:
     609                  if dependency not in dependencies:
     610                      dependencySearch(dependency)
     611
     612          for mibName in mibFileObj.mibs:
     613              dependencySearch(mibName)
     614
     615          dependencyFileNames.discard(mibFileObj.fileName)
     616          return dependencyFileNames
     617
     618 -    def savePythonCode(self, pythonCode, fileName):
     619          """
     620          Stores the smidump-generated Python code to a file.
     621          """
     622          if not os.path.exists(self.options.pythoncodedir):
     623              self.options.keeppythoncode = False
     624              self.log.warn('The directory %s to store converted MIB file code '
     625                  'does not exist.' % self.options.pythoncodedir)
     626              return
     627          try:
     628              pythonFileName = os.path.join(self.options.pythoncodedir,
     629              os.path.basename(fileName) ) + '.py'
     630              pythonFile = open(pythonFileName, 'w')
     631              pythonFile.write(pythonCode)
     632              pythonFile.close()
     633          except (SystemExit, KeyboardInterrupt): raise
     634          except:
     635              self.log.warn('Could not output converted MIB to %s' %
     636                  pythonFileName)
     637
     638 -    def generatePythonFromMib(self, fileName, dependencyFileNames,
     639              mibNamesInFile):
     640          """
     641          Use the smidump program to convert a MIB into Python code.
     642
     643          One major caveat: smidump by default only outputs the last MIB
     644          definition in a file. For that matter, it always outputs the last MIB
     645          definition in a file whether it is requested or not. Therefore, if
     646          there are multiple MIB definitions in a file, all but the last must be
     647          explicitly named on the command line. If you name the last, it will
     648          come out twice. We don't want that.
     649
     650          OK, so we need to determine if there are multiple MIB definitions
     651          in fileName and then add all but the last to the command line. That
     652          works except the resulting python code will create a dictionary
     653          for each MIB definition, all of them named MIB. Executing the code is
     654          equivalent to running a=1; a=2; a=3. You only wind up with a=3.
     655          Therefore, we separate each dictionary definition into its own string
     656          and return a list of strings so each one can be executed individually.
     657
     658          @param fileName: name of the file containing MIB definitions
     659          @type fileName: string
     660          @param dependencyFileNames: list of fileNames that fileName is
     661              dependent on
     662          @type dependencyFileNames: list of strings
     663          @param mibNamesInFile: names of MIB definitions in file
     664          @type mibNamesInFile: list of strings
     665          @return: list of dictionaries. Each dictionary containing the contents
     666              of a MIB definition. [ {'mibName': MIB data} ]
     667          @rtype: list
     668          """
     669          def infiniteLoopHandler(signum, frame):
     670              """
     671              Kills any smidump commands that have probably locked themselves
     672              into an infinite loop.
     673              """
     674              log.error("The command %s has probably gone into an infinite loop",
     675                        ' '.join(dumpCommand))
     676              log.error("Killing process id %s ...", proc.pid)
     677              try:
     678                  os.kill(proc.pid, signal.SIGKILL)
     679              except OSError:
     680                  pass
     681
     682
     683          dumpCommand =  ['smidump', '--keep-going', '--format', 'python']
     684          for dependencyFileName in dependencyFileNames:
     685              #  Add command-line flag for our dependent files
     686              dumpCommand.append('--preload')
     687              dumpCommand.append(dependencyFileName)
     688          dumpCommand.append(fileName)
     689
     690          # If more than one MIB definition exists in the file, name all but the
     691          # last on the command line. (See method description for reasons.)
     692          if len(mibNamesInFile) > 1:
     693              dumpCommand += mibNamesInFile[:-1]
     694
     695          self.log.debug('Running %s', ' '.join(dumpCommand))
     696          proc = Popen(dumpCommand, stdout=PIPE, stderr=PIPE)
     697
     698          log = self.log
     699          signal.signal(signal.SIGALRM, infiniteLoopHandler)
     700          signal.alarm(self.options.smidumptimeout)
     701          pythonCode, warnings = proc.communicate()
     702          proc.wait()
     703          signal.alarm(0) # Disable the alarm
     704          if proc.returncode:
     705              if warnings.strip():
     706                  self.log.error(warnings)
     707              return None
     708
     709          if warnings:
     710              self.log.debug("Found warnings while trying to import MIB:\n%s" \
     711                   % warnings)
     712
     713          if self.options.keeppythoncode:
     714              self.savePythonCode(pythonCode, fileName)
     715
     716          return self.evalPythonToMibs(pythonCode, fileName)
     717
     718 -    def evalPythonToMibs(self, pythonCode, name):
     719          """
     720          Evaluate the code and return an array of MIB dictionaries.
     721          """
     722          def executePythonCode(pythonCode, name):
     723              """
     724              Executes the python code generated smidump
     725
     726              @param pythonCode: Code generated by smidump
     727              @type pythonCode: string
     728              @return: a dictionary which contains one key: MIB
     729              @rtype: dictionary
     730              """
     731              result = {}
     732              try:
     733                  exec pythonCode in result
     734              except (SystemExit, KeyboardInterrupt): raise
     735              except:
     736                  self.log.exception("Unable to import Pythonized-MIB: %s",
     737                      name)
     738              return result.get('MIB', None)
     739
     740          # If more than one MIB definition exists in a file, pythonCode will
     741          # contain a 'MIB = {...}' section for each MIB definition. We must
     742          # split each section into its own string and return a string list.
     743          smiMibDelim = 'MIB = {'
     744          mibCodeParts = pythonCode.split(smiMibDelim)
     745          mibDicts = []
     746          if len(mibCodeParts) > 1:
     747              for mibCodePart in mibCodeParts[1:]:
     748                  mibDict = executePythonCode(smiMibDelim + mibCodePart, name)
     749                  if mibDict is not None:
     750                      mibDicts.append(mibDict)
     751          else:
     752              mibDict = executePythonCode(pythonCode, name)
     753              if mibDict is not None:
     754                  mibDicts = [mibDict]
     755
     756          return mibDicts
     757
     758 -    def evalAddSavedPythonCode(self):
     759          """
     760          Read the file named in the command-line option, evaluate it, and add
     761          it to our list of MIBs.
     762          """
     763          pythonMibs = []
     764          for fileName in set(self.options.evalSavedPython):
     765              try:
     766                  pythonCode = open(fileName).read()
     767              except IOError:
     768                  self.log.warn("Unable to open '%s' -- skipping",
     769                                fileName)
     770                  continue
     771              pythonMibs += self.evalPythonToMibs(pythonCode, fileName)
     772
     773          self.loadPythonMibs(pythonMibs)
     774
     775
     776 -    def getDmdMibDict(self, dmdMibDict, mibOrganizer):
     777          """
     778          Populate a dictionary containing the MIB definitions that have been
     779          loaded into the DMD Mibs directory
     780
     781          @param dmdMibDict: maps a MIB definition to the path where
     782                  it is located with in the DMD.
     783              Format:
     784                  {'mibName': 'DMD path where mibName is stored'}
     785              Example: MIB-Dell-10892 is located in the DMD tree at
     786                  Mibs/SITE/Dell, Directory entry is
     787                  {'MIB-Dell-10892': '/SITE/Dell'] }
     788          @param mibOrganizer: the DMD directory to be searched
     789          @type mibOrganizer: MibOrganizer
     790          """
     791          organizerPath = mibOrganizer.getOrganizerName()
     792
     793          # Record each module from this organizer as a dictionary entry.
     794          # mibOrganizer.mibs.objectItems() returns tuple:
     795          # ('mibName', <MibModule at /zport/dmd/Mibs/...>)
     796          for mibModule in mibOrganizer.mibs.objectItems():
     797              mibName = mibModule[0]
     798              if mibName not in dmdMibDict:
     799                  dmdMibDict[mibName] = organizerPath
     800              else:
     801                  self.log.warn('\nFound two copies of %s:'
     802                      '  %s and %s' %
     803                      (mibName, dmdMibDict[mibName],
     804                      mibOrganizer.getOrganizerName()))
     805
     806          # If there are suborganizers, recurse into them
     807          for childOrganizer in mibOrganizer.children():
     808              self.getDmdMibDict(dmdMibDict, childOrganizer)
     809
     810 -    def addMibEntries(self, leafType, pythonMib, mibModule):
     811          """
     812          Add the different MIB leaves (ie nodes, notifications) into the DMD.
     813
     814          @paramater leafType: 'nodes', 'notifications'
     815          @type leafType: string
     816          @paramater pythonMib: dictionary of nodes and notifications
     817          @type pythonMib: dictionary
     818          @paramater mibModule: class containing functions to load leaves
     819          @type mibModule: class
     820          @return: number of leaves added
     821          @rtype: int
     822          """
     823          entriesAdded = 0
     824          functor = { 'nodes':mibModule.createMibNode,
     825                      'notifications':mibModule.createMibNotification,
     826                     }.get(leafType, None)
     827          if not functor or leafType not in pythonMib:
     828              return entriesAdded
     829
     830          mibName = pythonMib['moduleName']
     831
     832          for name, values in pythonMib[leafType].items():
     833              try:
     834                  functor(name, **values)
     835                  entriesAdded += 1
     836              except BadRequest:
     837                  try:
     838                      self.log.warn("Unable to add %s id '%s' as this"
     839                                  " name is reserved for use by Zope",
     840                                  leafType, name)
     841                      newName = '_'.join([name, mibName])
     842                      self.log.warn("Trying to add %s '%s' as '%s'",
     843                                  leafType, name, newName)
     844                      functor(newName, **values)
     845                      entriesAdded += 1
     846                      self.log.warn("Renamed '%s' to '%s' and added to"
     847                                  " MIB %s", name, newName, leafType)
     848                  except (SystemExit, KeyboardInterrupt): raise
     849                  except:
     850                      self.log.warn("Unable to add %s id '%s' -- skipping",
     851                                  leafType, name)
     852              else:
     853                  if not entriesAdded % CHUNK_SIZE:
     854                      self.commit("Loaded MIB %s into the DMD" % mibName)
     855          self.commit("Loaded MIB %s into the DMD" % mibName)
     856          return entriesAdded
     857
     858 -    def loadMibFile(self, mibFileObj, dmdMibDict):
     859          """
     860          Attempt to load the MIB definitions in fileName into DMD
     861
     862          @param fileName: name of the MIB file to be loaded
     863          @type fileName: string
     864          @return: whether the MIB load was successful or not
     865          @rtype: boolean
     866          """
     867          fileName = mibFileObj.fileName
     868          self.log.debug('Attempting to load %s' % fileName)
     869
     870          # Check to see if any MIB definitions in fileName have already
     871          # been loaded into Zenoss. If so, warn but don't fail
     872          mibNamesInFile = mibFileObj.mibs
     873          for mibName in mibNamesInFile:
     874              if mibName in dmdMibDict:
     875                  dmdMibPath = dmdMibDict[mibName]
     876                  self.log.warn('MIB definition %s found in %s is already '
     877                      'loaded at %s.' % (mibName, fileName, dmdMibPath))
     878
     879          # Retrieve a list of all the files containing MIB definitions that are
     880          # required by the MIB definitions in fileName
     881          dependencyFileNames = self.getDependencyFileNames(mibFileObj)
     882
     883          # Convert the MIB file data into python dictionaries. pythonMibs
     884          # contains a list of dictionaries, one for each MIB definition in
     885          # fileName.
     886          pythonMibs = self.generatePythonFromMib(fileName, dependencyFileNames,
     887              mibNamesInFile)
     888          if not pythonMibs:
     889              return False
     890
     891          self.loadPythonMibs(pythonMibs)
     892          return True
     893
     894 -    def commit(self, message):
     895          if not self.options.nocommit:
     896              self.log.debug('Committing a batch of objects')
     897              trans = transaction.get()
     898              trans.setUser('zenmib')
     899              trans.note(message)
     900              trans.commit()
     901              self.syncdb()
     902
     903 -    def loadPythonMibs(self, pythonMibs):
     904          """
     905          Walk through the MIB dictionaries and add the MIBs to the DMD.
     906          """
     907          # Standard MIB attributes that we expect in all MIBs
     908          MIB_MOD_ATTS = ('language', 'contact', 'description')
     909
     910          self.syncdb()
     911
     912          # Add the MIB data for each MIB into Zenoss
     913          for pythonMib in pythonMibs:
     914              mibName = pythonMib['moduleName']
     915
     916              # Create the container for the MIBs and define meta-data.
     917              # In the DMD this creates another container class which
     918              # contains mibnodes.  These data types are found in
     919              # Products.ZenModel.MibModule and Products.ZenModel.MibNode
     920              mibModule = self.dmd.Mibs.createMibModule(
     921                  mibName, self.options.path)
     922
     923              def gen():
     924                  for key, val in pythonMib[mibName].iteritems():
     925                      if key in MIB_MOD_ATTS:
     926                          yield key, val
     927
     928              for i, attr in enumerate(gen()):
     929                  setattr(mibModule, *attr)
     930                  if not i % CHUNK_SIZE:
     931                      self.commit("Loaded MIB %s into the DMD" % mibName)
     932              self.commit("Loaded MIB %s into the DMD" % mibName)
     933
     934              nodesAdded = self.addMibEntries('nodes', pythonMib, mibModule)
     935              trapsAdded = self.addMibEntries('notifications', pythonMib, mibModule)
     936              self.log.info("Parsed %d nodes and %d notifications from %s",
     937                            nodesAdded, trapsAdded, mibName)
     938
     939              # Add the MIB tree permanently to the DMD unless --nocommit flag.
     940              msg = "Loaded MIB %s into the DMD" % mibName
     941              self.commit(msg)
     942              if not self.options.nocommit:
     943                  self.log.info(msg)
     944
     945
     946 -    def getAllMibDepFileNames(self):
     947          """
     948          Use command line parameters to create a list of files containing MIB
     949          definitions that will be used as a reference list for the files being
     950          loaded into the DMD
     951
     952          @return: set of file names
     953          @rtype: set
     954          """
     955          defaultMibDepDirs = [ 'ietf', 'iana', 'irtf', 'tubs', 'site' ]
     956          mibDepFileNames = set()
     957          for subdir in defaultMibDepDirs:
     958              depDir = os.path.join(self.options.mibdepsdir, subdir)
     959              mibDepFileNames.update(self.pkgMgr.processDir(depDir))
     960          return mibDepFileNames
     961
     962 -    def getMibsToImport(self):
     963          """
     964          Uses command-line parameters to create a list of files containing MIB
     965          definitions that are to be loaded into the DMD
     966
     967          @return: list of file names that are to be loaded into the DMD
     968          @rtype: list
     969          """
     970          loadFileNames = []
     971          if self.args:
     972              for fileName in self.args:
     973                  loadFileNames.extend(self.pkgMgr.downloadExtract(fileName))
     974          else:
     975              loadFileNames = self.pkgMgr.processDir(self.options.mibsdir)
     976
     977          if loadFileNames:
     978              self.log.debug("Will attempt to load the following files: %s",
     979                         loadFileNames)
     980          else:
     981              self.log.error("No MIB files to load!")
     982              sys.exit(1)
     983
     984          return set(loadFileNames)
     985
     986 -    def main(self):
     987          """
     988          Main loop of the program
     989          """
     990          if self.options.evalSavedPython:
     991              self.evalAddSavedPythonCode()
     992              return
     993
     994          # Verify MIBs search directory is valid. Fail if not
     995          if not os.path.exists(self.options.mibsdir):
     996              self.log.error("The directory %s doesn't exist!" %
     997                  self.options.mibsdir )
     998              sys.exit(1)
     999
    1000          self.pkgMgr = PackageManager(self.log, self.options.downloaddir,
    1001                                       self.options.extractdir)
    1002          self.mibToMibFile = {}
    1003
    1004          requestedFiles = self.getMibsToImport()
    1005          mibDepFileNames = self.getAllMibDepFileNames()
    1006          mibFileObjs = self.populateDependencyMap(requestedFiles, mibDepFileNames)
    1007
    1008          # dmdMibDict = {'mibName': 'DMD path to MIB'}
    1009          dmdMibDict = {}
    1010          self.getDmdMibDict(dmdMibDict, self.dmd.Mibs)
    1011
    1012          # Load the MIB files
    1013          self.log.info("Found %d MIBs to import.", len(mibFileObjs))
    1014          loadedMibFiles = 0
    1015          for mibFileObj in mibFileObjs:
    1016              try:
    1017                  if self.loadMibFile(mibFileObj, dmdMibDict):
    1018                      loadedMibFiles += 1
    1019              except (SystemExit, KeyboardInterrupt): raise
    1020              except Exception, ex:
    1021                  self.log.exception("Failed to load MIB: %s", mibFileObj.fileName)
    1022
    1023          action = "Loaded"
    1024          if self.options.nocommit:
    1025              action = "Processed"
    1026
    1027          self.log.info("%s %d MIB file(s)" % (action, loadedMibFiles))
    1028          self.pkgMgr.cleanup()
    1029
    1030          sys.exit(0)
    1031
    1032 -    def buildOptions(self):
    1033          """
    1034          Command-line options
    1035          """
    1036          ZCmdBase.buildOptions(self)
    1037          self.parser.add_option('--mibsdir',
    1038                                 dest='mibsdir', default=zenPath('share/mibs/site'),
    1039                         help="Directory of input MIB files [ default: %default ]")
    1040          self.parser.add_option('--mibdepsdir',
    1041                          dest='mibdepsdir', default=zenPath('share/mibs'),
    1042                         help="Directory of input MIB files [ default: %default ]")
    1043          self.parser.add_option('--path',
    1044                                 dest='path', default="/",
    1045                                 help="Path to load MIB into the DMD")
    1046          self.parser.add_option('--nocommit', action='store_true',
    1047                                 dest='nocommit', default=False,
    1048                             help="Don't commit the MIB to the DMD after loading")
    1049          self.parser.add_option('--keeppythoncode', action='store_true',
    1050                                 dest='keeppythoncode', default=False,
    1051                             help="Don't commit the MIB to the DMD after loading")
    1052          self.parser.add_option('--pythoncodedir', dest='pythoncodedir',
    1053              default=tempfile.gettempdir() + "/mib_pythoncode/",
    1054              help="This is the directory where the converted MIB will be output. " \
    1055                  "[ default: %default ]")
    1056          self.parser.add_option('--downloaddir', dest='downloaddir',
    1057              default=tempfile.gettempdir() + "/mib_downloads/",
    1058              help="This is the directory where the MIB will be downloaded. " \
    1059                  "[ default: %default ]")
    1060          self.parser.add_option('--extractdir', dest='extractdir',
    1061              default=tempfile.gettempdir() + "/mib_extract/",
    1062              help="This is the directory where unzipped MIB files will be stored. " \
    1063                  "[ default: %default ]")
    1064          self.parser.add_option('--smidumptimeout', dest='smidumptimeout',
    1065                             default=60,
    1066                             help="Kill smidump after this many seconds to " \
    1067                                  "stop infinite loops.")
    1068          self.parser.add_option('--evalSavedPython', dest='evalSavedPython',
    1069                             default=[], action='append',
    1070                             help="Execute the Python code previously generated" \
    1071                                  " and saved.")
    1072
    1073
    1074  if __name__ == '__main__':
    1075      zm = ZenMib()
    1076      zm.main()
    1077