Products.ZenModel.zenmib

1 ########################################################################### 2 # 3 # This program is part of Zenoss Core, an open source monitoring platform. 4 # Copyright (C) 2007, 2009 Zenoss Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify it 7 # under the terms of the GNU General Public License version 2 as published by 8 # the Free Software Foundation. 9 # 10 # For complete information please visit: http://www.zenoss.com/oss/ 11 # 12 ########################################################################### 13 __doc__ = """zenmib 14 The zenmib program converts MIBs into python data structures and then 15 (by default) adds the data to the Zenoss DMD. Essentially, zenmib is a 16 wrapper program around the smidump program, whose output (python code) is 17 then executed "inside" the Zope database. 18 19 Overview of terms: 20 SNMP Simple Network Management Protocol 21 A network protocol originally based on UDP which allows a management 22 application (ie SNMP manager) to contact and get information from a 23 device (ie router, computer, network-capable toaster). The software 24 on the device that is capable of understanding SNMP and responding 25 appropriately is called an SNMP agent. 26 27 MIB Management of Information Base 28 A description of what a particular SNMP agent provides and what 29 traps it sends. A MIB is a part of a tree structure based on a root 30 MIB. Since a MIB is a rooted tree, it allows for delegation of areas 31 under the tree to different organizations. 32 33 ASN Abstract Syntax Notation 34 The notation used to construct a MIB. 35 36 OID Object IDentifier 37 A MIB is constructed of unique identifiers 38 39 40 Background information: 41 http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol 42 Overview of SNMP. 43 44 http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en 45 The libsmi project is the creator of smidump. There are several 46 interesting sub-projects available. 47 48 http://net-snmp.sourceforge.net/ 49 Homepage for Net-SNMP which is used by Zenoss for SNMP management. 50 51 http://www.et.put.poznan.pl/snmp/asn1/asn1.html 52 An overview of Abstract Syntax Notation (ASN), the language in 53 which MIBs are written. 54 """ 55 56 import os 57 import os.path 58 import sys 59 import glob 60 import re 61 import logging 62 from subprocess import Popen, PIPE 63 import tempfile 64 import urllib 65 import tarfile 66 import zipfile 67 import signal 68 from urllib2 import urlopen 69 from urlparse import urljoin, urlsplit 70 71 import Globals 72 import transaction 73 74 from Products.ZenUtils.ZCmdBase import ZCmdBase 75 from Products.ZenUtils.Utils import zenPath 76 from zExceptions import BadRequest 77 78 79 CHUNK_SIZE = 50 80 81

82 -class MibFile:

83 """ 84 A MIB file has the meta-data for a MIB inside of it. 85 """

86 - def __init__(self, fileName, fileContents=""):

87 self.fileName = fileName 88 self.mibs = [] # Order of MIBs defined in the file 89 self.mibToDeps = {} # Dependency defintions for each MIB 90 self.fileContents = self.removeMibComments(fileContents) 91 self.mibDefinitions = self.splitFileToMIBs(self.fileContents) 92 self.mapMibToDependents(self.mibDefinitions) 93 94 # Reclaim some memory 95 self.fileContents = "" 96 self.mibDefinitions = ""

97

98 - def removeMibComments(self, fileContents):

99 """ 100 Parses the string provided as an argument and extracts 101 all of the ASN.1 comments from the string. 102 103 Assumes that fileContents contains the contents of a well-formed 104 (no errors) MIB file. 105 106 @param fileContents: entire contents of a MIB file 107 @type fileContents: string 108 @return: text without any comments 109 @rtype: string 110 """ 111 def findSingleLineCommentEndPos(startPos): 112 """ 113 Beginning at startPos + 2, searches fileContents for the end of 114 a single line comment. If comment ends with a newline 115 character, the newline is not included in the comment. 116 117 MIB single line comment rules: 118 1. Begins with '--' 119 2. Ends with '--' or newline 120 3. Any characters between the beginning and end of the comment 121 are ignored as part of the comment, including quotes and 122 start/end delimiters for block comments ( '/*' and '*/') 123 124 @param startPos: character position of the beginning of the single 125 line comment within fileContents 126 @type fileContents: string 127 @return: startPos, endPos (character position of the last character 128 in the comment + 1) 129 @rtype: tuple (integer, integer) 130 """ 131 commentEndPosDash = fileContents.find('--', startPos + 2) 132 commentEndPosNewline = fileContents.find('\n', startPos + 2) 133 if commentEndPosDash != -1: 134 if commentEndPosNewline != -1: 135 if commentEndPosDash < commentEndPosNewline: 136 endPos = commentEndPosDash + 2 137 else: 138 endPos = commentEndPosNewline 139 else: 140 endPos = commentEndPosDash + 2 141 else: 142 if commentEndPosNewline != -1: 143 endPos = commentEndPosNewline 144 else: 145 endPos = len(fileContents) 146 147 return startPos, endPos

148 149 def findBlockCommentEndPos(searchStartPos): 150 """ 151 Beginning at startPos + 2, searches fileContents for the end of 152 a block comment. If block comments are nested, the 153 function interates into each block comment by calling itself. 154 155 MIB block comment rules: 156 1. Begins with '/*' 157 2. Ends with '*/' 158 3. Block comments can be nested 159 3. Any characters between the beginning and end of the comment 160 are ignored as part of the comment, including quotes and 161 start/end delimiters for single line comments ('--'). Newlines 162 are included as part of the block comment. 163 164 @param startPos: character position of the beginning of the block 165 comment within fileContents 166 @type fileContents: string 167 @return: startPos, endPos (character position of the last character 168 in the comment + 1) 169 @rtype: tuple (integer, integer) 170 """ 171 # Locate the next start and end markers 172 nextBlockStartPos = fileContents.find('/*', searchStartPos + 2) 173 nextBlockEndPos = fileContents.find('*/', searchStartPos + 2) 174 175 # If a nested comment exists, find the end 176 if nextBlockStartPos != -1 and \ 177 nextBlockStartPos < nextBlockEndPos: 178 nestedComment = findBlockCommentEndPos(nextBlockStartPos) 179 nextBlockEndPos = fileContents.find('*/', nestedComment[1]) 180 181 return searchStartPos, nextBlockEndPos + 2

182 183 # START removeMibComments 184 if not fileContents: 185 return fileContents 186 187 # Get rid of any lines that are completely made up of hyphens 188 fileContents = re.sub(r'[ \t]*-{2}[ \t]*$', '', fileContents) 189 190 # commentRanges holds a list of tuples in the form (startPos, endPos) 191 # that define the beginning and end of comments within fileContents 192 commentRanges = [] 193 searchStartPos = 0 # character position within fileContents 194 functions = {'SINGLE': findSingleLineCommentEndPos, 195 'BLOCK': findBlockCommentEndPos} 196 197 # Parse fileContents, looking for single line comments, block comments 198 # and string literals 199 while searchStartPos < len(fileContents): 200 # Find the beginning of the next occurrance of each item 201 singleLineStartPos = fileContents.find('--', searchStartPos) 202 blockStartPos = fileContents.find('/*', searchStartPos) 203 stringStartPos = fileContents.find('\"', searchStartPos) 204 205 nextItemPos = sys.maxint 206 nextItemType = '' 207 208 # Compare the next starting point for each item type. 209 if singleLineStartPos != -1 and \ 210 singleLineStartPos < nextItemPos: 211 nextItemPos = singleLineStartPos 212 nextItemType = 'SINGLE' 213 214 if blockStartPos != -1 and \ 215 blockStartPos < nextItemPos: 216 nextItemPos = blockStartPos 217 nextItemType = 'BLOCK' 218 219 # If the next item type is a string literal, just search for the 220 # next double quote and continue from there. This works because 221 # all double quotes (that are not part of a comment) appear in 222 # pairs. Even double-double quotes (escaped quotes) will work 223 # with this method since the first double quote will look like a 224 # string literal close quote and the second double quote will look 225 # like the beginning of a string literal. 226 if stringStartPos != -1 and \ 227 stringStartPos < nextItemPos: 228 newSearchStartPos = \ 229 fileContents.find('\"', stringStartPos + 1) + 1 230 if newSearchStartPos > searchStartPos: 231 searchStartPos = newSearchStartPos 232 else: # Weird error case 233 break 234 235 # If the next item is a comment, use the functions dictionary 236 # to call the appropriate function 237 elif nextItemPos != sys.maxint: 238 commentRange = functions[nextItemType](nextItemPos) 239 commentRanges.append(commentRange) 240 #searchStartPos = commentRange[1] 241 if commentRange[1] > 0: 242 searchStartPos = commentRange[1] 243 244 else: # No other items are found! 245 break 246 247 startPos = 0 248 mibParts = [] 249 250 # Iterate through each comment, adding the non-comment parts 251 # to mibParts. Finally, return the text without comments. 252 for commentRange in commentRanges: 253 mibParts.append(fileContents[startPos:(commentRange[0])]) 254 startPos = commentRange[1] 255 if startPos != len(fileContents): 256 mibParts.append(fileContents[startPos:(len(fileContents))]) 257 return ''.join(mibParts) 258

259 - def splitFileToMIBs(self, fileContents):

260 """ 261 Isolates each MIB definition in fileContents into a separate string 262 263 @param fileContents: the complete contents of a MIB file 264 @type fileContents: string 265 @return: MIB definition strings 266 @rtype: list of strings 267 """ 268 if fileContents is None: 269 return [] 270 271 DEFINITIONS = re.compile(r'([A-Za-z-0-9]+\s+DEFINITIONS' 272 '(\s+EXPLICIT TAGS|\s+IMPLICIT TAGS|\s+AUTOMATIC TAGS|\s*)' 273 '(\s+EXTENSIBILITY IMPLIED|\s*)\s*::=\s*BEGIN)') 274 275 definitionSpans = [] 276 for definitionMatch in DEFINITIONS.finditer(fileContents): 277 definitionSpans.append(list(definitionMatch.span())) 278 # If more than one definiton in the file, set the end of the 279 # last span to the beginning of the current span 280 if len(definitionSpans) > 1: 281 definitionSpans[-2][1] = definitionSpans[-1][0] 282 283 # Use the start and end positions to create a string for each 284 # MIB definition 285 mibDefinitions = [] 286 if definitionSpans: 287 # Set the end of the last span to the end of the file 288 definitionSpans[-1][1] = len(fileContents) 289 for definitionSpan in definitionSpans: 290 mibDefinitions.append( 291 fileContents[definitionSpan[0]:definitionSpan[1]]) 292 293 return mibDefinitions

294

295 - def mapMibToDependents(self, mibDefinitions):

296 # ASN.1 syntax regular expressions for declaring MIBs 297 # 298 # An example from http://www.faqs.org/rfcs/rfc1212.html 299 # 300 # RFC1213-MIB DEFINITIONS ::= BEGIN 301 # 302 # IMPORTS 303 # experimental, OBJECT-TYPE, Counter 304 # FROM RFC1155-SMI; 305 # 306 # -- a MIB may or may not have an IMPORTS section 307 # 308 # root OBJECT IDENTIFIER ::= { experimental xx } 309 # 310 # END 311 IMPORTS = re.compile(r'\sIMPORTS\s.+;', re.DOTALL) 312 DEPENDENCIES = re.compile( 313 r'\sFROM\s+(?P<dependency>[A-Za-z-0-9]+)') 314 315 mibDependencies = [] 316 for definition in mibDefinitions: 317 mibName = re.split(r'([A-Za-z0-9-]+)', definition)[1] 318 dependencies = set() 319 importsMatch = IMPORTS.search(definition) 320 if importsMatch: 321 imports = importsMatch.group() 322 for dependencyMatch in DEPENDENCIES.finditer(imports): 323 dependencies.add(dependencyMatch.group('dependency')) 324 self.mibs.append(mibName) 325 self.mibToDeps[mibName] = dependencies

326

327 -class PackageManager:

328 """ 329 Given an URL, filename or archive (eg zip, tar), extract the files from 330 the package and return a list of filenames. 331 """

332 - def __init__(self, log, downloaddir, extractdir):

333 """ 334 Initialize the packagae manager. 335 336 @parameter log: logging object 337 @type log: logging class object 338 @parameter downloaddir: directory name to store downloads 339 @type downloaddir: string 340 @parameter extractdir: directory name to store downloads 341 @type extractdir: string 342 """ 343 self.log = log 344 self.downloaddir = downloaddir 345 if self.downloaddir[-1] != '/': 346 self.downloaddir += '/' 347 self.extractdir = extractdir 348 if self.extractdir[-1] != '/': 349 self.extractdir += '/'

350

351 - def downloadExtract(self, url):

352 """ 353 Download and extract the list of filenames. 354 """ 355 try: 356 localFile = self.download(url) 357 except (SystemExit, KeyboardInterrupt): raise 358 except: 359 self.log.error("Problems downloading the file from %s: %s" % ( 360 url, sys.exc_info()[1] ) ) 361 return [] 362 self.log.debug("Will attempt to load %s", localFile) 363 try: 364 return self.processPackage(localFile) 365 except IOError, ex: 366 self.log.error("Unable to process '%s' because: %s", 367 localFile, str(ex)) 368 sys.exit(1)

369

370 - def download(self, url):

371 """ 372 Download the package from the given URL, or if it's a filename, 373 return the filename. 374 """ 375 urlParts = urlsplit(url) 376 schema = urlParts[0] 377 path = urlParts[2] 378 if not schema: 379 return os.path.abspath(url) 380 file = path.split(os.sep)[-1] 381 os.makedirs(self.downloaddir) 382 downloadFile = os.path.join(self.downloaddir, file) 383 self.log.debug("Downloading to file '%s'", downloadFile) 384 filename, _ = urllib.urlretrieve(url, downloadFile) 385 return filename

386

387 - def processPackage(self, pkgFileName):

388 """ 389 Figure out what type of file we have and extract out any 390 files and then enumerate the file names. 391 """ 392 self.log.debug("Determining file type of %s" % pkgFileName) 393 if zipfile.is_zipfile(pkgFileName): 394 return self.unbundlePackage(pkgFileName, self.unzip) 395 396 elif tarfile.is_tarfile(pkgFileName): 397 return self.unbundlePackage(pkgFileName, self.untar) 398 399 elif os.path.isdir(pkgFileName): 400 return self.processDir(pkgFileName) 401 402 else: 403 return [ os.path.abspath(pkgFileName) ]

404

405 - def unzip(self, file):

406 """ 407 Unzip the given file into the current directory and return 408 the directory in which files can be loaded. 409 """ 410 pkgZip= zipfile.ZipFile(file, 'r') 411 if pkgZip.testzip() != None: 412 self.log.error("File %s is corrupted -- please download again", file) 413 return 414 415 for file in pkgZip.namelist(): 416 self.log.debug("Unzipping file/dir %s..." % file) 417 try: 418 if re.search(r'/$', file) != None: 419 os.makedirs(file) 420 else: 421 contents = pkgZip.read(file) 422 try: 423 unzipped = open(file, "w") 424 except IOError: # Directory missing? 425 os.makedirs(os.path.dirname(file)) 426 unzipped = open(file, "w") 427 unzipped.write(contents) 428 unzipped.close() 429 except (SystemExit, KeyboardInterrupt): raise 430 except: 431 self.log.error("Error in extracting %s because %s" % ( 432 file, sys.exc_info()[1] ) ) 433 return 434 435 return os.getcwd()

436

437 - def untar(self, file):

438 """ 439 Given a tar file, extract from the tar into the current directory. 440 """ 441 try: 442 self.log.debug("Extracting files from tar...") 443 pkgTar = tarfile.open(file, 'r') 444 for tarInfo in pkgTar: 445 pkgTar.extract(tarInfo) 446 pkgTar.close() 447 except (SystemExit, KeyboardInterrupt): raise 448 except: 449 self.log.error("Error in un-tarring %s because %s" % ( file, 450 sys.exc_info()[1] ) ) 451 return 452 return os.getcwd()

453

454 - def processDir(self, dir):

455 """ 456 Note all of the files in a directory. 457 """ 458 fileList = [] 459 self.log.debug("Enumerating files in %s", dir) 460 if not os.path.isdir(dir): 461 self.log.debug("%s is not a directory", dir) 462 return [] 463 for directoryName, _, fileNames in os.walk(dir): 464 for fileName in fileNames: 465 fileList.append(os.path.join(directoryName, fileName)) 466 return fileList

467

468 - def unbundlePackage(self, package, unpackageMethod):

469 """ 470 Extract the files and then add to the list of files. 471 """ 472 self.makeExtractionDir() 473 baseDir = unpackageMethod(package) 474 if baseDir is not None: 475 return self.processDir(baseDir) 476 return []

477

478 - def makeExtractionDir(self):

479 """ 480 Create an uniquely named extraction directory starting from a base 481 extraction directory. 482 """ 483 try: 484 if not os.path.isdir(self.extractdir): 485 os.makedirs(self.extractdir) 486 extractDir = tempfile.mkdtemp(prefix=self.extractdir) 487 except (SystemExit, KeyboardInterrupt): raise 488 except: 489 self.log.error("Error in creating temp dir because %s", 490 sys.exc_info()[1] ) 491 sys.exit(1) 492 os.chdir(extractDir)

493

494 - def cleanup(self):

495 """ 496 Remove any clutter left over from the installation. 497 """ 498 self.cleanupDir(self.downloaddir) 499 self.cleanupDir(self.extractdir)

500

501 - def cleanupDir(self, dirName):

502 for root, dirs, files in os.walk(dirName, topdown=False): 503 if root == os.sep: # Should *never* get here 504 break 505 for name in files: 506 os.remove(os.path.join(root, name)) 507 for name in dirs: 508 try: 509 os.removedirs(os.path.join(root, name)) 510 except OSError: 511 pass 512 try: 513 os.removedirs(dirName) 514 except OSError: 515 pass

516 517

518 -class ZenMib(ZCmdBase):

519 """ 520 Wrapper around the smidump utilities used to convert MIB definitions into 521 python code which is in turn loaded into the DMD tree. 522 """

523 - def makeMibFileObj(self, fileName):

524 """ 525 Scan the MIB file to determine what MIBs are defined in the file and 526 what their dependencies are. 527 528 @param fileName: MIB fileName 529 @type fileName: string 530 @return: dependencyDict, indexDict 531 dependencyDict - a dictionary that associates MIB definitions 532 found in fileName with their dependencies 533 indexDict - a dictionary that associates MIB definitions with their 534 ordinal position within fileName} 535 @rtype: 536 dependencyDict = {mibName: [string list of MIB definition names 537 that mibName is dependant on]} 538 indexDict = {mibname: number} 539 """ 540 # Retrieve the entire contents of the MIB file 541 self.log.debug("Processing %s", fileName) 542 file = open(fileName) 543 fileContents = file.read() 544 file.close() 545 return MibFile(fileName, fileContents)

546

547 - def populateDependencyMap(self, importFileNames, depFileNames):

548 """ 549 Populates the self.mibToMibFile instance object with data. 550 Exit the program if we're missing any files. 551 552 @param importFileNames: fully qualified file names of MIB files to import 553 @type importFileNames: list of strings 554 @param depFileNames: fully qualified file names of all MIB files 555 @type depFileNames: list of strings 556 @return: mibFileObjects of files to import 557 @rtype: MibFile 558 """ 559 self.log.debug("Collecting MIB meta-data and creating depedency map.") 560 toImportMibFileObjs = [] 561 for fileName in depFileNames.union(importFileNames): 562 try: 563 mibFileObj = self.makeMibFileObj(fileName) 564 except IOError: 565 self.log.error("Couldn't open file %s", fileName) 566 continue 567 568 mibDependencies = mibFileObj.mibToDeps 569 if not mibDependencies: 570 self.log.warn("Unable to parse information from " 571 "%s -- skipping", fileName) 572 continue 573 574 if fileName in importFileNames: 575 toImportMibFileObjs.append(mibFileObj) 576 577 for mibName, dependencies in mibDependencies.items(): 578 self.mibToMibFile[mibName] = mibFileObj 579 return toImportMibFileObjs

580

581 - def getDependencyFileNames(self, mibFileObj):

582 """ 583 smidump needs to know the list of dependent files for a MIB file in 584 order to properly resolve external references. 585 586 @param mibFileObj: MibFile object 587 @type mibFileObj: MibFile 588 @return: list of dependency fileNames 589 @rtype: list of strings 590 """ 591 dependencies = [] 592 dependencyFileNames = set() 593 594 def dependencySearch(mibName): 595 """ 596 Create a list of files required by a MIB definition. 597 598 @param mibName: name of MIB definition 599 @type mibName: string 600 """ 601 dependencies.append(mibName) 602 mibFileObj = self.mibToMibFile.get(mibName) 603 if not mibFileObj: 604 self.log.warn("Unable to find a file that defines %s", mibName) 605 return 606 607 dependencyFileNames.add(mibFileObj.fileName) 608 for dependency in mibFileObj.mibToDeps[mibName]: 609 if dependency not in dependencies: 610 dependencySearch(dependency)

611 612 for mibName in mibFileObj.mibs: 613 dependencySearch(mibName) 614 615 dependencyFileNames.discard(mibFileObj.fileName) 616 return dependencyFileNames

617

618 - def savePythonCode(self, pythonCode, fileName):

619 """ 620 Stores the smidump-generated Python code to a file. 621 """ 622 if not os.path.exists(self.options.pythoncodedir): 623 self.options.keeppythoncode = False 624 self.log.warn('The directory %s to store converted MIB file code ' 625 'does not exist.' % self.options.pythoncodedir) 626 return 627 try: 628 pythonFileName = os.path.join(self.options.pythoncodedir, 629 os.path.basename(fileName) ) + '.py' 630 pythonFile = open(pythonFileName, 'w') 631 pythonFile.write(pythonCode) 632 pythonFile.close() 633 except (SystemExit, KeyboardInterrupt): raise 634 except: 635 self.log.warn('Could not output converted MIB to %s' % 636 pythonFileName)

637

638 - def generatePythonFromMib(self, fileName, dependencyFileNames, 639 mibNamesInFile):

640 """ 641 Use the smidump program to convert a MIB into Python code. 642 643 One major caveat: smidump by default only outputs the last MIB 644 definition in a file. For that matter, it always outputs the last MIB 645 definition in a file whether it is requested or not. Therefore, if 646 there are multiple MIB definitions in a file, all but the last must be 647 explicitly named on the command line. If you name the last, it will 648 come out twice. We don't want that. 649 650 OK, so we need to determine if there are multiple MIB definitions 651 in fileName and then add all but the last to the command line. That 652 works except the resulting python code will create a dictionary 653 for each MIB definition, all of them named MIB. Executing the code is 654 equivalent to running a=1; a=2; a=3. You only wind up with a=3. 655 Therefore, we separate each dictionary definition into its own string 656 and return a list of strings so each one can be executed individually. 657 658 @param fileName: name of the file containing MIB definitions 659 @type fileName: string 660 @param dependencyFileNames: list of fileNames that fileName is 661 dependent on 662 @type dependencyFileNames: list of strings 663 @param mibNamesInFile: names of MIB definitions in file 664 @type mibNamesInFile: list of strings 665 @return: list of dictionaries. Each dictionary containing the contents 666 of a MIB definition. [ {'mibName': MIB data} ] 667 @rtype: list 668 """ 669 def infiniteLoopHandler(signum, frame): 670 """ 671 Kills any smidump commands that have probably locked themselves 672 into an infinite loop. 673 """ 674 log.error("The command %s has probably gone into an infinite loop", 675 ' '.join(dumpCommand)) 676 log.error("Killing process id %s ...", proc.pid) 677 try: 678 os.kill(proc.pid, signal.SIGKILL) 679 except OSError: 680 pass

681 682 683 dumpCommand = ['smidump', '--keep-going', '--format', 'python'] 684 for dependencyFileName in dependencyFileNames: 685 # Add command-line flag for our dependent files 686 dumpCommand.append('--preload') 687 dumpCommand.append(dependencyFileName) 688 dumpCommand.append(fileName) 689 690 # If more than one MIB definition exists in the file, name all but the 691 # last on the command line. (See method description for reasons.) 692 if len(mibNamesInFile) > 1: 693 dumpCommand += mibNamesInFile[:-1] 694 695 self.log.debug('Running %s', ' '.join(dumpCommand)) 696 proc = Popen(dumpCommand, stdout=PIPE, stderr=PIPE) 697 698 log = self.log 699 signal.signal(signal.SIGALRM, infiniteLoopHandler) 700 signal.alarm(self.options.smidumptimeout) 701 pythonCode, warnings = proc.communicate() 702 proc.wait() 703 signal.alarm(0) # Disable the alarm 704 if proc.returncode: 705 if warnings.strip(): 706 self.log.error(warnings) 707 return None 708 709 if warnings: 710 self.log.debug("Found warnings while trying to import MIB:\n%s" \ 711 % warnings) 712 713 if self.options.keeppythoncode: 714 self.savePythonCode(pythonCode, fileName) 715 716 return self.evalPythonToMibs(pythonCode, fileName) 717

718 - def evalPythonToMibs(self, pythonCode, name):

719 """ 720 Evaluate the code and return an array of MIB dictionaries. 721 """ 722 def executePythonCode(pythonCode, name): 723 """ 724 Executes the python code generated smidump 725 726 @param pythonCode: Code generated by smidump 727 @type pythonCode: string 728 @return: a dictionary which contains one key: MIB 729 @rtype: dictionary 730 """ 731 result = {} 732 try: 733 exec pythonCode in result 734 except (SystemExit, KeyboardInterrupt): raise 735 except: 736 self.log.exception("Unable to import Pythonized-MIB: %s", 737 name) 738 return result.get('MIB', None)

739 740 # If more than one MIB definition exists in a file, pythonCode will 741 # contain a 'MIB = {...}' section for each MIB definition. We must 742 # split each section into its own string and return a string list. 743 smiMibDelim = 'MIB = {' 744 mibCodeParts = pythonCode.split(smiMibDelim) 745 mibDicts = [] 746 if len(mibCodeParts) > 1: 747 for mibCodePart in mibCodeParts[1:]: 748 mibDict = executePythonCode(smiMibDelim + mibCodePart, name) 749 if mibDict is not None: 750 mibDicts.append(mibDict) 751 else: 752 mibDict = executePythonCode(pythonCode, name) 753 if mibDict is not None: 754 mibDicts = [mibDict] 755 756 return mibDicts 757

758 - def evalAddSavedPythonCode(self):

759 """ 760 Read the file named in the command-line option, evaluate it, and add 761 it to our list of MIBs. 762 """ 763 pythonMibs = [] 764 for fileName in set(self.options.evalSavedPython): 765 try: 766 pythonCode = open(fileName).read() 767 except IOError: 768 self.log.warn("Unable to open '%s' -- skipping", 769 fileName) 770 continue 771 pythonMibs += self.evalPythonToMibs(pythonCode, fileName) 772 773 self.loadPythonMibs(pythonMibs)

774 775

776 - def getDmdMibDict(self, dmdMibDict, mibOrganizer):

777 """ 778 Populate a dictionary containing the MIB definitions that have been 779 loaded into the DMD Mibs directory 780 781 @param dmdMibDict: maps a MIB definition to the path where 782 it is located with in the DMD. 783 Format: 784 {'mibName': 'DMD path where mibName is stored'} 785 Example: MIB-Dell-10892 is located in the DMD tree at 786 Mibs/SITE/Dell, Directory entry is 787 {'MIB-Dell-10892': '/SITE/Dell'] } 788 @param mibOrganizer: the DMD directory to be searched 789 @type mibOrganizer: MibOrganizer 790 """ 791 organizerPath = mibOrganizer.getOrganizerName() 792 793 # Record each module from this organizer as a dictionary entry. 794 # mibOrganizer.mibs.objectItems() returns tuple: 795 # ('mibName', <MibModule at /zport/dmd/Mibs/...>) 796 for mibModule in mibOrganizer.mibs.objectItems(): 797 mibName = mibModule[0] 798 if mibName not in dmdMibDict: 799 dmdMibDict[mibName] = organizerPath 800 else: 801 self.log.warn('\nFound two copies of %s:' 802 ' %s and %s' % 803 (mibName, dmdMibDict[mibName], 804 mibOrganizer.getOrganizerName())) 805 806 # If there are suborganizers, recurse into them 807 for childOrganizer in mibOrganizer.children(): 808 self.getDmdMibDict(dmdMibDict, childOrganizer)

809

810 - def addMibEntries(self, leafType, pythonMib, mibModule):

811 """ 812 Add the different MIB leaves (ie nodes, notifications) into the DMD. 813 814 @paramater leafType: 'nodes', 'notifications' 815 @type leafType: string 816 @paramater pythonMib: dictionary of nodes and notifications 817 @type pythonMib: dictionary 818 @paramater mibModule: class containing functions to load leaves 819 @type mibModule: class 820 @return: number of leaves added 821 @rtype: int 822 """ 823 entriesAdded = 0 824 functor = { 'nodes':mibModule.createMibNode, 825 'notifications':mibModule.createMibNotification, 826 }.get(leafType, None) 827 if not functor or leafType not in pythonMib: 828 return entriesAdded 829 830 mibName = pythonMib['moduleName'] 831 832 for name, values in pythonMib[leafType].items(): 833 try: 834 functor(name, **values) 835 entriesAdded += 1 836 except BadRequest: 837 try: 838 self.log.warn("Unable to add %s id '%s' as this" 839 " name is reserved for use by Zope", 840 leafType, name) 841 newName = '_'.join([name, mibName]) 842 self.log.warn("Trying to add %s '%s' as '%s'", 843 leafType, name, newName) 844 functor(newName, **values) 845 entriesAdded += 1 846 self.log.warn("Renamed '%s' to '%s' and added to" 847 " MIB %s", name, newName, leafType) 848 except (SystemExit, KeyboardInterrupt): raise 849 except: 850 self.log.warn("Unable to add %s id '%s' -- skipping", 851 leafType, name) 852 else: 853 if not entriesAdded % CHUNK_SIZE: 854 self.commit("Loaded MIB %s into the DMD" % mibName) 855 self.commit("Loaded MIB %s into the DMD" % mibName) 856 return entriesAdded

857

858 - def loadMibFile(self, mibFileObj, dmdMibDict):

859 """ 860 Attempt to load the MIB definitions in fileName into DMD 861 862 @param fileName: name of the MIB file to be loaded 863 @type fileName: string 864 @return: whether the MIB load was successful or not 865 @rtype: boolean 866 """ 867 fileName = mibFileObj.fileName 868 self.log.debug('Attempting to load %s' % fileName) 869 870 # Check to see if any MIB definitions in fileName have already 871 # been loaded into Zenoss. If so, warn but don't fail 872 mibNamesInFile = mibFileObj.mibs 873 for mibName in mibNamesInFile: 874 if mibName in dmdMibDict: 875 dmdMibPath = dmdMibDict[mibName] 876 self.log.warn('MIB definition %s found in %s is already ' 877 'loaded at %s.' % (mibName, fileName, dmdMibPath)) 878 879 # Retrieve a list of all the files containing MIB definitions that are 880 # required by the MIB definitions in fileName 881 dependencyFileNames = self.getDependencyFileNames(mibFileObj) 882 883 # Convert the MIB file data into python dictionaries. pythonMibs 884 # contains a list of dictionaries, one for each MIB definition in 885 # fileName. 886 pythonMibs = self.generatePythonFromMib(fileName, dependencyFileNames, 887 mibNamesInFile) 888 if not pythonMibs: 889 return False 890 891 self.loadPythonMibs(pythonMibs) 892 return True

893

894 - def commit(self, message):

895 if not self.options.nocommit: 896 self.log.debug('Committing a batch of objects') 897 trans = transaction.get() 898 trans.setUser('zenmib') 899 trans.note(message) 900 trans.commit() 901 self.syncdb()

902

903 - def loadPythonMibs(self, pythonMibs):

904 """ 905 Walk through the MIB dictionaries and add the MIBs to the DMD. 906 """ 907 # Standard MIB attributes that we expect in all MIBs 908 MIB_MOD_ATTS = ('language', 'contact', 'description') 909 910 self.syncdb() 911 912 # Add the MIB data for each MIB into Zenoss 913 for pythonMib in pythonMibs: 914 mibName = pythonMib['moduleName'] 915 916 # Create the container for the MIBs and define meta-data. 917 # In the DMD this creates another container class which 918 # contains mibnodes. These data types are found in 919 # Products.ZenModel.MibModule and Products.ZenModel.MibNode 920 mibModule = self.dmd.Mibs.createMibModule( 921 mibName, self.options.path) 922 923 def gen(): 924 for key, val in pythonMib[mibName].iteritems(): 925 if key in MIB_MOD_ATTS: 926 yield key, val

927 928 for i, attr in enumerate(gen()): 929 setattr(mibModule, *attr) 930 if not i % CHUNK_SIZE: 931 self.commit("Loaded MIB %s into the DMD" % mibName) 932 self.commit("Loaded MIB %s into the DMD" % mibName) 933 934 nodesAdded = self.addMibEntries('nodes', pythonMib, mibModule) 935 trapsAdded = self.addMibEntries('notifications', pythonMib, mibModule) 936 self.log.info("Parsed %d nodes and %d notifications from %s", 937 nodesAdded, trapsAdded, mibName) 938 939 # Add the MIB tree permanently to the DMD unless --nocommit flag. 940 msg = "Loaded MIB %s into the DMD" % mibName 941 self.commit(msg) 942 if not self.options.nocommit: 943 self.log.info(msg) 944 945

946 - def getAllMibDepFileNames(self):

947 """ 948 Use command line parameters to create a list of files containing MIB 949 definitions that will be used as a reference list for the files being 950 loaded into the DMD 951 952 @return: set of file names 953 @rtype: set 954 """ 955 defaultMibDepDirs = [ 'ietf', 'iana', 'irtf', 'tubs', 'site' ] 956 mibDepFileNames = set() 957 for subdir in defaultMibDepDirs: 958 depDir = os.path.join(self.options.mibdepsdir, subdir) 959 mibDepFileNames.update(self.pkgMgr.processDir(depDir)) 960 return mibDepFileNames

961

962 - def getMibsToImport(self):

963 """ 964 Uses command-line parameters to create a list of files containing MIB 965 definitions that are to be loaded into the DMD 966 967 @return: list of file names that are to be loaded into the DMD 968 @rtype: list 969 """ 970 loadFileNames = [] 971 if self.args: 972 for fileName in self.args: 973 loadFileNames.extend(self.pkgMgr.downloadExtract(fileName)) 974 else: 975 loadFileNames = self.pkgMgr.processDir(self.options.mibsdir) 976 977 if loadFileNames: 978 self.log.debug("Will attempt to load the following files: %s", 979 loadFileNames) 980 else: 981 self.log.error("No MIB files to load!") 982 sys.exit(1) 983 984 return set(loadFileNames)

985

986 - def main(self):

987 """ 988 Main loop of the program 989 """ 990 if self.options.evalSavedPython: 991 self.evalAddSavedPythonCode() 992 return 993 994 # Verify MIBs search directory is valid. Fail if not 995 if not os.path.exists(self.options.mibsdir): 996 self.log.error("The directory %s doesn't exist!" % 997 self.options.mibsdir ) 998 sys.exit(1) 999 1000 self.pkgMgr = PackageManager(self.log, self.options.downloaddir, 1001 self.options.extractdir) 1002 self.mibToMibFile = {} 1003 1004 requestedFiles = self.getMibsToImport() 1005 mibDepFileNames = self.getAllMibDepFileNames() 1006 mibFileObjs = self.populateDependencyMap(requestedFiles, mibDepFileNames) 1007 1008 # dmdMibDict = {'mibName': 'DMD path to MIB'} 1009 dmdMibDict = {} 1010 self.getDmdMibDict(dmdMibDict, self.dmd.Mibs) 1011 1012 # Load the MIB files 1013 self.log.info("Found %d MIBs to import.", len(mibFileObjs)) 1014 loadedMibFiles = 0 1015 for mibFileObj in mibFileObjs: 1016 try: 1017 if self.loadMibFile(mibFileObj, dmdMibDict): 1018 loadedMibFiles += 1 1019 except (SystemExit, KeyboardInterrupt): raise 1020 except Exception, ex: 1021 self.log.exception("Failed to load MIB: %s", mibFileObj.fileName) 1022 1023 action = "Loaded" 1024 if self.options.nocommit: 1025 action = "Processed" 1026 1027 self.log.info("%s %d MIB file(s)" % (action, loadedMibFiles)) 1028 self.pkgMgr.cleanup() 1029 1030 sys.exit(0)

1031

1032 - def buildOptions(self):

1033 """ 1034 Command-line options 1035 """ 1036 ZCmdBase.buildOptions(self) 1037 self.parser.add_option('--mibsdir', 1038 dest='mibsdir', default=zenPath('share/mibs/site'), 1039 help="Directory of input MIB files [ default: %default ]") 1040 self.parser.add_option('--mibdepsdir', 1041 dest='mibdepsdir', default=zenPath('share/mibs'), 1042 help="Directory of input MIB files [ default: %default ]") 1043 self.parser.add_option('--path', 1044 dest='path', default="/", 1045 help="Path to load MIB into the DMD") 1046 self.parser.add_option('--nocommit', action='store_true', 1047 dest='nocommit', default=False, 1048 help="Don't commit the MIB to the DMD after loading") 1049 self.parser.add_option('--keeppythoncode', action='store_true', 1050 dest='keeppythoncode', default=False, 1051 help="Don't commit the MIB to the DMD after loading") 1052 self.parser.add_option('--pythoncodedir', dest='pythoncodedir', 1053 default=tempfile.gettempdir() + "/mib_pythoncode/", 1054 help="This is the directory where the converted MIB will be output. " \ 1055 "[ default: %default ]") 1056 self.parser.add_option('--downloaddir', dest='downloaddir', 1057 default=tempfile.gettempdir() + "/mib_downloads/", 1058 help="This is the directory where the MIB will be downloaded. " \ 1059 "[ default: %default ]") 1060 self.parser.add_option('--extractdir', dest='extractdir', 1061 default=tempfile.gettempdir() + "/mib_extract/", 1062 help="This is the directory where unzipped MIB files will be stored. " \ 1063 "[ default: %default ]") 1064 self.parser.add_option('--smidumptimeout', dest='smidumptimeout', 1065 default=60, 1066 help="Kill smidump after this many seconds to " \ 1067 "stop infinite loops.") 1068 self.parser.add_option('--evalSavedPython', dest='evalSavedPython', 1069 default=[], action='append', 1070 help="Execute the Python code previously generated" \ 1071 " and saved.")

1072 1073 1074 if __name__ == '__main__': 1075 zm = ZenMib() 1076 zm.main() 1077

Source Code for Module Products.ZenModel.zenmib