Products.ZenModel.zenmib

1 ############################################################################## 2 # 3 # Copyright (C) Zenoss, Inc. 2007, 2009, all rights reserved. 4 # 5 # This content is made available according to terms specified in 6 # License.zenoss under the directory where your Zenoss product is installed. 7 # 8 ############################################################################## 9 10 11 __doc__ = """zenmib 12 The zenmib program converts MIBs into python data structures and then 13 (by default) adds the data to the Zenoss DMD. Essentially, zenmib is a 14 wrapper program around the smidump program, whose output (python code) is 15 then executed "inside" the Zope database. 16 17 Overview of terms: 18 SNMP Simple Network Management Protocol 19 A network protocol originally based on UDP which allows a management 20 application (ie SNMP manager) to contact and get information from a 21 device (ie router, computer, network-capable toaster). The software 22 on the device that is capable of understanding SNMP and responding 23 appropriately is called an SNMP agent. 24 25 MIB Management of Information Base 26 A description of what a particular SNMP agent provides and what 27 traps it sends. A MIB is a part of a tree structure based on a root 28 MIB. Since a MIB is a rooted tree, it allows for delegation of areas 29 under the tree to different organizations. 30 31 ASN Abstract Syntax Notation 32 The notation used to construct a MIB. 33 34 OID Object IDentifier 35 A MIB is constructed of unique identifiers 36 37 38 Background information: 39 http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol 40 Overview of SNMP. 41 42 http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en 43 The libsmi project is the creator of smidump. There are several 44 interesting sub-projects available. 45 46 http://net-snmp.sourceforge.net/ 47 Homepage for Net-SNMP which is used by Zenoss for SNMP management. 48 49 http://www.et.put.poznan.pl/snmp/asn1/asn1.html 50 An overview of Abstract Syntax Notation (ASN), the language in 51 which MIBs are written. 52 """ 53 54 import os 55 import os.path 56 import sys 57 import re 58 from subprocess import Popen, PIPE 59 import tempfile 60 import urllib 61 import tarfile 62 import zipfile 63 from urlparse import urlsplit 64 from collections import OrderedDict 65 66 import Globals 67 import transaction 68 69 from Products.ZenUtils.ZCmdBase import ZCmdBase 70 from Products.ZenUtils.Utils import zenPath 71 from zExceptions import BadRequest 72 73 74 CHUNK_SIZE = 50 75 76

77 -class MibFile(object):

78 """ 79 A MIB file has the meta-data for a MIB inside of it. 80 """

81 - def __init__(self, fileName, fileContents=""):

82 self.fileName = fileName 83 self.mibs = [] # Order of MIBs defined in the file 84 self.mibToDeps = {} # Dependency defintions for each MIB 85 self.fileContents = self.removeMibComments(fileContents) 86 self.mibDefinitions = self.splitFileToMIBs(self.fileContents) 87 self.mapMibToDependents(self.mibDefinitions) 88 89 # Reclaim some memory 90 self.fileContents = "" 91 self.mibDefinitions = ""

92

93 - def removeMibComments(self, fileContents):

94 """ 95 Parses the string provided as an argument and extracts 96 all of the ASN.1 comments from the string. 97 98 Assumes that fileContents contains the contents of a well-formed 99 (no errors) MIB file. 100 101 @param fileContents: entire contents of a MIB file 102 @type fileContents: string 103 @return: text without any comments 104 @rtype: string 105 """ 106 if not fileContents: 107 return fileContents 108 109 cursor = 0 110 fc = [] 111 while cursor < len(fileContents): 112 searchParty = [ fileContents.find('"',cursor), fileContents.find('/*',cursor), fileContents.find('--',cursor) ] 113 try: 114 cursor = min( x for x in searchParty if x >= 0 ) 115 except ValueError: 116 break 117 118 if searchParty[0] == cursor: 119 ''' 120 MIB block quote rules 121 1. Begins with '"' 122 2. Ends with '"' 123 3. Quotes can be nested with '""' 124 4. Any characters between the beginning and the end of the string 125 are considered part of the string. All comment characters like 126 ('--') and ('/*...*/') are not ignored. Newlines are included 127 in the blockquote 128 ''' 129 cursor = fileContents.find('"',cursor+1) + 1 130 while 0 < cursor < len(fileContents) and fileContents[cursor] == '"': 131 cursor = fileContents.find('"', cursor + 1) + 1 132 133 # Syntax error if no close quote found 134 if not cursor: 135 raise ValueError("Syntax Error: missing close (\")") 136 137 continue 138 elif searchParty[1] == cursor: 139 ''' 140 MIB block comment rules: 141 1. Begins with '/*' 142 2. Ends with '*/' 143 3. Block comments can be nested 144 4. Any characters between the beginning and end of the comment 145 are ignored as part of the comment, including quotes and 146 start/end delimiters for single line comments ('--'). Newlines 147 are included as part of the block comment. 148 ''' 149 endComment = fileContents.find("*/",cursor+2) 150 if endComment < 0: 151 raise ValueError("Syntax Error: missing close (*/)") 152 153 nestCount = 1 154 # Check for a nested block 155 while nestCount != fileContents.count("/*", cursor, endComment): 156 157 # Corner case for /*/ 158 if fileContents[endComment-1] == '/': 159 fc_list = list(fileContents) 160 fc_list[endComment-1] = ' ' 161 fileContents = ''.join(fc_list) 162 endComment = fileContents.find("*/", endComment+2) 163 if endComment < 0: 164 raise ValueError("Syntax Error: missing close (*/)") 165 166 nestCount += 1 167 168 endComment += 2 169 elif searchParty[2] == cursor: 170 ''' 171 MIB single line comment rules: 172 1. Begins with '--' 173 2. Ends with '--' or newline 174 3. Any characters between the beginning and end of the comment 175 are ignored as part of the comment, including quotes and 176 start/end delimiters for block comments ( '/*' and '*/') 177 ''' 178 endComment = fileContents.find('\n',cursor+2) 179 180 #EOF reached 181 if endComment < 0: 182 endComment = len(fileContents) 183 184 # Ignore dash lines 185 dashline = fileContents[cursor:endComment].strip() 186 if dashline != '-' * len(dashline): 187 ec = fileContents.find('--', cursor+2, endComment) 188 if ec > -1: 189 endComment = ec+2 190 191 fc.append(fileContents[:cursor]) 192 fileContents = fileContents[endComment:] 193 cursor = 0 194 195 fc.append(fileContents) 196 return ''.join(fc)

197

198 - def splitFileToMIBs(self, fileContents):

199 """ 200 Isolates each MIB definition in fileContents into a separate string 201 202 @param fileContents: the complete contents of a MIB file 203 @type fileContents: string 204 @return: MIB definition strings 205 @rtype: list of strings 206 """ 207 if fileContents is None: 208 return [] 209 210 DEFINITIONS = re.compile(r'([A-Za-z-0-9]+\s+DEFINITIONS' 211 '(\s+EXPLICIT TAGS|\s+IMPLICIT TAGS|\s+AUTOMATIC TAGS|\s*)' 212 '(\s+EXTENSIBILITY IMPLIED|\s*)\s*::=\s*BEGIN)') 213 214 definitionSpans = [] 215 for definitionMatch in DEFINITIONS.finditer(fileContents): 216 definitionSpans.append(list(definitionMatch.span())) 217 # If more than one definiton in the file, set the end of the 218 # last span to the beginning of the current span 219 if len(definitionSpans) > 1: 220 definitionSpans[-2][1] = definitionSpans[-1][0] 221 222 # Use the start and end positions to create a string for each 223 # MIB definition 224 mibDefinitions = [] 225 if definitionSpans: 226 # Set the end of the last span to the end of the file 227 definitionSpans[-1][1] = len(fileContents) 228 for definitionSpan in definitionSpans: 229 mibDefinitions.append( 230 fileContents[definitionSpan[0]:definitionSpan[1]]) 231 232 return mibDefinitions

233

234 - def mapMibToDependents(self, mibDefinitions):

235 # ASN.1 syntax regular expressions for declaring MIBs 236 # 237 # An example from http://www.faqs.org/rfcs/rfc1212.html 238 # 239 # RFC1213-MIB DEFINITIONS ::= BEGIN 240 # 241 # IMPORTS 242 # experimental, OBJECT-TYPE, Counter 243 # FROM RFC1155-SMI; 244 # 245 # -- a MIB may or may not have an IMPORTS section 246 # 247 # root OBJECT IDENTIFIER ::= { experimental xx } 248 # 249 # END 250 IMPORTS = re.compile(r'\sIMPORTS\s.+;', re.DOTALL) 251 DEPENDENCIES = re.compile( 252 r'\sFROM\s+(?P<dependency>[A-Za-z-0-9]+)') 253 254 for definition in mibDefinitions: 255 mibName = re.split(r'([A-Za-z0-9-]+)', definition)[1] 256 dependencies = set() 257 importsMatch = IMPORTS.search(definition) 258 if importsMatch: 259 imports = importsMatch.group() 260 for dependencyMatch in DEPENDENCIES.finditer(imports): 261 dependencies.add(dependencyMatch.group('dependency')) 262 self.mibs.append(mibName) 263 self.mibToDeps[mibName] = dependencies

264

265 -class PackageManager:

266 """ 267 Given an URL, filename or archive (eg zip, tar), extract the files from 268 the package and return a list of filenames. 269 """

270 - def __init__(self, log, downloaddir, extractdir):

271 """ 272 Initialize the packagae manager. 273 274 @parameter log: logging object 275 @type log: logging class object 276 @parameter downloaddir: directory name to store downloads 277 @type downloaddir: string 278 @parameter extractdir: directory name to store downloads 279 @type extractdir: string 280 """ 281 self.log = log 282 self.downloaddir = downloaddir 283 if self.downloaddir[-1] != '/': 284 self.downloaddir += '/' 285 self.extractdir = extractdir 286 if self.extractdir[-1] != '/': 287 self.extractdir += '/'

288

289 - def downloadExtract(self, url):

290 """ 291 Download and extract the list of filenames. 292 """ 293 try: 294 localFile = self.download(url) 295 except Exception: 296 self.log.error("Problems downloading the file from %s: %s" % ( 297 url, sys.exc_info()[1] ) ) 298 return [] 299 self.log.debug("Will attempt to load %s", localFile) 300 try: 301 return self.processPackage(localFile) 302 except IOError, ex: 303 self.log.error("Unable to process '%s' because: %s", 304 localFile, str(ex)) 305 sys.exit(1)

306

307 - def download(self, url):

308 """ 309 Download the package from the given URL, or if it's a filename, 310 return the filename. 311 """ 312 urlParts = urlsplit(url) 313 schema = urlParts[0] 314 path = urlParts[2] 315 if not schema: 316 return os.path.abspath(url) 317 file = path.split(os.sep)[-1] 318 os.makedirs(self.downloaddir) 319 downloadFile = os.path.join(self.downloaddir, file) 320 self.log.debug("Downloading to file '%s'", downloadFile) 321 filename, _ = urllib.urlretrieve(url, downloadFile) 322 return filename

323

324 - def processPackage(self, pkgFileName):

325 """ 326 Figure out what type of file we have and extract out any 327 files and then enumerate the file names. 328 """ 329 self.log.debug("Determining file type of %s" % pkgFileName) 330 if zipfile.is_zipfile(pkgFileName): 331 return self.unbundlePackage(pkgFileName, self.unzip) 332 333 elif tarfile.is_tarfile(pkgFileName): 334 return self.unbundlePackage(pkgFileName, self.untar) 335 336 elif os.path.isdir(pkgFileName): 337 return self.processDir(pkgFileName) 338 339 else: 340 return [ os.path.abspath(pkgFileName) ]

341

342 - def unzip(self, file):

343 """ 344 Unzip the given file into the current directory and return 345 the directory in which files can be loaded. 346 """ 347 pkgZip= zipfile.ZipFile(file, 'r') 348 if pkgZip.testzip() != None: 349 self.log.error("File %s is corrupted -- please download again", file) 350 return 351 352 for file in pkgZip.namelist(): 353 self.log.debug("Unzipping file/dir %s..." % file) 354 try: 355 if re.search(r'/$', file) != None: 356 os.makedirs(file) 357 else: 358 contents = pkgZip.read(file) 359 try: 360 unzipped = open(file, "w") 361 except IOError: # Directory missing? 362 os.makedirs(os.path.dirname(file)) 363 unzipped = open(file, "w") 364 unzipped.write(contents) 365 unzipped.close() 366 except (SystemExit, KeyboardInterrupt): raise 367 except: 368 self.log.error("Error in extracting %s because %s" % ( 369 file, sys.exc_info()[1] ) ) 370 return 371 372 return os.getcwd()

373

374 - def untar(self, file):

375 """ 376 Given a tar file, extract from the tar into the current directory. 377 """ 378 try: 379 self.log.debug("Extracting files from tar...") 380 pkgTar = tarfile.open(file, 'r') 381 for tarInfo in pkgTar: 382 pkgTar.extract(tarInfo) 383 pkgTar.close() 384 except (SystemExit, KeyboardInterrupt): raise 385 except: 386 self.log.error("Error in un-tarring %s because %s" % ( file, 387 sys.exc_info()[1] ) ) 388 return 389 return os.getcwd()

390

391 - def processDir(self, dir):

392 """ 393 Note all of the files in a directory. 394 """ 395 fileList = [] 396 self.log.debug("Enumerating files in %s", dir) 397 if not os.path.isdir(dir): 398 self.log.debug("%s is not a directory", dir) 399 return [] 400 for directoryName, _, fileNames in os.walk(dir): 401 for fileName in fileNames: 402 fileList.append(os.path.join(directoryName, fileName)) 403 return fileList

404

405 - def unbundlePackage(self, package, unpackageMethod):

406 """ 407 Extract the files and then add to the list of files. 408 """ 409 self.makeExtractionDir() 410 baseDir = unpackageMethod(package) 411 if baseDir is not None: 412 return self.processDir(baseDir) 413 return []

414

415 - def makeExtractionDir(self):

416 """ 417 Create an uniquely named extraction directory starting from a base 418 extraction directory. 419 """ 420 try: 421 if not os.path.isdir(self.extractdir): 422 os.makedirs(self.extractdir) 423 extractDir = tempfile.mkdtemp(prefix=self.extractdir) 424 except (SystemExit, KeyboardInterrupt): raise 425 except: 426 self.log.error("Error in creating temp dir because %s", 427 sys.exc_info()[1] ) 428 sys.exit(1) 429 os.chdir(extractDir)

430

431 - def cleanup(self):

432 """ 433 Remove any clutter left over from the installation. 434 """ 435 self.cleanupDir(self.downloaddir) 436 self.cleanupDir(self.extractdir)

437

438 - def cleanupDir(self, dirName):

439 for root, dirs, files in os.walk(dirName, topdown=False): 440 if root == os.sep: # Should *never* get here 441 break 442 for name in files: 443 os.remove(os.path.join(root, name)) 444 for name in dirs: 445 try: 446 os.removedirs(os.path.join(root, name)) 447 except OSError: 448 pass 449 try: 450 os.removedirs(dirName) 451 except OSError: 452 pass

453 454

455 -class ZenMib(ZCmdBase):

456 """ 457 Wrapper around the smidump utilities used to convert MIB definitions into 458 python code which is in turn loaded into the DMD tree. 459 """

460 - def makeMibFileObj(self, fileName):

461 """ 462 Scan the MIB file to determine what MIBs are defined in the file and 463 what their dependencies are. 464 465 @param fileName: MIB fileName 466 @type fileName: string 467 @return: dependencyDict, indexDict 468 dependencyDict - a dictionary that associates MIB definitions 469 found in fileName with their dependencies 470 indexDict - a dictionary that associates MIB definitions with their 471 ordinal position within fileName} 472 @rtype: 473 dependencyDict = {mibName: [string list of MIB definition names 474 that mibName is dependant on]} 475 indexDict = {mibname: number} 476 """ 477 # Retrieve the entire contents of the MIB file 478 self.log.debug("Processing %s", fileName) 479 file = open(fileName) 480 fileContents = file.read() 481 file.close() 482 return MibFile(fileName, fileContents)

483

484 - def populateDependencyMap(self, importFileNames, depFileNames):

485 """ 486 Populates the self.mibToMibFile instance object with data. 487 Exit the program if we're missing any files. 488 489 @param importFileNames: fully qualified file names of MIB files to import 490 @type importFileNames: list of strings 491 @param depFileNames: fully qualified file names of all MIB files 492 @type depFileNames: list of strings 493 @return: mibFileObjects of files to import 494 @rtype: MibFile 495 """ 496 self.log.debug("Collecting MIB meta-data and creating depedency map.") 497 toImportMibFileObjs = [] 498 for fileName in depFileNames.union(importFileNames): 499 try: 500 mibFileObj = self.makeMibFileObj(fileName) 501 except IOError: 502 self.log.error("Couldn't open file %s", fileName) 503 continue 504 505 mibDependencies = mibFileObj.mibToDeps 506 if not mibDependencies: 507 self.log.warn("Unable to parse information from " 508 "%s -- skipping", fileName) 509 continue 510 511 if fileName in importFileNames: 512 toImportMibFileObjs.append(mibFileObj) 513 514 for mibName, dependencies in mibDependencies.items(): 515 self.mibToMibFile[mibName] = mibFileObj 516 return toImportMibFileObjs

517

518 - def getDependencyFileNames(self, mibFileObj):

519 """ 520 smidump needs to know the list of dependent files for a MIB file in 521 order to properly resolve external references. 522 523 @param mibFileObj: MibFile object 524 @type mibFileObj: MibFile 525 @return: list of dependency fileNames 526 @rtype: list of strings 527 """ 528 dependencies = [] 529 dependencyFileNames = OrderedDict() 530 531 def dependencySearch(mibName): 532 """ 533 Create a list of files required by a MIB definition. 534 535 @param mibName: name of MIB definition 536 @type mibName: string 537 """ 538 dependencies.append(mibName) 539 mibFileObj = self.mibToMibFile.get(mibName) 540 if not mibFileObj: 541 self.log.warn("Unable to find a file that defines %s", mibName) 542 return 543 544 dependencyFileNames[mibFileObj.fileName] = None 545 for dependency in mibFileObj.mibToDeps[mibName]: 546 if dependency not in dependencies: 547 dependencySearch(dependency)

548 549 for mibName in mibFileObj.mibs: 550 dependencySearch(mibName) 551 552 if mibFileObj.fileName in dependencyFileNames: 553 del dependencyFileNames[mibFileObj.fileName] 554 names = dependencyFileNames.keys() 555 names.reverse() 556 return names

557

558 - def savePythonCode(self, pythonCode, fileName):

559 """ 560 Stores the smidump-generated Python code to a file. 561 """ 562 if not os.path.exists(self.options.pythoncodedir): 563 self.options.keeppythoncode = False 564 self.log.warn('The directory %s to store converted MIB file code ' 565 'does not exist.' % self.options.pythoncodedir) 566 return 567 try: 568 pythonFileName = os.path.join(self.options.pythoncodedir, 569 os.path.basename(fileName) ) + '.py' 570 pythonFile = open(pythonFileName, 'w') 571 pythonFile.write(pythonCode) 572 pythonFile.close() 573 except (SystemExit, KeyboardInterrupt): raise 574 except: 575 self.log.warn('Could not output converted MIB to %s' % 576 pythonFileName)

577

578 - def generatePythonFromMib(self, fileName, dependencyFileNames, 579 mibNamesInFile):

580 """ 581 Use the smidump program to convert a MIB into Python code. 582 583 One major caveat: smidump by default only outputs the last MIB 584 definition in a file. For that matter, it always outputs the last MIB 585 definition in a file whether it is requested or not. Therefore, if 586 there are multiple MIB definitions in a file, all but the last must be 587 explicitly named on the command line. If you name the last, it will 588 come out twice. We don't want that. 589 590 OK, so we need to determine if there are multiple MIB definitions 591 in fileName and then add all but the last to the command line. That 592 works except the resulting python code will create a dictionary 593 for each MIB definition, all of them named MIB. Executing the code is 594 equivalent to running a=1; a=2; a=3. You only wind up with a=3. 595 Therefore, we separate each dictionary definition into its own string 596 and return a list of strings so each one can be executed individually. 597 598 @param fileName: name of the file containing MIB definitions 599 @type fileName: string 600 @param dependencyFileNames: list of fileNames that fileName is 601 dependent on 602 @type dependencyFileNames: list of strings 603 @param mibNamesInFile: names of MIB definitions in file 604 @type mibNamesInFile: list of strings 605 @return: list of dictionaries. Each dictionary containing the contents 606 of a MIB definition. [ {'mibName': MIB data} ] 607 @rtype: list 608 """ 609 dumpCommand = ['smidump', '--keep-going', '--format', 'python'] 610 for dependencyFileName in dependencyFileNames: 611 # Add command-line flag for our dependent files 612 dumpCommand.append('--preload') 613 dumpCommand.append(dependencyFileName) 614 dumpCommand.append(fileName) 615 616 # If more than one MIB definition exists in the file, name all but the 617 # last on the command line. (See method description for reasons.) 618 if len(mibNamesInFile) > 1: 619 dumpCommand += mibNamesInFile[:-1] 620 621 self.log.debug('Running %s', ' '.join(dumpCommand)) 622 sys.stdout.flush() 623 proc = Popen(dumpCommand, stdout=PIPE, stderr=PIPE, close_fds=True) 624 625 # If the child process generates enough output to the PIPE, 626 # then proc.wait() blocks waiting for the OS pipe buffer to accept more data 627 # (ie forever). Poll from time to time to avoid the problem with large MIBs. 628 pythonCode = '' 629 warnings = '' 630 while proc.poll() is None: 631 output, err = proc.communicate() 632 pythonCode += output 633 warnings += err 634 635 if proc.poll() != 0 or proc.returncode: 636 if warnings.strip(): 637 self.log.error(warnings) 638 return None 639 640 if warnings: 641 self.log.debug("Found warnings while trying to import MIB:\n%s" \ 642 % warnings) 643 644 if self.options.keeppythoncode: 645 self.savePythonCode(pythonCode, fileName) 646 647 return self.evalPythonToMibs(pythonCode, fileName)

648

649 - def evalPythonToMibs(self, pythonCode, name):

650 """ 651 Evaluate the code and return an array of MIB dictionaries. 652 """ 653 def executePythonCode(pythonCode, name): 654 """ 655 Executes the python code generated smidump 656 657 @param pythonCode: Code generated by smidump 658 @type pythonCode: string 659 @return: a dictionary which contains one key: MIB 660 @rtype: dictionary 661 """ 662 result = {} 663 try: 664 exec pythonCode in result 665 except (SystemExit, KeyboardInterrupt): raise 666 except: 667 self.log.exception("Unable to import Pythonized-MIB: %s", 668 name) 669 return result.get('MIB', None)

670 671 # If more than one MIB definition exists in a file, pythonCode will 672 # contain a 'MIB = {...}' section for each MIB definition. We must 673 # split each section into its own string and return a string list. 674 smiMibDelim = 'MIB = {' 675 mibCodeParts = pythonCode.split(smiMibDelim) 676 mibDicts = [] 677 if len(mibCodeParts) > 1: 678 for mibCodePart in mibCodeParts[1:]: 679 mibDict = executePythonCode(smiMibDelim + mibCodePart, name) 680 if mibDict is not None: 681 mibDicts.append(mibDict) 682 else: 683 mibDict = executePythonCode(pythonCode, name) 684 if mibDict is not None: 685 mibDicts = [mibDict] 686 687 return mibDicts 688

689 - def evalAddSavedPythonCode(self):

690 """ 691 Read the file named in the command-line option, evaluate it, and add 692 it to our list of MIBs. 693 """ 694 pythonMibs = [] 695 for fileName in set(self.options.evalSavedPython): 696 try: 697 pythonCode = open(fileName).read() 698 except IOError: 699 self.log.warn("Unable to open '%s' -- skipping", 700 fileName) 701 continue 702 pythonMibs += self.evalPythonToMibs(pythonCode, fileName) 703 704 self.loadPythonMibs(pythonMibs)

705 706

707 - def getDmdMibDict(self, dmdMibDict, mibOrganizer):

708 """ 709 Populate a dictionary containing the MIB definitions that have been 710 loaded into the DMD Mibs directory 711 712 @param dmdMibDict: maps a MIB definition to the path where 713 it is located with in the DMD. 714 Format: 715 {'mibName': 'DMD path where mibName is stored'} 716 Example: MIB-Dell-10892 is located in the DMD tree at 717 Mibs/SITE/Dell, Directory entry is 718 {'MIB-Dell-10892': '/SITE/Dell'] } 719 @param mibOrganizer: the DMD directory to be searched 720 @type mibOrganizer: MibOrganizer 721 """ 722 organizerPath = mibOrganizer.getOrganizerName() 723 724 # Record each module from this organizer as a dictionary entry. 725 # mibOrganizer.mibs.objectItems() returns tuple: 726 # ('mibName', <MibModule at /zport/dmd/Mibs/...>) 727 for mibModule in mibOrganizer.mibs.objectItems(): 728 mibName = mibModule[0] 729 if mibName not in dmdMibDict: 730 dmdMibDict[mibName] = organizerPath 731 else: 732 self.log.warn('\nFound two copies of %s:' 733 ' %s and %s' % 734 (mibName, dmdMibDict[mibName], 735 mibOrganizer.getOrganizerName())) 736 737 # If there are suborganizers, recurse into them 738 for childOrganizer in mibOrganizer.children(): 739 self.getDmdMibDict(dmdMibDict, childOrganizer)

740

741 - def addMibEntries(self, leafType, pythonMib, mibModule):

742 """ 743 Add the different MIB leaves (ie nodes, notifications) into the DMD. 744 745 @paramater leafType: 'nodes', 'notifications' 746 @type leafType: string 747 @paramater pythonMib: dictionary of nodes and notifications 748 @type pythonMib: dictionary 749 @paramater mibModule: class containing functions to load leaves 750 @type mibModule: class 751 @return: number of leaves added 752 @rtype: int 753 """ 754 entriesAdded = 0 755 functor = { 'nodes':mibModule.createMibNode, 756 'notifications':mibModule.createMibNotification, 757 }.get(leafType, None) 758 if not functor or leafType not in pythonMib: 759 return entriesAdded 760 761 mibName = pythonMib['moduleName'] 762 763 for name, values in pythonMib[leafType].items(): 764 try: 765 functor(name, **values) 766 entriesAdded += 1 767 except BadRequest: 768 try: 769 self.log.warn("Unable to add %s id '%s' as this" 770 " name is reserved for use by Zope", 771 leafType, name) 772 newName = '_'.join([name, mibName]) 773 self.log.warn("Trying to add %s '%s' as '%s'", 774 leafType, name, newName) 775 functor(newName, **values) 776 entriesAdded += 1 777 self.log.warn("Renamed '%s' to '%s' and added to" 778 " MIB %s", name, newName, leafType) 779 except (SystemExit, KeyboardInterrupt): raise 780 except: 781 self.log.warn("Unable to add %s id '%s' -- skipping", 782 leafType, name) 783 else: 784 if not entriesAdded % CHUNK_SIZE: 785 self.commit("Loaded MIB %s into the DMD" % mibName) 786 self.commit("Loaded MIB %s into the DMD" % mibName) 787 return entriesAdded

788

789 - def loadMibFile(self, mibFileObj, dmdMibDict):

790 """ 791 Attempt to load the MIB definitions in fileName into DMD 792 793 @param fileName: name of the MIB file to be loaded 794 @type fileName: string 795 @return: whether the MIB load was successful or not 796 @rtype: boolean 797 """ 798 fileName = mibFileObj.fileName 799 self.log.debug('Attempting to load %s' % fileName) 800 801 # Check to see if any MIB definitions in fileName have already 802 # been loaded into Zenoss. If so, warn but don't fail 803 mibNamesInFile = mibFileObj.mibs 804 for mibName in mibNamesInFile: 805 if mibName in dmdMibDict: 806 dmdMibPath = dmdMibDict[mibName] 807 self.log.warn('MIB definition %s found in %s is already ' 808 'loaded at %s.' % (mibName, fileName, dmdMibPath)) 809 810 # Retrieve a list of all the files containing MIB definitions that are 811 # required by the MIB definitions in fileName 812 dependencyFileNames = self.getDependencyFileNames(mibFileObj) 813 814 # Convert the MIB file data into python dictionaries. pythonMibs 815 # contains a list of dictionaries, one for each MIB definition in 816 # fileName. 817 pythonMibs = self.generatePythonFromMib(fileName, dependencyFileNames, 818 mibNamesInFile) 819 if not pythonMibs: 820 return False 821 822 self.loadPythonMibs(pythonMibs) 823 return True

824

825 - def commit(self, message):

826 if not self.options.nocommit: 827 self.log.debug('Committing a batch of objects') 828 trans = transaction.get() 829 trans.setUser('zenmib') 830 trans.note(message) 831 trans.commit() 832 self.syncdb()

833

834 - def loadPythonMibs(self, pythonMibs):

835 """ 836 Walk through the MIB dictionaries and add the MIBs to the DMD. 837 """ 838 # Standard MIB attributes that we expect in all MIBs 839 MIB_MOD_ATTS = ('language', 'contact', 'description') 840 841 self.syncdb() 842 843 # Add the MIB data for each MIB into Zenoss 844 for pythonMib in pythonMibs: 845 mibName = pythonMib['moduleName'] 846 847 # Create the container for the MIBs and define meta-data. 848 # In the DMD this creates another container class which 849 # contains mibnodes. These data types are found in 850 # Products.ZenModel.MibModule and Products.ZenModel.MibNode 851 mibModule = self.dmd.Mibs.createMibModule( 852 mibName, self.options.path) 853 854 def gen(): 855 for key, val in pythonMib[mibName].iteritems(): 856 if key in MIB_MOD_ATTS: 857 yield key, val

858 859 for i, attr in enumerate(gen()): 860 setattr(mibModule, *attr) 861 if not i % CHUNK_SIZE: 862 self.commit("Loaded MIB %s into the DMD" % mibName) 863 self.commit("Loaded MIB %s into the DMD" % mibName) 864 865 nodesAdded = self.addMibEntries('nodes', pythonMib, mibModule) 866 trapsAdded = self.addMibEntries('notifications', pythonMib, mibModule) 867 self.log.info("Parsed %d nodes and %d notifications from %s", 868 nodesAdded, trapsAdded, mibName) 869 870 # Add the MIB tree permanently to the DMD unless --nocommit flag. 871 msg = "Loaded MIB %s into the DMD" % mibName 872 self.commit(msg) 873 if not self.options.nocommit: 874 self.log.info(msg) 875 876

877 - def getAllMibDepFileNames(self):

878 """ 879 Use command line parameters to create a list of files containing MIB 880 definitions that will be used as a reference list for the files being 881 loaded into the DMD 882 883 @return: set of file names 884 @rtype: set 885 """ 886 defaultMibDepDirs = [ 'ietf', 'iana', 'irtf', 'tubs', 'site' ] 887 mibDepFileNames = set() 888 for subdir in defaultMibDepDirs: 889 depDir = os.path.join(self.options.mibdepsdir, subdir) 890 mibDepFileNames.update(self.pkgMgr.processDir(depDir)) 891 return mibDepFileNames

892

893 - def getMibsToImport(self):

894 """ 895 Uses command-line parameters to create a list of files containing MIB 896 definitions that are to be loaded into the DMD 897 898 @return: list of file names that are to be loaded into the DMD 899 @rtype: list 900 """ 901 loadFileNames = [] 902 if self.args: 903 for fileName in self.args: 904 loadFileNames.extend(self.pkgMgr.downloadExtract(fileName)) 905 else: 906 loadFileNames = self.pkgMgr.processDir(self.options.mibsdir) 907 908 if loadFileNames: 909 self.log.debug("Will attempt to load the following files: %s", 910 loadFileNames) 911 else: 912 self.log.error("No MIB files to load!") 913 sys.exit(1) 914 915 return set(loadFileNames)

916

917 - def main(self):

918 """ 919 Main loop of the program 920 """ 921 if self.options.evalSavedPython: 922 self.evalAddSavedPythonCode() 923 return 924 925 # Verify MIBs search directory is valid. Fail if not 926 if not os.path.exists(self.options.mibsdir): 927 self.log.error("The directory %s doesn't exist!" % 928 self.options.mibsdir ) 929 sys.exit(1) 930 931 self.pkgMgr = PackageManager(self.log, self.options.downloaddir, 932 self.options.extractdir) 933 self.mibToMibFile = {} 934 935 requestedFiles = self.getMibsToImport() 936 mibDepFileNames = self.getAllMibDepFileNames() 937 mibFileObjs = self.populateDependencyMap(requestedFiles, mibDepFileNames) 938 939 # dmdMibDict = {'mibName': 'DMD path to MIB'} 940 dmdMibDict = {} 941 self.getDmdMibDict(dmdMibDict, self.dmd.Mibs) 942 943 # Load the MIB files 944 self.log.info("Found %d MIBs to import.", len(mibFileObjs)) 945 loadedMibFiles = 0 946 for mibFileObj in mibFileObjs: 947 try: 948 if self.loadMibFile(mibFileObj, dmdMibDict): 949 loadedMibFiles += 1 950 except Exception: 951 self.log.exception("Failed to load MIB: %s", mibFileObj.fileName) 952 953 action = "Loaded" 954 if self.options.nocommit: 955 action = "Processed" 956 957 self.log.info("%s %d MIB file(s)" % (action, loadedMibFiles)) 958 self.pkgMgr.cleanup() 959 960 sys.exit(0)

961

962 - def buildOptions(self):

963 """ 964 Command-line options 965 """ 966 ZCmdBase.buildOptions(self) 967 self.parser.add_option('--mibsdir', 968 dest='mibsdir', default=zenPath('share/mibs/site'), 969 help="Directory of input MIB files [ default: %default ]") 970 self.parser.add_option('--mibdepsdir', 971 dest='mibdepsdir', default=zenPath('share/mibs'), 972 help="Directory of input MIB files [ default: %default ]") 973 self.parser.add_option('--path', 974 dest='path', default="/", 975 help="Path to load MIB into the DMD") 976 self.parser.add_option('--nocommit', action='store_true', 977 dest='nocommit', default=False, 978 help="Don't commit the MIB to the DMD after loading") 979 self.parser.add_option('--keeppythoncode', action='store_true', 980 dest='keeppythoncode', default=False, 981 help="Don't commit the MIB to the DMD after loading") 982 self.parser.add_option('--pythoncodedir', dest='pythoncodedir', 983 default=tempfile.gettempdir() + "/mib_pythoncode/", 984 help="This is the directory where the converted MIB will be output. " \ 985 "[ default: %default ]") 986 self.parser.add_option('--downloaddir', dest='downloaddir', 987 default=tempfile.gettempdir() + "/mib_downloads/", 988 help="This is the directory where the MIB will be downloaded. " \ 989 "[ default: %default ]") 990 self.parser.add_option('--extractdir', dest='extractdir', 991 default=tempfile.gettempdir() + "/mib_extract/", 992 help="This is the directory where unzipped MIB files will be stored. " \ 993 "[ default: %default ]") 994 self.parser.add_option('--evalSavedPython', dest='evalSavedPython', 995 default=[], action='append', 996 help="Execute the Python code previously generated" \ 997 " and saved.")

998 999 1000 if __name__ == '__main__': 1001 zm = ZenMib() 1002 zm.main() 1003

Source Code for Module Products.ZenModel.zenmib