Products.ZenModel.zenmib

1 ########################################################################### 2 # 3 # This program is part of Zenoss Core, an open source monitoring platform. 4 # Copyright (C) 2007, Zenoss Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify it 7 # under the terms of the GNU General Public License version 2 as published by 8 # the Free Software Foundation. 9 # 10 # For complete information please visit: http://www.zenoss.com/oss/ 11 # 12 ########################################################################### 13 __doc__= """zenmib 14 The zenmib program converts MIBs into python data structures and then 15 (by default) adds the data to the Zenoss DMD. Essentially, zenmib is a 16 wrapper program around the smidump program, whose output (python code) is 17 then executed "inside" the Zope database. 18 19 Overview of terms: 20 SNMP Simple Network Management Protocol 21 A network protocol originally based on UDP which allows a management 22 application (ie SNMP manager) to contact and get information from a 23 device (ie router, computer, network-capable toaster). The software 24 on the device that is capable of understanding SNMP and responding 25 appropriately is called an SNMP agent. 26 27 MIB Management of Information Base 28 A description of what a particular SNMP agent provides and what 29 traps it sends. A MIB is a part of a tree structure based on a root 30 MIB. Since a MIB is a rooted tree, it allows for delegation of areas 31 under the tree to different organizations. 32 33 ASN Abstract Syntax Notation 34 The notation used to construct a MIB. 35 36 OID Object IDentifier 37 A MIB is constructed of unique identifiers 38 39 40 Background information: 41 http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol 42 Overview of SNMP. 43 44 http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en 45 The libsmi project is the creator of smidump. There are several 46 interesting sub-projects available. 47 48 http://net-snmp.sourceforge.net/ 49 Homepage for Net-SNMP which is used by Zenoss for SNMP management. 50 51 http://www.et.put.poznan.pl/snmp/asn1/asn1.html 52 An overview of Abstract Syntax Notation (ASN), the language in 53 which MIBs are written. 54 """ 55 56 import os 57 import os.path 58 import sys 59 import glob 60 import re 61 from subprocess import Popen, PIPE 62 63 import Globals 64 import transaction 65 66 from Products.ZenUtils.ZCmdBase import ZCmdBase 67 from Products.ZenUtils.Utils import zenPath 68 from zExceptions import BadRequest 69 70

71 -def walk(*dirs):

72 """ 73 Generator function to create a list of absolute paths 74 for all files in a directory tree starting from the list 75 of directories given as arguments to this function. 76 77 @param *dirs: list of directories to investigate 78 @type *dirs: list of strings 79 @return: directory to investigate 80 @rtype: string 81 """ 82 for dir in dirs: 83 for dirname, _, filenames in os.walk(dir): 84 for filename in filenames: 85 yield os.path.join(dirname, filename)

86 87 88

89 -class DependencyMap:

90 """ 91 A dependency is a reference to another part of the MIB tree. 92 All MIB definitions start from the base of the tree (ie .1). 93 Generally dependencies are from MIB definitions external to 94 the MIB under inspection. 95 """ 96

97 - def __init__(self):

98 self.fileMap = {} 99 self.depMap = {}

100 101

102 - def add(self, filename, name, dependencies):

103 """ 104 Add a dependency to the dependency tree if it's not already there. 105 106 @param filename: name of MIB file to import 107 @type filename: string 108 @param name: name of MIB 109 @type name: string 110 @param dependencies: dependency 111 @type dependencies: dependency object 112 """ 113 if name not in self.depMap: 114 self.fileMap[filename] = name 115 self.depMap[name] = (filename, dependencies)

116 117

118 - def getName(self, filename):

119 """ 120 Given a filename, return the name of the MIB tree defined in it. 121 Makes the assumption that there's only one MIB tree per file. 122 123 @param filename: MIB filename 124 @type filename: string 125 @return: MIB name 126 @rtype: string 127 @todo: to allow for multiple MIBs in a file, should return a list 128 """ 129 return self.fileMap.get(filename, None)

130 131

132 - def getDependencies(self, name):

133 """ 134 Given a name of the MIB tree, return the filename that it's from 135 and its dependencies. 136 137 @param name: name of MIB 138 @type name: string 139 @return: dependency tree 140 @rtype: tuple of ( name, dependency object) 141 """ 142 return self.depMap.get(name, None)

143 144 145

146 -class zenmib(ZCmdBase):

147 """ 148 Wrapper around the smidump utilities to load MIB files into 149 the DMD tree. 150 """

151 - def map_file_to_dependents(self, mibfile):

152 """ 153 Scan the MIB file to determine what MIB trees the file is dependent on. 154 155 @param mibfile: MIB filename 156 @type mibfile: string 157 @return: dependency tree 158 @rtype: tuple of ( name, dependency object) 159 """ 160 # Slurp in the whole file 161 fp = open(mibfile) 162 mib = fp.read() 163 fp.close() 164 return self.map_mib_to_dependents(mib)

165

166 - def map_mib_to_dependents(self, mib):

167 # ASN.1 syntax regular expressions for declaring MIBs 168 # 169 # An example from http://www.faqs.org/rfcs/rfc1212.html 170 # 171 # RFC1213-MIB DEFINITIONS ::= BEGIN 172 # 173 # IMPORTS 174 # experimental, OBJECT-TYPE, Counter 175 # FROM RFC1155-SMI; 176 # 177 # -- contact IANA for actual number 178 # 179 # root OBJECT IDENTIFIER ::= { experimental xx } 180 # 181 # END 182 183 DEFINITIONS = re.compile(r'(?P<mib_name>[A-Za-z-0-9]+) +DEFINITIONS *::= *BEGIN') 184 DEPENDS = re.compile(r'FROM *(?P<mib_dependency>[A-Za-z-0-9]+)') 185 186 # Split up the file and determine what OIDs need what other OIDs 187 # 188 # TODO: Fix the code so that it takes care of the case where there are multiple 189 # OBJECT IDENTIFIER sections in the MIB. 190 parts = mib.split('OBJECT IDENTIFIER', 1) 191 mib_prologue = parts[0] 192 match = DEFINITIONS.search(mib_prologue) 193 if not match: 194 # Special case: bootstrap MIB for the root of the MIB tree 195 return None, [] 196 197 # Search through the prologue to find all of the IMPORTS. 198 # An example from a real MIB 199 # 200 # IMPORTS 201 # MODULE-IDENTITY, OBJECT-TYPE, enterprises, Integer32, 202 # TimeTicks,NOTIFICATION-TYPE FROM SNMPv2-SMI 203 # DisplayString FROM RFC1213-MIB 204 # MODULE-COMPLIANCE, OBJECT-GROUP, 205 # NOTIFICATION-GROUP FROM SNMPv2-CONF; 206 depends = [] 207 name = match.group('mib_name') 208 start = match.end(0) # Jump to past the first FROM token 209 while 1: 210 match = DEPENDS.search(mib_prologue, start) 211 if not match: 212 break 213 214 depends.append(match.group('mib_dependency')) 215 216 # Skip to just past the next FROM token 217 start = match.end(0) 218 219 return name, depends

220 221 222

223 - def dependencies(self, filenames):

224 """ 225 Create a dependency map for all MIB files. 226 Exit the program if we're missing any files. 227 228 @param filenames: names of MIB files to import 229 @type filenames: list of strings 230 @return: dependency tree 231 @rtype: DependencyMap 232 """ 233 missing_files = 0 234 result = DependencyMap() 235 for filename in filenames: 236 try: 237 defines, depends = self.map_file_to_dependents(filename) 238 239 except IOError: 240 self.log.error( "Couldn't open file %s", filename) 241 missing_files += 1 242 continue 243 244 if defines == None: 245 self.log.debug( "Unable to parse information from %s -- skipping", filename) 246 else: 247 result.add(filename, defines, depends) 248 249 if missing_files > 0: 250 self.log.error( "Missing %s files", missing_files ) 251 sys.exit(1) 252 253 return result

254 255 256

257 - def getDependencies(self, filename, depMap):

258 """ 259 smidump needs to know the list of dependent files for a MIB file in 260 order to properly resolve external references. 261 262 @param filename: name of MIB file to import 263 @type filename: string 264 @param depMap: dependency tree 265 @type depMap: DependencyMap 266 @return: list of dependencies 267 @rtype: list of strings 268 """ 269 # Sanity check: if a file doesn't need anything else, it 270 # has no dependencies. Avoid further work. 271 name = depMap.getName(filename) 272 if not name: 273 return [] 274 275 # Find the files required by the OID tree in the file. 276 deps = [] 277 def dependency_search(name): 278 """ 279 Create a list of files required by an OID. 280 281 @param name: name of OID 282 @type name: string 283 """ 284 fileAndDeps = depMap.getDependencies(name) 285 if not fileAndDeps: 286 self.log.warn( "Unable to find a file providing the OID %s", name) 287 return 288 289 mib_file, dependent_oids = fileAndDeps 290 if mib_file and mib_file not in deps: 291 deps.append(mib_file) 292 293 for unresolved_oid in dependent_oids: 294 dependency_search(unresolved_oid)

295 296 # Search the dependency map 297 dependency_search(name) 298 if deps[1:]: 299 return deps[1:] 300 301 return []

302 303 304

305 - def generate_python_from_mib( self, mibname, dependencies ):

306 """ 307 Use the smidump program to convert a MIB into Python code" 308 309 @param mibname: name of the MIB 310 @type mibname: string 311 @param dependencies: list of dependent files 312 @type dependencies: list of strings 313 @return: the newly created MIB 314 @rtype: MIB object 315 """ 316 dump_command = [ 'smidump', '-k', '-fpython' ] 317 for dep in dependencies[1:]: 318 # Add command-line flag for our dependent files 319 dump_command.append( '-p') 320 dump_command.append( dep ) 321 322 dump_command.append( mibname ) 323 self.log.debug('Running %s', ' '.join( dump_command)) 324 proc = Popen( dump_command, stdout=PIPE, stderr=PIPE ) 325 326 python_code, warnings = proc.communicate() 327 proc.wait() 328 if proc.returncode: 329 self.log.error(warnings) 330 return None 331 332 if len(warnings) > 0: 333 self.log.debug("Found warnings while trying to import MIB:\n%s" \ 334 % warnings) 335 336 # Now we'll be brave and try to execute the MIB-to-python code 337 # and store the resulting dictionary in 'result' 338 result = {} 339 try: 340 exec python_code in result 341 342 except (SystemExit, KeyboardInterrupt): raise 343 except: 344 self.log.exception("Unable to import Pythonized-MIB: %s", mibname) 345 return None 346 347 # Now look for the start of the MIB 348 mib = result.get( 'MIB', None) 349 return mib

350 351 352 353 # Standard MIB attributes that we expect in all MIBs 354 MIB_MOD_ATTS = ('language', 'contact', 'description') 355

356 - def load_mib(self, mibs, mibname, depmap):

357 """ 358 Attempt to load a MIB after generating its dependency tree 359 360 @param mibs: filenames of the MIBs to load 361 @type mibs: list of strings 362 @param mibname: name of the MIB 363 @type mibname: string 364 @param dependencies: list of dependent files 365 @type dependencies: string 366 @return: whether the MIB load was successful or not 367 @rtype: boolean 368 """ 369 dependencies = self.getDependencies(mibname, depmap) 370 371 mib = self.generate_python_from_mib( mibname, dependencies ) 372 if not mib: 373 return False 374 375 # Check for duplicates -- or maybe not... 376 modname = mib['moduleName'] 377 # TODO: Find out Why this is commented out 378 #mod = mibs.findMibModule(modname) 379 mod = None 380 if mod: 381 self.log.warn( "Skipping %s as it is already loaded", modname) 382 return False 383 384 # Create the container for the MIBs and define meta-data 385 # In the DMD this creates another container class which 386 # contains mibnodes. These data types are found in 387 # Products.ZenModel.MibModule and Products.ZenModel.MibNode 388 mod = mibs.createMibModule(modname, self.options.path) 389 for key, val in mib[modname].items(): 390 if key in self.MIB_MOD_ATTS: 391 setattr(mod, key, val) 392 393 # Add regular OIDs to the mibmodule + mibnode relationship tree 394 nodes_added = 0 395 if 'nodes' in mib: 396 for name, values in mib['nodes'].items(): 397 try: 398 mod.createMibNode(name, **values) 399 nodes_added += 1 400 except BadRequest: 401 try: 402 self.log.warn("Unable to add node id '%s' as this" 403 " name is reserved for use by Zope", 404 name) 405 newName = '_'.join([name, modname]) 406 self.log.warn("Trying to add node '%s' as '%s'", 407 name, newName) 408 mod.createMibNode(newName, **values) 409 self.log.warn("Renamed '%s' to '%s' and added to MIB" 410 " nodes", name, newName) 411 except: 412 self.log.warn("Unable to add '%s' -- skipping", 413 name) 414 415 # Put SNMP trap information into Products.ZenModel.MibNotification 416 traps_added = 0 417 if 'notifications' in mib: 418 for name, values in mib['notifications'].items(): 419 try: 420 mod.createMibNotification(name, **values) 421 traps_added += 1 422 except BadRequest: 423 try: 424 self.log.warn("Unable to add trap id '%s' as this" 425 " name is reserved for use by Zope", 426 name) 427 newName = '_'.join([name, modname]) 428 self.log.warn("Trying to add trap '%s' as '%s'", 429 name, newName) 430 mod.createMibNotification(newName, **values) 431 self.log.warn("Renamed '%s' to '%s' and added to MIB" 432 " traps", name, newName) 433 except: 434 self.log.warn("Unable to add '%s' -- skipping", 435 name) 436 437 self.log.info("Parsed %d nodes and %d notifications", nodes_added, 438 traps_added) 439 440 # Add the MIB tree permanently to the DMD except if we get the 441 # --nocommit flag. 442 if not self.options.nocommit: 443 trans = transaction.get() 444 trans.setUser( "zenmib" ) 445 trans.note("Loaded MIB %s into the DMD" % modname) 446 trans.commit() 447 self.log.info("Loaded MIB %s into the DMD", modname) 448 449 return True

450 451 452

453 - def main(self):

454 """ 455 Main loop of the program 456 """ 457 # Prepare to load the default MIBs 458 smimibdir = self.options.mibsdir 459 if not os.path.exists( smimibdir ): 460 self.log.error("The directory %s doesn't exist!" % smimibdir ) 461 sys.exit(1) 462 463 ietf, iana, irtf, tubs, site = \ 464 map(lambda x: os.path.join(smimibdir, x), 465 'ietf iana irtf tubs site'.split()) 466 467 # Either load MIBs from the command-line or from the default 468 # location where MIBs are stored by Zenoss. 469 if len(self.args) > 0: 470 mibnames = self.args 471 depMap = self.dependencies(list(walk(ietf, iana, irtf, tubs)) 472 + mibnames) 473 else: 474 mibnames = glob.glob(os.path.join(smimibdir, 'site', '*')) 475 depMap = self.dependencies(walk(ietf, iana, irtf, tubs, site)) 476 477 # Make connection to the DMD at the start of the MIB tree 478 mibs = self.dmd.Mibs 479 480 # Process all of the MIBs that we've found 481 loaded_mib_files = 0 482 for mibname in mibnames: 483 try: 484 if self.load_mib( mibs, mibname, depMap): 485 loaded_mib_files += 1 486 487 except (SystemExit, KeyboardInterrupt): raise 488 except Exception, ex: 489 self.log.exception("Failed to load MIB: %s" % mibname) 490 491 action = "Loaded" 492 if self.options.nocommit: 493 action = "Processed" 494 self.log.info( "%s %d MIB file(s)" % ( action, loaded_mib_files)) 495 sys.exit(0)

496 497

498 - def buildOptions(self):

499 """ 500 Command-line options 501 """ 502 ZCmdBase.buildOptions(self) 503 self.parser.add_option('--mibsdir', 504 dest='mibsdir', default=zenPath('share/mibs'), 505 help="Directory of input MIB files [ default: %default ]") 506 self.parser.add_option('--path', 507 dest='path', default="/", 508 help="Path to load MIB into the DMD") 509 self.parser.add_option('--nocommit', action='store_true', 510 dest='nocommit', default=False, 511 help="Don't commit the MIB to the DMD after loading")

512 513 514 if __name__ == '__main__': 515 zm = zenmib() 516 zm.main() 517

Source Code for Module Products.ZenModel.zenmib