ZenModel.zenmib

1 ########################################################################### 2 # 3 # This program is part of Zenoss Core, an open source monitoring platform. 4 # Copyright (C) 2007, Zenoss Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify it 7 # under the terms of the GNU General Public License version 2 as published by 8 # the Free Software Foundation. 9 # 10 # For complete information please visit: http://www.zenoss.com/oss/ 11 # 12 ########################################################################### 13 __doc__= """zenmib 14 The zenmib program converts MIBs into python data structures and then 15 (by default) adds the data to the Zenoss DMD. Essentially, zenmib is a 16 wrapper program around the smidump program, whose output (python code) is 17 then executed "inside" the Zope database. 18 19 Overview of terms: 20 SNMP Simple Network Management Protocol 21 A network protocol originally based on UDP which allows a management 22 application (ie SNMP manager) to contact and get information from a 23 device (ie router, computer, network-capable toaster). The software 24 on the device that is capable of understanding SNMP and responding 25 appropriately is called an SNMP agent. 26 27 MIB Management of Information Base 28 A description of what a particular SNMP agent provides and what 29 traps it sends. A MIB is a part of a tree structure based on a root 30 MIB. Since a MIB is a rooted tree, it allows for delegation of areas 31 under the tree to different organizations. 32 33 ASN Abstract Syntax Notation 34 The notation used to construct a MIB. 35 36 OID Object IDentifier 37 A MIB is constructed of unique identifiers 38 39 40 Background information: 41 http://en.wikipedia.org/wiki/Simple_Network_Management_Protocol 42 Overview of SNMP. 43 44 http://www.ibr.cs.tu-bs.de/projects/libsmi/index.html?lang=en 45 The libsmi project is the creator of smidump. There are several 46 interesting sub-projects available. 47 48 http://net-snmp.sourceforge.net/ 49 Homepage for Net-SNMP which is used by Zenoss for SNMP management. 50 51 http://www.et.put.poznan.pl/snmp/asn1/asn1.html 52 An overview of Abstract Syntax Notation (ASN), the language in 53 which MIBs are written. 54 """ 55 56 import os 57 import os.path 58 import sys 59 import glob 60 import re 61 from subprocess import Popen, PIPE 62 63 import Globals 64 import transaction 65 66 from Products.ZenUtils.ZCmdBase import ZCmdBase 67 from Products.ZenUtils.Utils import zenPath 68 from zExceptions import BadRequest 69 70

71 -def walk(*dirs):

72 """ 73 Generator function to create a list of absolute paths 74 for all files in a directory tree starting from the list 75 of directories given as arguments to this function. 76 77 @param *dirs: list of directories to investigate 78 @type *dirs: list of strings 79 @return: directory to investigate 80 @rtype: string 81 """ 82 for dir in dirs: 83 for dirname, _, filenames in os.walk(dir): 84 for filename in filenames: 85 yield os.path.join(dirname, filename)

86 87 88

89 -class DependencyMap:

90 """ 91 A dependency is a reference to another part of the MIB tree. 92 All MIB definitions start from the base of the tree (ie .1). 93 Generally dependencies are from MIB definitions external to 94 the MIB under inspection. 95 """ 96

97 - def __init__(self):

98 self.fileMap = {} 99 self.depMap = {}

100 101

102 - def add(self, filename, name, dependencies):

103 """ 104 Add a dependency to the dependency tree if it's not already there. 105 106 @param filename: name of MIB file to import 107 @type filename: string 108 @param name: name of MIB 109 @type name: string 110 @param dependencies: dependency 111 @type dependencies: dependency object 112 """ 113 if not self.depMap.has_key(name): 114 self.fileMap[filename] = name 115 self.depMap[name] = (filename, dependencies)

116 117

118 - def getName(self, filename):

119 """ 120 Given a filename, return the name of the MIB tree defined in it. 121 Makes the assumption that there's only one MIB tree per file. 122 123 @param filename: MIB filename 124 @type filename: string 125 @return: MIB name 126 @rtype: string 127 @todo: to allow for multiple MIBs in a file, should return a list 128 """ 129 return self.fileMap.get(filename, None)

130 131

132 - def getDependencies(self, name):

133 """ 134 Given a name of the MIB tree, return the filename that it's from 135 and its dependencies. 136 137 @param name: name of MIB 138 @type name: string 139 @return: dependency tree 140 @rtype: tuple of ( name, dependency object) 141 """ 142 return self.depMap.get(name, None)

143 144 145

146 -class zenmib(ZCmdBase):

147 """ 148 Wrapper around the smidump utilities to load MIB files into 149 the DMD tree. 150 """ 151

152 - def map_file_to_dependents(self, mibfile):

153 """ 154 Scan the MIB file to determine what MIB trees the file is dependent on. 155 156 @param mibfile: MIB filename 157 @type mibfile: string 158 @return: dependency tree 159 @rtype: tuple of ( name, dependency object) 160 """ 161 # Slurp in the whole file 162 fp = open(mibfile) 163 mib = fp.read() 164 fp.close() 165 166 # ASN.1 syntax regular expressions for declaring MIBs 167 # 168 # An example from http://www.faqs.org/rfcs/rfc1212.html 169 # 170 # RFC1213-MIB DEFINITIONS ::= BEGIN 171 # 172 # IMPORTS 173 # experimental, OBJECT-TYPE, Counter 174 # FROM RFC1155-SMI; 175 # 176 # -- contact IANA for actual number 177 # 178 # root OBJECT IDENTIFIER ::= { experimental xx } 179 # 180 # END 181 182 DEFINITIONS = re.compile(r'(?P<mib_name>[A-Za-z-0-9]+) +DEFINITIONS *::= *BEGIN') 183 DEPENDS = re.compile(r'FROM *(?P<mib_dependency>[A-Za-z-0-9]+)') 184 185 # Split up the file and determine what OIDs need what other OIDs 186 # 187 # TODO: Fix the code so that it takes care of the case where there are multiple 188 # OBJECT IDENTIFIER sections in the MIB. 189 parts = mib.split('OBJECT IDENTIFIER', 1) 190 mib_prologue = parts[0] 191 match = DEFINITIONS.search(mib_prologue) 192 if not match: 193 # Special case: bootstrap MIB for the root of the MIB tree 194 return None, [] 195 196 # Search through the prologue to find all of the IMPORTS. 197 # An example from a real MIB 198 # 199 # IMPORTS 200 # MODULE-IDENTITY, OBJECT-TYPE, enterprises, Integer32, 201 # TimeTicks,NOTIFICATION-TYPE FROM SNMPv2-SMI 202 # DisplayString FROM RFC1213-MIB 203 # MODULE-COMPLIANCE, OBJECT-GROUP, 204 # NOTIFICATION-GROUP FROM SNMPv2-CONF; 205 depends = [] 206 name = match.group('mib_name') 207 start = match.end(0) # Jump to past the first FROM token 208 while 1: 209 match = DEPENDS.search(mib_prologue, start) 210 if not match: 211 break 212 213 depends.append(match.group('mib_dependency')) 214 215 # Skip to just past the next FROM token 216 start = match.end(0) 217 218 return name, depends

219 220 221

222 - def dependencies(self, filenames):

223 """ 224 Create a dependency map for all MIB files. 225 Exit the program if we're missing any files. 226 227 @param filenames: names of MIB files to import 228 @type filenames: list of strings 229 @return: dependency tree 230 @rtype: DependencyMap 231 """ 232 missing_files = 0 233 result = DependencyMap() 234 for filename in filenames: 235 try: 236 defines, depends = self.map_file_to_dependents(filename) 237 238 except IOError: 239 self.log.error( "Couldn't open file %s", filename) 240 missing_files += 1 241 continue 242 243 if defines == None: 244 self.log.debug( "Unable to parse information from %s -- skipping", filename) 245 else: 246 result.add(filename, defines, depends) 247 248 if missing_files > 0: 249 self.log.error( "Missing %s files", missing_files ) 250 sys.exit(1) 251 252 return result

253 254 255

256 - def getDependencies(self, filename, depMap):

257 """ 258 smidump needs to know the list of dependent files for a MIB file in 259 order to properly resolve external references. 260 261 @param filename: name of MIB file to import 262 @type filename: string 263 @param depMap: dependency tree 264 @type depMap: DependencyMap 265 @return: list of dependencies 266 @rtype: list of strings 267 """ 268 # Sanity check: if a file doesn't need anything else, it 269 # has no dependencies. Avoid further work. 270 name = depMap.getName(filename) 271 if not name: 272 return [] 273 274 # Find the files required by the OID tree in the file. 275 deps = [] 276 def dependency_search(name): 277 """ 278 Create a list of files required by an OID. 279 280 @param name: name of OID 281 @type name: string 282 """ 283 fileAndDeps = depMap.getDependencies(name) 284 if not fileAndDeps: 285 self.log.warn( "Unable to find a file providing the OID %s", name) 286 return 287 288 mib_file, dependent_oids = fileAndDeps 289 if mib_file and mib_file not in deps: 290 deps.append(mib_file) 291 292 for unresolved_oid in dependent_oids: 293 dependency_search(unresolved_oid)

294 295 # Search the dependency map 296 dependency_search(name) 297 if deps[1:]: 298 return deps[1:] 299 300 return []

301 302 303

304 - def generate_python_from_mib( self, mibname, dependencies ):

305 """ 306 Use the smidump program to convert a MIB into Python code" 307 308 @param mibname: name of the MIB 309 @type mibname: string 310 @param dependencies: list of dependent files 311 @type dependencies: list of strings 312 @return: the newly created MIB 313 @rtype: MIB object 314 """ 315 dump_command = [ 'smidump', '-k', '-fpython' ] 316 for dep in dependencies[1:]: 317 # Add command-line flag for our dependent files 318 dump_command.append( '-p') 319 dump_command.append( dep ) 320 321 dump_command.append( mibname ) 322 self.log.debug('Running %s', ' '.join( dump_command)) 323 proc = Popen( dump_command, stdout=PIPE, stderr=PIPE ) 324 325 python_code, warnings = proc.communicate() 326 proc.wait() 327 if proc.returncode: 328 self.log.error(warnings) 329 return None 330 331 if len(warnings) > 0: 332 self.log.debug("Found warnings while trying to import MIB:\n%s" \ 333 % warnings) 334 335 # Now we'll be brave and try to execute the MIB-to-python code 336 # and store the resulting dictionary in 'result' 337 result = {} 338 try: 339 exec python_code in result 340 341 except (SystemExit, KeyboardInterrupt): raise 342 except: 343 self.log.exception("Unable to import Pythonized-MIB: %s", mibname) 344 return None 345 346 # Now look for the start of the MIB 347 mib = result.get( 'MIB', None) 348 return mib

349 350 351 352 # Standard MIB attributes that we expect in all MIBs 353 MIB_MOD_ATTS = ('language', 'contact', 'description') 354

355 - def load_mib(self, mibs, mibname, depmap):

356 """ 357 Attempt to load a MIB after generating its dependency tree 358 359 @param mibs: filenames of the MIBs to load 360 @type mibs: list of strings 361 @param mibname: name of the MIB 362 @type mibname: string 363 @param dependencies: list of dependent files 364 @type dependencies: string 365 @return: whether the MIB load was successful or not 366 @rtype: boolean 367 """ 368 dependencies = self.getDependencies(mibname, depmap) 369 370 mib = self.generate_python_from_mib( mibname, dependencies ) 371 if not mib: 372 return False 373 374 # Check for duplicates -- or maybe not... 375 modname = mib['moduleName'] 376 # TODO: Find out Why this is commented out 377 #mod = mibs.findMibModule(modname) 378 mod = None 379 if mod: 380 self.log.warn( "Skipping %s as it is already loaded", modname) 381 return False 382 383 # Create the container for the MIBs and define meta-data 384 # In the DMD this creates another container class which 385 # contains mibnodes. These data types are found in 386 # Products.ZenModel.MibModule and Products.ZenModel.MibNode 387 mod = mibs.createMibModule(modname, self.options.path) 388 for key, val in mib[modname].items(): 389 if key in self.MIB_MOD_ATTS: 390 setattr(mod, key, val) 391 392 # Add regular OIDs to the mibmodule + mibnode relationship tree 393 if mib.has_key('nodes'): 394 for name, values in mib['nodes'].items(): 395 try: 396 mod.createMibNode(name, **values) 397 except BadRequest: 398 try: 399 self.log.warn("Unable to add node id '%s' as this" 400 " name is reserved for use by Zope", 401 name) 402 newName = '_'.join([name, modname]) 403 self.log.warn("Trying to add node '%s' as '%s'", 404 name, newName) 405 mod.createMibNode(newName, **values) 406 self.log.warn("Renamed '%s' to '%s' and added to MIB" 407 " nodes", name, newName) 408 except: 409 self.log.warn("Unable to add '%s' -- skipping", 410 name) 411 412 # Put SNMP trap information into Products.ZenModel.MibNotification 413 if mib.has_key('notifications'): 414 for name, values in mib['notifications'].items(): 415 try: 416 mod.createMibNotification(name, **values) 417 except BadRequest: 418 try: 419 self.log.warn("Unable to add trap id '%s' as this" 420 " name is reserved for use by Zope", 421 name) 422 newName = '_'.join([name, modname]) 423 self.log.warn("Trying to add trap '%s' as '%s'", 424 name, newName) 425 mod.createMibNotification(newName, **values) 426 self.log.warn("Renamed '%s' to '%s' and added to MIB" 427 " traps", name, newName) 428 except: 429 self.log.warn("Unable to add '%s' -- skipping", 430 name) 431 432 # Add the MIB tree permanently to the DMD except if we get the 433 # --nocommit flag. 434 if not self.options.nocommit: 435 trans = transaction.get() 436 trans.setUser( "zenmib" ) 437 trans.note("Loaded MIB %s into the DMD" % modname) 438 trans.commit() 439 self.log.info("Loaded MIB %s into the DMD", modname) 440 441 return True

442 443 444

445 - def main(self):

446 """ 447 Main loop of the program 448 """ 449 # Prepare to load the default MIBs 450 smimibdir = self.options.mibsdir 451 if not os.path.exists( smimibdir ): 452 self.log.error("The directory %s doesn't exist!" % smimibdir ) 453 sys.exit(1) 454 455 ietf, iana, irtf, tubs, site = \ 456 map(lambda x: os.path.join(smimibdir, x), 457 'ietf iana irtf tubs site'.split()) 458 459 # Either load MIBs from the command-line or from the default 460 # location where MIBs are stored by Zenoss. 461 if len(self.args) > 0: 462 mibnames = self.args 463 depMap = self.dependencies(list(walk(ietf, iana, irtf, tubs)) 464 + mibnames) 465 else: 466 mibnames = glob.glob(os.path.join(smimibdir, 'site', '*')) 467 depMap = self.dependencies(walk(ietf, iana, irtf, tubs, site)) 468 469 # Make connection to the DMD at the start of the MIB tree 470 mibs = self.dmd.Mibs 471 472 # Process all of the MIBs that we've found 473 loaded_mib_files = 0 474 for mibname in mibnames: 475 try: 476 if self.load_mib( mibs, mibname, depMap): 477 loaded_mib_files += 1 478 479 except (SystemExit, KeyboardInterrupt): raise 480 except Exception, ex: 481 self.log.exception("Failed to load MIB: %s" % mibname) 482 483 action = "Loaded" 484 if self.options.nocommit: 485 action = "Processed" 486 self.log.info( "%s %d MIB file(s)" % ( action, loaded_mib_files)) 487 sys.exit(0)

488 489

490 - def buildOptions(self):

491 """ 492 Command-line options 493 """ 494 ZCmdBase.buildOptions(self) 495 self.parser.add_option('--mibsdir', 496 dest='mibsdir', default=zenPath('share/mibs'), 497 help="Directory of input MIB files [ default: %default ]") 498 self.parser.add_option('--path', 499 dest='path', default="/", 500 help="Path to load MIB into the DMD") 501 self.parser.add_option('--nocommit', action='store_true', 502 dest='nocommit', default=False, 503 help="Don't commit the MIB to the DMD after loading")

504 505 506 if __name__ == '__main__': 507 zm = zenmib() 508 zm.main() 509

Source Code for Module ZenModel.zenmib