Products.ZenModel.BatchDeviceLoader

1 ############################################################################## 2 # 3 # Copyright (C) Zenoss, Inc. 2009, 2011, all rights reserved. 4 # 5 # This content is made available according to terms specified in 6 # License.zenoss under the directory where your Zenoss product is installed. 7 # 8 ############################################################################## 9 10 11 __doc__ = """zenbatchload 12 13 zenbatchload loads a list of devices read from a file. 14 """ 15 16 import sys 17 import re 18 from traceback import format_exc 19 import socket 20 21 import Globals 22 from DateTime import DateTime 23 from ZODB.POSException import ConflictError 24 from ZODB.transact import transact 25 from zope.component import getUtility 26 27 from zExceptions import BadRequest 28 29 from ZPublisher.Converters import type_converters 30 from Products.ZenModel.interfaces import IDeviceLoader 31 from Products.ZenUtils.ZCmdBase import ZCmdBase 32 from Products.ZenModel.Device import Device 33 from Products.ZenRelations.ZenPropertyManager import iszprop 34 from Products.ZenModel.ZenModelBase import iscustprop 35 from Products.ZenEvents.ZenEventClasses import Change_Add 36 37 from zenoss.protocols.protobufs.zep_pb2 import SEVERITY_INFO, SEVERITY_ERROR

38 39 40 -class BatchDeviceLoader(ZCmdBase):

41 """ 42 Base class wrapping around dmd.DeviceLoader 43 """ 44 45 sample_configs = """# 46 # Example zenbatchloader device file 47 # 48 # This file is formatted with one entry per line, like this: 49 # 50 # /Devices/device_class_name Python-expression 51 # hostname Python-expression 52 # 53 # For organizers (ie the /Devices path), the Python-expression 54 # is used to define defaults to be used for devices listed 55 # after the organizer. The defaults that can be specified are: 56 # 57 # * loader arguments (use the --show_options flag to show these) 58 # * zPropertie (from a device, use the More -> zProperties 59 # menu option to see the available ones.) 60 # 61 # NOTE: new zProperties *cannot* be created through this file 62 # 63 # The Python-expression is used to create a dictionary of settings. 64 # device_settings = eval( 'dict(' + python-expression + ')' ) 65 # 66 67 68 # Setting locations 69 /Locations/Canada address="Canada" 70 71 /Locations/Canada/Alberta address="Alberta, Canada" 72 73 /Locations/Canada/Alberta/Calgary address="Calgary, Alberta, Canada" 74 75 76 # If no organizer is specified at the beginning of the file, 77 # defaults to the /Devices/Discovered device class. 78 device0 comments="A simple device" 79 # All settings must be seperated by a comma. 80 device1 comments="A simple device", zSnmpCommunity='blue', zSnmpVer='v1' 81 82 # Notes for this file: 83 # * Oraganizer names *must* start with '/' 84 # 85 /Devices/Server/Linux zSnmpPort=1543 86 # Python strings can use either ' or " -- there's no difference. 87 # As a special case, it is also possible to specify the IP address 88 linux_device1 setManageIp='10.10.10.77', zSnmpCommunity='blue', zSnmpVer="v2c" 89 # A '\' at the end of the line allows you to place more 90 # expressions on a new line. Don't forget the comma... 91 linux_device2 zLinks="<a href='http://example.org'>Support site</a>", \ 92 zTelnetEnable=True, \ 93 zTelnetPromptTimeout=15.3 94 95 # A new organizer drops all previous settings, and allows 96 # for new ones to be used. Settings do not span files. 97 /Devices/Server/Windows zWinUser="administrator", zWinPassword='fred' 98 # Bind templates 99 windows_device1 zDeviceTemplates=[ 'Device', 'myTemplate' ] 100 # Override the default from the organizer setting. 101 windows_device2 zWinUser="administrator", zWinPassword='thomas' 102 103 # Apply other settings to the device 104 settingsDevice setManageIp='10.10.10.77', setLocation="123 Elm Street", \ 105 setSystems=['/mySystems'], setPerformanceMonitor='remoteCollector1', \ 106 setHWSerialNumber="abc123456789", setGroups=['/myGroup'], \ 107 setHWProduct=('myproductName','manufacturer'), setOSProduct=('OS Name','manufacturer') 108 109 # If the device or device class contains a space, then it must be quoted (either ' or ") 110 "/Server/Windows/WMI/Active Directory/2008" 111 112 # Now, what if we have a device that isn't really a device, and requires 113 # a special loader? 114 # The 'loader' setting requires a registered utility, and 'loader_arg_keys' is 115 # a list from which any other settings will be passed into the loader callable. 116 # 117 # Here is a commmented-out example of how a VMware endpoint might be added: 118 # 119 #/Devices/VMware loader='vmware', loader_arg_keys=['host', 'username', 'password', 'useSsl', 'id'] 120 #esxwin2 id='esxwin2', host='esxwin2.zenoss.loc', username='testuser', password='password', useSsl=True 121 122 # Apply custom schema properties (c-properties) to a device 123 windows_device7 cDateTest='2010/02/28' 124 125 """ 126

127 - def __init__(self, *args, **kwargs):

128 ZCmdBase.__init__(self, *args, **kwargs) 129 self.defaults = {} 130 131 self.loader = self.dmd.DeviceLoader.loadDevice 132 133 self.fqdn = socket.getfqdn() 134 self.baseEvent = dict( 135 device=self.fqdn, 136 component='', 137 agent='zenbatchload', 138 monitor='localhost', 139 manager=self.fqdn, 140 severity=SEVERITY_ERROR, 141 # Note: Change_Add is probably a better event class, but these 142 # events get sent to history by the default Zen property stuff 143 # on the event class. zendisc uses Status_Snmp, so so will we. 144 eventClass=Change_Add, 145 ) 146 147 # Create the list of options we want people to know about 148 self.loader_args = dict.fromkeys( self.loader.func_code.co_varnames ) 149 unsupportable_args = [ 150 'REQUEST', 'device', 'self', 'xmlrpc', 'e', 'handler', 151 ] 152 for opt in unsupportable_args: 153 if opt in self.loader_args: 154 del self.loader_args[opt]

155

156 - def loadDeviceList(self, args=None):

157 """ 158 Read through all of the files listed as arguments and 159 return a list of device entries. 160 161 @parameter args: list of filenames (uses self.args is this is None) 162 @type args: list of strings 163 @return: list of device specifications 164 @rtype: list of dictionaries 165 """ 166 if args is None: 167 args = self.args 168 169 device_list = [] 170 for filename in args: 171 if filename.strip() != '': 172 try: 173 data = open(filename,'r').readlines() 174 except IOError: 175 msg = "Unable to open the file '%s'" % filename 176 self.reportException(msg) 177 continue 178 179 temp_dev_list = self.parseDevices(data) 180 if temp_dev_list: 181 device_list += temp_dev_list 182 183 return device_list

184

185 - def applyZProps(self, device, device_specs):

186 """ 187 Apply zProperty settings (if any) to the device. 188 189 @parameter device: device to modify 190 @type device: DMD device object 191 @parameter device_specs: device creation dictionary 192 @type device_specs: dictionary 193 """ 194 self.log.debug( "Applying zProperties..." ) 195 # Returns a list of (key, value) pairs. 196 # Convert it to a dictionary. 197 dev_zprops = dict( device.zenPropertyItems() ) 198 199 for zprop, value in device_specs.items(): 200 self.log.debug( "Evaluating zProperty <%s -> %s> on %s" % (zprop, value, device.id) ) 201 if not iszprop(zprop): 202 self.log.debug( "Evaluating zProperty <%s -> %s> on %s: not iszprop()" % (zprop, value, device.id) ) 203 continue 204 205 if zprop in dev_zprops: 206 try: 207 self.log.debug( "Setting zProperty <%s -> %s> on %s (currently set to %s)" % ( 208 zprop, value, device.id, getattr(device, zprop, 'notset')) ) 209 device.setZenProperty(zprop, value) 210 except BadRequest: 211 self.log.warn( "Object %s zproperty %s is invalid or duplicate" % ( 212 device.titleOrId(), zprop) ) 213 except Exception, ex: 214 self.log.warn( "Object %s zproperty %s not set (%s)" % ( 215 device.titleOrId(), zprop, ex) ) 216 self.log.debug( "Set zProperty <%s -> %s> on %s (now set to %s)" % ( 217 zprop, value, device.id, getattr(device, zprop, 'notset')) ) 218 else: 219 self.log.warn( "The zproperty %s doesn't exist in %s" % ( 220 zprop, device_specs.get('deviceName', device.id)))

221

222 - def applyCustProps(self, device, device_specs):

223 """ 224 Custom schema properties 225 """ 226 self.log.debug( "Applying custom schema properties..." ) 227 dev_cprops = device.custPropertyMap() 228 229 for cprop, value in device_specs.items(): 230 if not iscustprop(cprop): 231 continue 232 233 matchProps = [prop for prop in dev_cprops if prop['id'] == cprop] 234 if matchProps: 235 ctype = matchProps[0]['type'] 236 if ctype == 'password': 237 ctype = 'string' 238 if ctype in type_converters and value: 239 value = type_converters[ctype](value) 240 device.setZenProperty( cprop, value) 241 else: 242 self.log.warn( "The cproperty %s doesn't exist in %s" % ( 243 cprop, device_specs.get('deviceName', device.id)))

244

245 - def addAllLGSOrganizers(self, device_specs):

246 location = device_specs.get('setLocation') 247 if location: 248 self.addLGSOrganizer('Locations', (location,) ) 249 250 systems = device_specs.get('setSystems') 251 if systems: 252 if not isinstance(systems, list) and not isinstance(systems, tuple): 253 systems = (systems,) 254 self.addLGSOrganizer('Systems', systems) 255 256 groups = device_specs.get('setGroups') 257 if groups: 258 if not isinstance(groups, list) and not isinstance(groups, tuple): 259 groups = (groups,) 260 self.addLGSOrganizer('Groups', groups)

261

262 - def addLGSOrganizer(self, lgsType, paths=[]):

263 """ 264 Add any new locations, groups or organizers 265 """ 266 prefix = '/zport/dmd/' + lgsType 267 base = getattr(self.dmd, lgsType) 268 if hasattr(base, 'sync'): 269 base.sync() 270 existing = [x.getPrimaryUrlPath().replace(prefix, '') \ 271 for x in base.getSubOrganizers()] 272 for path in paths: 273 if path in existing: 274 continue 275 try: 276 base.manage_addOrganizer(path) 277 except BadRequest: 278 pass

279 280 @transact

281 - def addOrganizer(self, device_specs):

282 """ 283 Add any organizers as required, and apply zproperties to them. 284 """ 285 path = device_specs.get('devicePath') 286 baseOrg = path.split('/', 2)[1] 287 base = getattr(self.dmd, baseOrg, None) 288 if base is None: 289 self.log.error("The base of path %s (%s) does not exist -- skipping", 290 baseOrg, path) 291 return 292 293 try: 294 org = base.getDmdObj(path) 295 except KeyError: 296 try: 297 self.log.info("Creating organizer %s", path) 298 @transact 299 def inner(): 300 base.manage_addOrganizer(path)

301 inner() 302 org = base.getDmdObj(path) 303 except IOError, ex: 304 self.log.error("Unable to create organizer! Is Rabbit up and configured correctly?") 305 sys.exit(1) 306 self.applyZProps(org, device_specs) 307 self.applyOtherProps(org, device_specs)

308

309 - def applyOtherProps(self, device, device_specs):

310 """ 311 Apply non-zProperty settings (if any) to the device. 312 313 @parameter device: device to modify 314 @type device: DMD device object 315 @parameter device_specs: device creation dictionary 316 @type device_specs: dictionary 317 """ 318 self.log.debug( "Applying other properties..." ) 319 internalVars = [ 320 'deviceName', 'devicePath', 'comments', 'loader', 'loader_arg_keys', 321 ] 322 @transact 323 def setNamedProp(org, name, description): 324 setattr(org, name, description)

325 326 for functor, value in device_specs.items(): 327 if iszprop(functor) or iscustprop(functor) or functor in internalVars: 328 continue 329 330 # Special case for organizers which can take a description 331 if functor in ('description', 'address'): 332 if hasattr(device, functor): 333 setNamedProp(device, functor, value) 334 continue 335 336 try: 337 self.log.debug("For %s, calling device.%s(%s)", 338 device.id, functor, value) 339 func = getattr(device, functor, None) 340 if func is None or not callable(func): 341 self.log.warn("The function '%s' for device %s is not found.", 342 functor, device.id) 343 elif isinstance(value, (list, tuple)): 344 # The function either expects a list or arguments 345 try: # arguments 346 func(*value) 347 except TypeError: # Try as a list 348 func(value) 349 else: 350 func(value) 351 except ConflictError: 352 raise 353 except Exception: 354 msg = "Device %s device.%s(%s) failed" % (device.id, functor, value) 355 self.reportException(msg, device.id) 356

357 - def runLoader(self, loader, device_specs):

358 """ 359 It's up to the loader now to figure out what's going on. 360 361 @parameter loader: device loader 362 @type loader: callable 363 @parameter device_specs: device entries 364 @type device_specs: dictionary 365 """ 366 argKeys = device_specs.get('loader_arg_keys', []) 367 loader_args = {} 368 for key in argKeys: 369 if key in device_specs: 370 loader_args[key] = device_specs[key] 371 372 result = loader().load_device(self.dmd, **loader_args) 373 374 # If the loader returns back a device object, carry 375 # on processing 376 if isinstance(result, Device): 377 return result 378 return None

379

380 - def processDevices(self, device_list):

381 """ 382 Read the input and process the devices 383 * create the device entry 384 * set zproperties 385 * set custom schema properties 386 * model the device 387 388 @parameter device_list: list of device entries 389 @type device_list: list of dictionaries 390 """ 391 392 def transactional(f): 393 return f if self.options.nocommit else transact(f)

394 395 processed = {'total':0, 'errors':0} 396 397 @transactional 398 def _process(device_specs): 399 # Get the latest bits 400 self.dmd.zport._p_jar.sync() 401 402 loaderName = device_specs.get('loader') 403 if loaderName is not None: 404 try: 405 orgName = device_specs['devicePath'] 406 organizer = self.dmd.getObjByPath('dmd' + orgName) 407 deviceLoader = getUtility(IDeviceLoader, loaderName, organizer) 408 devobj = self.runLoader(deviceLoader, device_specs) 409 except ConflictError: 410 raise 411 except Exception: 412 devName = device_specs.get('device_specs', 'Unkown Device') 413 msg = "Ignoring device loader issue for %s" % devName 414 self.reportException(msg, devName, specs=str(device_specs)) 415 processed['errors'] += 1 416 return 417 else: 418 devobj = self.getDevice(device_specs) 419 deviceLoader = None 420 421 if devobj is None: 422 if deviceLoader is not None: 423 processed['total'] += 1 424 else: 425 self.addAllLGSOrganizers(device_specs) 426 self.applyZProps(devobj, device_specs) 427 self.applyCustProps(devobj, device_specs) 428 self.applyOtherProps(devobj, device_specs) 429 430 return devobj 431 432 @transactional 433 def _snmp_community(device_specs, devobj): 434 # Discover the SNMP community if it isn't explicitly set. 435 if 'zSnmpCommunity' not in device_specs: 436 self.log.debug('Discovering SNMP version and community') 437 devobj.manage_snmpCommunity() 438 439 @transactional 440 def _model(devobj): 441 try: 442 devobj.collectDevice(setlog=self.options.showModelOutput) 443 except ConflictError: 444 raise 445 except Exception, ex: 446 msg = "Modeling error for %s" % devobj.id 447 self.reportException(msg, devobj.id, exception=str(ex)) 448 processed['errors'] += 1 449 processed['total'] += 1 450 451 for device_specs in device_list: 452 devobj = _process(device_specs) 453 454 # We need to commit in order to model, so don't bother 455 # trying to model unless we can do both 456 if devobj and not self.options.nocommit and not self.options.nomodel: 457 _snmp_community(device_specs, devobj) 458 _model(devobj) 459 460 self.reportResults(processed, len(device_list)) 461

462 - def reportException(self, msg, devName='', **kwargs):

463 """ 464 Report exceptions back to the the event console 465 """ 466 self.log.exception(msg) 467 if not self.options.nocommit: 468 evt = self.baseEvent.copy() 469 evt.update(dict( 470 summary=msg, 471 traceback=format_exc() 472 )) 473 evt.update(kwargs) 474 if devName: 475 evt['device'] = devName 476 self.dmd.ZenEventManager.sendEvent(evt)

477

478 - def reportResults(self, processed, totalDevices):

479 """ 480 Report the success + total counts from loading devices. 481 """ 482 msg = "Modeled %d of %d devices, with %d errors" % ( 483 processed['total'], totalDevices, processed['errors'] ) 484 self.log.info(msg) 485 486 if not self.options.nocommit: 487 evt = self.baseEvent.copy() 488 evt.update(dict( 489 severity=SEVERITY_INFO, 490 summary=msg, 491 modeled=processed['total'], 492 errors=processed['errors'], 493 total=totalDevices, 494 )) 495 self.dmd.ZenEventManager.sendEvent(evt)

496

497 - def notifyNewDeviceCreated(self, deviceName):

498 """ 499 Report that we added a new device. 500 """ 501 if not self.options.nocommit: 502 evt = self.baseEvent.copy() 503 evt.update(dict( 504 severity=SEVERITY_INFO, 505 summary= "Added new device %s" % deviceName 506 )) 507 self.dmd.ZenEventManager.sendEvent(evt)

508

509 - def getDevice(self, device_specs):

510 """ 511 Find or create the specified device 512 513 @parameter device_specs: device creation dictionary 514 @type device_specs: dictionary 515 @return: device or None 516 @rtype: DMD device object 517 """ 518 if 'deviceName' not in device_specs: 519 return None 520 name = device_specs['deviceName'] 521 devobj = self.dmd.Devices.findDevice(name) 522 if devobj is not None: 523 self.log.info("Found existing device %s" % name) 524 return devobj 525 526 specs = {} 527 for key in self.loader_args: 528 if key in device_specs: 529 specs[key] = device_specs[key] 530 531 try: 532 self.log.info("Creating device %s" % name) 533 534 # Do NOT model at this time 535 specs['discoverProto'] = 'none' 536 537 self.loader(**specs) 538 devobj = self.dmd.Devices.findDevice(name) 539 if devobj is None: 540 self.log.error("Unable to find newly created device %s -- skipping" \ 541 % name) 542 else: 543 self.notifyNewDeviceCreated(name) 544 545 except Exception: 546 msg = "Unable to load %s -- skipping" % name 547 self.reportException(msg, name) 548 549 return devobj

550

551 - def buildOptions(self):

552 """ 553 Add our command-line options to the basics 554 """ 555 ZCmdBase.buildOptions(self) 556 557 self.parser.add_option('--show_options', 558 dest="show_options", default=False, 559 action="store_true", 560 help="Show the various options understood by the loader") 561 562 self.parser.add_option('--sample_configs', 563 dest="sample_configs", default=False, 564 action="store_true", 565 help="Show an example configuration file.") 566 567 self.parser.add_option('--showModelOutput', 568 dest="showModelOutput", default=True, 569 action="store_false", 570 help="Show modelling activity") 571 572 self.parser.add_option('--nocommit', 573 dest="nocommit", default=False, 574 action="store_true", 575 help="Don't commit changes to the ZODB. Use for verifying config file.") 576 577 self.parser.add_option('--nomodel', 578 dest="nomodel", default=False, 579 action="store_true", 580 help="Don't model the remote devices. Must be able to commit changes.")

581

582 - def parseDevices(self, data):

583 """ 584 From the list of strings in rawDevices, construct a list 585 of device dictionaries, ready to load into Zenoss. 586 587 @parameter data: list of strings representing device entries 588 @type data: list of strings 589 @return: list of parsed device entries 590 @rtype: list of dictionaries 591 """ 592 if not data: 593 return [] 594 595 comment = re.compile(r'^\s*#.*') 596 597 defaults = {'devicePath':"/Discovered" } 598 finalList = [] 599 i = 0 600 while i < len(data): 601 line = data[i] 602 line = re.sub(comment, '', line).strip() 603 if line == '': 604 i += 1 605 continue 606 607 # Check for line continuation character '\' 608 while line[-1] == '\\' and i < len(data): 609 i += 1 610 line = line[:-1] + data[i] 611 line = re.sub(comment, '', line).strip() 612 613 if line[0] == '/' or line[1] == '/': # Found an organizer 614 defaults = self.parseDeviceEntry(line, {}) 615 if defaults is None: 616 defaults = {'devicePath':"/Discovered" } 617 else: 618 defaults['devicePath'] = defaults['deviceName'] 619 del defaults['deviceName'] 620 self.addOrganizer(defaults) 621 622 else: 623 configs = self.parseDeviceEntry(line, defaults) 624 if configs: 625 finalList.append(configs) 626 i += 1 627 628 return finalList

629

630 - def parseDeviceEntry(self, line, defaults):

631 """ 632 Build a dictionary of properties from one line's input 633 634 @parameter line: string containing one device's info 635 @type line: string 636 @parameter defaults: dictionary of default settings 637 @type defaults: dictionary 638 @return: parsed device entry 639 @rtype: dictionary 640 """ 641 options = [] 642 # Note: organizers and device names can have spaces in them 643 if line[0] in ["'", '"']: 644 delim = line[0] 645 eom = line.find(delim, 1) 646 if eom == -1: 647 self.log.error("While reading name, unable to parse" \ 648 " the entry for %s -- skipping", line ) 649 return None 650 name = line[1:eom] 651 options = line[eom+1:] 652 653 else: 654 options = line.split(None, 1) 655 name = options.pop(0) 656 if options: 657 options = options.pop(0) 658 659 configs = defaults.copy() 660 configs['deviceName'] = name 661 662 if options: 663 try: 664 # Add a newline to allow for trailing comments 665 evalString = 'dict(' + options + '\n)' 666 configs.update(eval(evalString)) 667 except: 668 self.log.error( "Unable to parse the entry for %s -- skipping" % name ) 669 self.log.error( "Raw string: %s" % options ) 670 return None 671 672 return configs

673 674 675 if __name__=='__main__': 676 batchLoader = BatchDeviceLoader() 677 if batchLoader.options.show_options: 678 print "Options = %s" % sorted( batchLoader.loader_args.keys() ) 679 help(batchLoader.loader) 680 sys.exit(0) 681 682 if batchLoader.options.sample_configs: 683 print batchLoader.sample_configs 684 sys.exit(0) 685 686 device_list = batchLoader.loadDeviceList() 687 if not device_list: 688 batchLoader.log.warn("No device entries found to load.") 689 sys.exit(1) 690 691 batchLoader.processDevices(device_list) 692 sys.exit(0) 693

Source Code for Module Products.ZenModel.BatchDeviceLoader