Products.ZenRelations.ImportRM

47 """ 48 Wrapper module to interface between Zope and the Python SAX XML library. 49 The xml.sax.parse() calls different routines depending on what it finds. 50 51 A simple example of a valid XML file can be found in the objects.xml file 52 for a ZenPack. 53 54 <?xml version="1.0"?> 55 <objects> 56  57 <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'> 58 <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property> 59 <tomanycont id='datasources'> 60 <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'> 61 <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property> 62 <property type="boolean" id="enabled" mode="w" > True </property> 63 <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property> 64 <property type="int" id="severity" mode="w" > 3 </property> 65 <property type="int" id="cycletime" mode="w" > 300 </property> 66 <property type="boolean" id="usessh" mode="w" > False </property> 67 <tomanycont id='datapoints'> 68 <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'> 69 <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property> 70 <property type="boolean" id="isrow" mode="w" > True </property> 71 </object> 72 </tomanycont> 73 </object> 74 75  76 77 </objects> 78 """ 79 rootpath = '' 80 skipobj = 0 81

82 - def __init__(self, noopts=0, app=None, keeproot=False):

83 """ 84 Initializer 85 86 @param noopts: don't use sys.argv for command-line options 87 @type noopts: boolean 88 @param app: app 89 @type app: object 90 @param keeproot: keeproot 91 @type keeproot: boolean 92 """ 93 ZCmdBase.__init__(self, noopts, app, keeproot) 94 ContentHandler.__init__(self)

95

96 - def context(self):

97 """ 98 Return the bottom object in the stack 99 100 @return: 101 @rtype: object 102 """ 103 return self.objstack[-1]

104

105 - def cleanattrs(self, attrs):

106 """ 107 Convert all attributes to string values 108 109 @param attrs: (key,val) pairs 110 @type attrs: list 111 @return: same list, but with all values as strings 112 @rtype: list 113 """ 114 myattrs = {} 115 for (key, val) in attrs.items(): 116 myattrs[key] = str(val) 117 return myattrs

118

119 - def startElement(self, name, attrs):

120 """ 121 Function called when the parser finds the starting element 122 in the XML file. 123 124 @param name: name of the element 125 @type name: string 126 @param attrs: list of (key, value) tuples 127 @type attrs: list 128 """ 129 ignoredElements = [ 'objects' ] 130 attrs = self.cleanattrs(attrs) 131 if self.skipobj > 0: 132 self.skipobj += 1 133 return 134 135 self.log.debug('tag %s, context %s, line %s' % ( 136 name, self.context().id, self._locator.getLineNumber() )) 137 138 if name == 'object': 139 if attrs.get('class') == 'Device': 140 devId = attrs['id'].split('/')[-1] 141 dev = self.dmd.Devices.findDeviceByIdOrIp(devId) 142 if dev: 143 msg = 'The device %s already exists on this system! (Line %s)' % \ 144 (devId, self._locator.getLineNumber()) 145 raise Exception(msg) 146 147 if attrs.get('class') == 'IpAddress': 148 ipAddress = attrs['id'] 149 dev = self.dmd.Devices.findDeviceByIdOrIp(ipAddress) 150 if dev: 151 msg = 'The IP address %s already exists on this system! (Line %s)' % \ 152 (ipAddress, self._locator.getLineNumber()) 153 raise Exception(msg) 154 155 obj = self.createObject(attrs) 156 if obj is None: 157 formattedAttrs = '' 158 for key, value in attrs.items(): 159 formattedAttrs += ' * %s: %s\n' % (key, value) 160 raise Exception('Unable to create object using the following ' 161 'attributes:\n%s' % formattedAttrs) 162 163 if not self.options.noindex and hasattr(aq_base(obj), 164 'reIndex') and not self.rootpath: 165 self.rootpath = obj.getPrimaryId() 166 167 self.objstack.append(obj) 168 169 elif name == 'tomanycont' or name == 'tomany': 170 nextobj = self.context()._getOb(attrs['id'], None) 171 if nextobj is None: 172 self.skipobj = 1 173 return 174 else: 175 self.objstack.append(nextobj) 176 elif name == 'toone': 177 relname = attrs.get('id') 178 self.log.debug('toone %s, on object %s', relname, 179 self.context().id) 180 rel = getattr(aq_base(self.context()), relname, None) 181 if rel is None: 182 return 183 objid = attrs.get('objid') 184 self.addLink(rel, objid) 185 elif name == 'link': 186 self.addLink(self.context(), attrs['objid']) 187 elif name == 'property': 188 self.curattrs = attrs 189 elif name in ignoredElements: 190 pass 191 else: 192 self.log.warning( "Ignoring an unknown XML element type: %s" % name )

193

194 - def endElement(self, name):

195 """ 196 Function called when the parser finds the starting element 197 in the XML file. 198 199 @param name: name of the ending element 200 @type name: string 201 """ 202 ignoredElements = [ 'toone', 'link' ] 203 if self.skipobj > 0: 204 self.skipobj -= 1 205 return 206 207 noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0 208 209 if name in ('object', 'tomany', 'tomanycont'): 210 obj = self.objstack.pop() 211 notify(IndexingEvent(obj)) 212 if hasattr(aq_base(obj), 'index_object'): 213 obj.index_object() 214 if self.rootpath == obj.getPrimaryId(): 215 self.log.info('Calling reIndex %s', obj.getPrimaryId()) 216 obj.reIndex() 217 self.rootpath = '' 218 if (not noIncrementalCommit and 219 not self.objectnumber % self.options.chunk_size): 220 self.log.debug("Committing a batch of %s objects" % 221 self.options.chunk_size) 222 self.commit() 223 224 elif name == 'objects': # ie end of the file 225 self.log.info('End loading objects') 226 self.log.info('Processing links') 227 self.processLinks() 228 if not self.options.noCommit: 229 self.commit() 230 self.log.info('Loaded %d objects into the ZODB database' 231 % self.objectnumber) 232 else: 233 self.log.info('Would have created %d objects in the ZODB database' 234 % self.objectnumber) 235 236 elif name == 'property': 237 self.setProperty(self.context(), self.curattrs, 238 self.charvalue) 239 # We've closed off a tag, so now we need to re-initialize 240 # the area that stores the contents of elements 241 self.charvalue = '' 242 243 elif name in ignoredElements: 244 pass 245 else: 246 self.log.warning( "Ignoring an unknown XML element type: %s" % name )

247

248 - def characters(self, chars):

249 """ 250 Called by xml.sax.parse() with data found in an element 251 eg <object>my characters stuff</object> 252 253 Note that this can be called character by character. 254 255 @param chars: chars 256 @type chars: string 257 """ 258 self.charvalue += saxutils.unescape(chars)

259

260 - def createObject(self, attrs):

261 """ 262 Create an object and set it into its container 263 264 @param attrs: attrs 265 @type attrs: string 266 @return: newly created object 267 @rtype: object 268 """ 269 # Does the object exist already? 270 id = attrs.get('id') 271 obj = None 272 try: 273 if id.startswith('/'): 274 obj = getObjByPath(self.app, id) 275 else: 276 obj = self.context()._getOb(id) 277 except (KeyError, AttributeError, NotFound): 278 pass 279 280 if obj is None: 281 klass = importClass(attrs.get('module'), attrs.get('class')) 282 if id.find('/') > -1: 283 (contextpath, id) = os.path.split(id) 284 try: 285 pathobj = getObjByPath(self.context(), contextpath) 286 except (KeyError, AttributeError, NotFound): 287 self.log.warn( "Unable to find context path %s (line %s ?) for %s" % ( 288 contextpath, self._locator.getLineNumber(), id )) 289 if not self.options.noCommit: 290 self.log.warn( "Not committing any changes" ) 291 self.options.noCommit = True 292 return None 293 self.objstack.append(pathobj) 294 self.log.debug('Building instance %s of class %s',id,klass.__name__) 295 try: 296 if klass.__name__ == 'AdministrativeRole': 297 user = [x for x in self.dmd.ZenUsers.objectValues() if x.id == id] 298 if user: 299 obj = klass(user[0], self.context().device()) 300 else: 301 msg = "No AdminRole user %s exists (line %s)" % ( 302 id, self._locator.getLineNumber()) 303 self.log.error(msg) 304 raise Exception(msg) 305 else: 306 obj = klass(id) 307 except TypeError, ex: 308 # This happens when the constructor expects more arguments 309 self.log.exception("Unable to build %s instance of class %s (line %s)", 310 id, klass.__name__, self._locator.getLineNumber()) 311 raise 312 self.context()._setObject(obj.id, obj) 313 obj = self.context()._getOb(obj.id) 314 self.objectnumber += 1 315 self.log.debug('Added object %s to database' 316 % obj.getPrimaryId()) 317 else: 318 self.log.debug('Object %s already exists -- skipping' % id) 319 return obj

320

321 - def setZendoc(self, obj, value):

322 zendocObj = zope.component.queryAdapter(obj, IZenDocProvider) 323 if zendocObj is not None: 324 zendocObj.setZendoc( value ) 325 elif value: 326 self.log.warn('zendoc property could not be set to' + 327 ' %s on object %s' % ( value, obj.id ) )

328

329 - def setProperty(self, obj, attrs, value):

330 """ 331 Set the value of a property on an object. 332 333 @param obj: obj 334 @type obj: string 335 @param attrs: attrs 336 @type attrs: string 337 @param value: value 338 @type value: string 339 @return: 340 @rtype: 341 """ 342 name = attrs.get('id') 343 proptype = attrs.get('type') 344 setter = attrs.get('setter', None) 345 self.log.debug('Setting object %s attr %s type %s value %s' 346 % (obj.id, name, proptype, value)) 347 linenum = self._locator.getLineNumber() 348 349 # Sanity check the value 350 value = value.strip() 351 try: 352 value = str(value) 353 except UnicodeEncodeError: 354 self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \ 355 ' str(%s) for object %s attribute %s -- ignoring' % ( 356 obj.id, name, proptype, value)) 357 358 if name == 'zendoc': 359 return self.setZendoc( obj, value ) 360 361 # Guess at how to interpret the value given the property type 362 if proptype == 'selection': 363 try: 364 firstElement = getattr(obj, name)[0] 365 if isinstance(firstElement, basestring): 366 proptype = 'string' 367 except (TypeError, IndexError): 368 self.log.warn("Error at line %s when trying to " % linenum + \ 369 " use (%s) as the value for object %s attribute %s -- assuming a string" 370 % (obj.id, name, proptype, value)) 371 proptype = 'string' 372 373 if proptype == 'date': 374 try: 375 value = float(value) 376 except ValueError: 377 pass 378 value = DateTime(value) 379 380 elif proptype not in _STRING_PROPERTY_TYPES: 381 try: 382 value = eval(value) 383 except NameError: 384 self.log.warn('Error trying to evaluate %s at line %s', 385 value, linenum) 386 except SyntaxError: 387 self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % ( 388 linenum, value) ) 389 390 # Actually use the value 391 if not obj.hasProperty(name): 392 obj._setProperty(name, value, type=proptype, setter=setter) 393 else: 394 obj._updateProperty(name, value)

395

396 - def addLink(self, rel, objid):

397 """ 398 Build list of links to form after all objects have been created 399 make sure that we don't add other side of a bidirectional relation 400 401 @param rel: relationship object 402 @type rel: relation object 403 @param objid: objid 404 @type objid: string 405 """ 406 self.links.append((rel.getPrimaryId(), objid))

407

408 - def processLinks(self):

409 """ 410 Walk through all the links that we saved and link them up 411 """ 412 for (relid, objid) in self.links: 413 try: 414 self.log.debug('Linking relation %s to object %s', 415 relid, objid) 416 rel = getObjByPath(self.app, relid) 417 obj = getObjByPath(self.app, objid) 418 if not rel.hasobject(obj): 419 rel.addRelation(obj) 420 except: 421 self.log.critical('Failed linking relation %s to object %s' % ( 422 relid, objid))

423

424 - def buildOptions(self):

425 """ 426 Command-line options specific to this command 427 """ 428 ZCmdBase.buildOptions(self) 429 self.parser.add_option('-i', '--infile', dest='infile', 430 help='Input file for import. Default is stdin' 431 ) 432 self.parser.add_option('--noindex', dest='noindex', 433 action='store_true', default=False, 434 help='Do not try to index the freshly loaded objects.' 435 ) 436 self.parser.add_option('--chunksize', dest='chunk_size', 437 help='Number of objects to commit at a time.', 438 type='int', 439 default=100 440 ) 441 self.parser.add_option( 442 '-n', 443 '--noCommit', 444 dest='noCommit', 445 action='store_true', 446 default=0, 447 help='Do not store changes to the DMD (ie for debugging purposes)', 448 )

449

450 - def loadObjectFromXML(self, xmlfile=''):

451 """ 452 This method can be used to load data for the root of Zenoss (default 453 behavior) or it can be used to operate on a specific point in the 454 Zenoss hierarchy (ZODB). 455 456 Upon loading the XML file to be processed, the content of the XML file 457 is handled by the methods in this class when called by xml.sax.parse(). 458 459 Reads from a file if xmlfile is specified, otherwise reads 460 from the command-line option --infile. If no files are found from 461 any of these places, read from standard input. 462 463 @param xmlfile: Name of XML file to load, or file-like object 464 @type xmlfile: string or file-like object 465 """ 466 self.objstack = [self.app] 467 self.links = [] 468 self.objectnumber = 0 469 self.charvalue = '' 470 if xmlfile and isinstance(xmlfile, basestring): 471 self.infile = open(xmlfile) 472 elif hasattr(xmlfile, 'read'): 473 self.infile = xmlfile 474 elif self.options.infile: 475 self.infile = open(self.options.infile) 476 else: 477 self.infile = sys.stdin 478 parser = make_parser() 479 parser.setContentHandler(self) 480 try: 481 parser.parse(self.infile) 482 except SAXParseException, ex: 483 self.log.error("XML parse error at line %d column %d: %s", 484 ex.getLineNumber(), ex.getColumnNumber(), 485 ex.getMessage()) 486 finally: 487 self.infile.close()

488

489 - def loadDatabase(self):

490 """ 491 The default behavior of loadObjectFromXML() will be to use the Zope 492 app object, and thus operatate on the whole of Zenoss. 493 """ 494 self.loadObjectFromXML()

495

496 - def commit(self):

497 """ 498 Wrapper around the Zope database commit() 499 """ 500 trans = transaction.get() 501 trans.note('Import from file %s using %s' 502 % (self.options.infile, self.__class__.__name__)) 503 trans.commit() 504 if hasattr(self, 'connection'): 505 # It's safe to call syncdb() 506 self.syncdb()

Source Code for Module Products.ZenRelations.ImportRM