Products.ZenRelations.ImportRM

52 """ 53 Wrapper module to interface between Zope and the Python SAX XML library. 54 The xml.sax.parse() calls different routines depending on what it finds. 55 56 A simple example of a valid XML file can be found in the objects.xml file 57 for a ZenPack. 58 59 <?xml version="1.0"?> 60 <objects> 61  62 <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'> 63 <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property> 64 <tomanycont id='datasources'> 65 <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'> 66 <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property> 67 <property type="boolean" id="enabled" mode="w" > True </property> 68 <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property> 69 <property type="int" id="severity" mode="w" > 3 </property> 70 <property type="int" id="cycletime" mode="w" > 300 </property> 71 <property type="boolean" id="usessh" mode="w" > False </property> 72 <tomanycont id='datapoints'> 73 <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'> 74 <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property> 75 <property type="boolean" id="isrow" mode="w" > True </property> 76 </object> 77 </tomanycont> 78 </object> 79 80  81 82 </objects> 83 """ 84 rootpath = '' 85 skipobj = 0 86

87 - def __init__(self, noopts=0, app=None, keeproot=False):

88 """ 89 Initializer 90 91 @param noopts: don't use sys.argv for command-line options 92 @type noopts: boolean 93 @param app: app 94 @type app: object 95 @param keeproot: keeproot 96 @type keeproot: boolean 97 """ 98 ZCmdBase.__init__(self, noopts, app, keeproot) 99 ContentHandler.__init__(self)

100

101 - def context(self):

102 """ 103 Return the bottom object in the stack 104 105 @return: 106 @rtype: object 107 """ 108 return self.objstack[-1]

109

110 - def cleanattrs(self, attrs):

111 """ 112 Convert all attributes to string values 113 114 @param attrs: (key,val) pairs 115 @type attrs: list 116 @return: same list, but with all values as strings 117 @rtype: list 118 """ 119 myattrs = {} 120 for (key, val) in attrs.items(): 121 myattrs[key] = str(val) 122 return myattrs

123

124 - def startElement(self, name, attrs):

125 """ 126 Function called when the parser finds the starting element 127 in the XML file. 128 129 @param name: name of the element 130 @type name: string 131 @param attrs: list of (key, value) tuples 132 @type attrs: list 133 """ 134 ignoredElements = [ 'objects' ] 135 attrs = self.cleanattrs(attrs) 136 if self.skipobj > 0: 137 self.skipobj += 1 138 return 139 140 self.log.debug('tag %s, context %s, line %s' % ( 141 name, self.context().id, self._locator.getLineNumber() )) 142 143 if name == 'object': 144 145 obj = self.createObject(attrs) 146 147 if obj is None: 148 formattedAttrs = '' 149 for key, value in attrs.items(): 150 formattedAttrs += ' * %s: %s\n' % (key, value) 151 raise Exception('Unable to create object using the following ' 152 'attributes:\n%s' % formattedAttrs) 153 154 if not self.options.noindex and hasattr(aq_base(obj), 155 'reIndex') and not self.rootpath: 156 self.rootpath = obj.getPrimaryId() 157 158 self.objstack.append(obj) 159 160 elif name == 'tomanycont' or name == 'tomany': 161 nextobj = self.context()._getOb(attrs['id'], None) 162 if nextobj is None: 163 self.skipobj = 1 164 return 165 else: 166 self.objstack.append(nextobj) 167 elif name == 'toone': 168 relname = attrs.get('id') 169 self.log.debug('toone %s, on object %s', relname, 170 self.context().id) 171 rel = getattr(aq_base(self.context()), relname, None) 172 if rel is None: 173 return 174 objid = attrs.get('objid') 175 self.addLink(rel, objid) 176 elif name == 'link': 177 self.addLink(self.context(), attrs['objid']) 178 elif name == 'property': 179 self.curattrs = attrs 180 elif name in ignoredElements: 181 pass 182 else: 183 self.log.warning( "Ignoring an unknown XML element type: %s" % name )

184

185 - def endElement(self, name):

186 """ 187 Function called when the parser finds the starting element 188 in the XML file. 189 190 @param name: name of the ending element 191 @type name: string 192 """ 193 ignoredElements = [ 'toone', 'link' ] 194 if self.skipobj > 0: 195 self.skipobj -= 1 196 return 197 198 noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0 199 200 if name in ('object', 'tomany', 'tomanycont'): 201 obj = self.objstack.pop() 202 notify(IndexingEvent(obj)) 203 if hasattr(aq_base(obj), 'index_object'): 204 obj.index_object() 205 if self.rootpath == obj.getPrimaryId(): 206 self.log.info('Calling reIndex %s', obj.getPrimaryId()) 207 obj.reIndex() 208 self.rootpath = '' 209 if (not noIncrementalCommit and 210 not self.objectnumber % self.options.chunk_size): 211 self.log.debug("Committing a batch of %s objects" % 212 self.options.chunk_size) 213 self.commit() 214 215 elif name == 'objects': # ie end of the file 216 self.log.info('End loading objects') 217 self.log.info('Processing links') 218 self.processLinks() 219 if not self.options.noCommit: 220 self.commit() 221 self.log.info('Loaded %d objects into the ZODB database' 222 % self.objectnumber) 223 else: 224 self.log.info('Would have created %d objects in the ZODB database' 225 % self.objectnumber) 226 227 elif name == 'property': 228 self.setProperty(self.context(), self.curattrs, 229 self.charvalue) 230 # We've closed off a tag, so now we need to re-initialize 231 # the area that stores the contents of elements 232 self.charvalue = '' 233 234 elif name in ignoredElements: 235 pass 236 else: 237 self.log.warning( "Ignoring an unknown XML element type: %s" % name )

238

239 - def characters(self, chars):

240 """ 241 Called by xml.sax.parse() with data found in an element 242 eg <object>my characters stuff</object> 243 244 Note that this can be called character by character. 245 246 @param chars: chars 247 @type chars: string 248 """ 249 self.charvalue += saxutils.unescape(chars)

250

251 - def createObject(self, attrs):

252 """ 253 Create an object and set it into its container 254 255 @param attrs: attrs 256 @type attrs: string 257 @return: newly created object 258 @rtype: object 259 """ 260 # Does the object exist already? 261 id = attrs.get('id') 262 obj = None 263 try: 264 if id.startswith('/'): 265 obj = getObjByPath(self.app, id) 266 else: 267 obj = self.context()._getOb(id) 268 except (KeyError, AttributeError, NotFound): 269 pass 270 271 if obj is None: 272 klass = importClass(attrs.get('module'), attrs.get('class')) 273 if id.find('/') > -1: 274 (contextpath, id) = os.path.split(id) 275 try: 276 pathobj = getObjByPath(self.context(), contextpath) 277 except (KeyError, AttributeError, NotFound): 278 self.log.warn( "Unable to find context path %s (line %s ?) for %s" % ( 279 contextpath, self._locator.getLineNumber(), id )) 280 if not self.options.noCommit: 281 self.log.warn( "Not committing any changes" ) 282 self.options.noCommit = True 283 return None 284 self.objstack.append(pathobj) 285 self.log.debug('Building instance %s of class %s',id,klass.__name__) 286 obj = klass(id) 287 self.context()._setObject(obj.id, obj) 288 obj = self.context()._getOb(obj.id) 289 self.objectnumber += 1 290 if self.objectnumber % 5000 == 0: 291 transaction.savepoint() 292 self.log.debug('Added object %s to database' 293 % obj.getPrimaryId()) 294 else: 295 self.log.debug('Object %s already exists -- skipping' % id) 296 return obj

297

298 - def setZendoc(self, obj, value):

299 zendocObj = zope.component.queryAdapter(obj, IZenDocProvider) 300 if zendocObj is not None: 301 zendocObj.setZendoc( value ) 302 elif value: 303 self.log.warn('zendoc property could not be set to' + 304 ' %s on object %s' % ( value, obj.id ) )

305

306 - def setProperty(self, obj, attrs, value):

307 """ 308 Set the value of a property on an object. 309 310 @param obj: obj 311 @type obj: string 312 @param attrs: attrs 313 @type attrs: string 314 @param value: value 315 @type value: string 316 @return: 317 @rtype: 318 """ 319 name = attrs.get('id') 320 proptype = attrs.get('type') 321 setter = attrs.get('setter', None) 322 self.log.debug('Setting object %s attr %s type %s value %s' 323 % (obj.id, name, proptype, value)) 324 linenum = self._locator.getLineNumber() 325 326 # Sanity check the value 327 value = value.strip() 328 try: 329 value = str(value) 330 except UnicodeEncodeError: 331 self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \ 332 ' str(%s) for object %s attribute %s -- ignoring' % ( 333 obj.id, name, proptype, value)) 334 335 if name == 'zendoc': 336 return self.setZendoc( obj, value ) 337 338 # Guess at how to interpret the value given the property type 339 if proptype == 'selection': 340 try: 341 firstElement = getattr(obj, name)[0] 342 if type(firstElement) in types.StringTypes: 343 proptype = 'string' 344 except (TypeError, IndexError): 345 self.log.warn("Error at line %s when trying to " % linenum + \ 346 " use (%s) as the value for object %s attribute %s -- assuming a string" 347 % (obj.id, name, proptype, value)) 348 proptype = 'string' 349 350 if proptype == 'date': 351 try: 352 value = float(value) 353 except ValueError: 354 pass 355 value = DateTime(value) 356 357 elif proptype not in _STRING_PROPERTY_TYPES: 358 try: 359 value = eval(value) 360 except NameError: 361 self.log.exception( 'Error trying to evaluate %s', value ) 362 raise 363 except SyntaxError: 364 self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % ( 365 linenum, value) ) 366 367 # Actually use the value 368 if not obj.hasProperty(name): 369 obj._setProperty(name, value, type=proptype, setter=setter) 370 else: 371 obj._updateProperty(name, value)

372

373 - def addLink(self, rel, objid):

374 """ 375 Build list of links to form after all objects have been created 376 make sure that we don't add other side of a bidirectional relation 377 378 @param rel: relationship object 379 @type rel: relation object 380 @param objid: objid 381 @type objid: string 382 """ 383 self.links.append((rel.getPrimaryId(), objid))

384

385 - def processLinks(self):

386 """ 387 Walk through all the links that we saved and link them up 388 """ 389 for (relid, objid) in self.links: 390 try: 391 self.log.debug('Linking relation %s to object %s', 392 relid, objid) 393 rel = getObjByPath(self.app, relid) 394 obj = getObjByPath(self.app, objid) 395 if not rel.hasobject(obj): 396 rel.addRelation(obj) 397 except: 398 self.log.critical('Failed linking relation %s to object %s' % ( 399 relid, objid))

400

401 - def buildOptions(self):

402 """ 403 Command-line options specific to this command 404 """ 405 ZCmdBase.buildOptions(self) 406 self.parser.add_option('-i', '--infile', dest='infile', 407 help='Input file for import. Default is stdin' 408 ) 409 self.parser.add_option('--noindex', dest='noindex', 410 action='store_true', default=False, 411 help='Do not try to index the freshly loaded objects.' 412 ) 413 self.parser.add_option('--chunksize', dest='chunk_size', 414 help='Number of objects to commit at a time.', 415 type='int', 416 default=100 417 ) 418 self.parser.add_option( 419 '-n', 420 '--noCommit', 421 dest='noCommit', 422 action='store_true', 423 default=0, 424 help='Do not store changes to the DMD (ie for debugging purposes)', 425 )

426

427 - def loadObjectFromXML(self, xmlfile=''):

428 """ 429 This method can be used to load data for the root of Zenoss (default 430 behavior) or it can be used to operate on a specific point in the 431 Zenoss hierarchy (ZODB). 432 433 Upon loading the XML file to be processed, the content of the XML file 434 is handled by the methods in this class when called by xml.sax.parse(). 435 436 Reads from a file if xmlfile is specified, otherwise reads 437 from the command-line option --infile. If no files are found from 438 any of these places, read from standard input. 439 440 @param xmlfile: Name of XML file to load, or file-like object 441 @type xmlfile: string or file-like object 442 """ 443 self.objstack = [self.app] 444 self.links = [] 445 self.objectnumber = 0 446 self.charvalue = '' 447 if xmlfile and type(xmlfile) in types.StringTypes: 448 self.infile = open(xmlfile) 449 elif hasattr(xmlfile, 'read'): 450 self.infile = xmlfile 451 elif self.options.infile: 452 self.infile = open(self.options.infile) 453 else: 454 self.infile = sys.stdin 455 parser = make_parser() 456 parser.setContentHandler(self) 457 try: 458 parser.parse(self.infile) 459 except SAXParseException, ex: 460 self.log.error("XML parse error at line %d column %d: %s", 461 ex.getLineNumber(), ex.getColumnNumber(), 462 ex.getMessage()) 463 finally: 464 self.infile.close()

465

466 - def loadDatabase(self):

467 """ 468 The default behavior of loadObjectFromXML() will be to use the Zope 469 app object, and thus operatate on the whole of Zenoss. 470 """ 471 self.loadObjectFromXML()

472

473 - def commit(self):

474 """ 475 Wrapper around the Zope database commit() 476 """ 477 trans = transaction.get() 478 trans.note('Import from file %s using %s' 479 % (self.options.infile, self.__class__.__name__)) 480 trans.commit() 481 if hasattr(self, 'connection'): 482 # It's safe to call syncdb() 483 self.syncdb()

Source Code for Module Products.ZenRelations.ImportRM