Package Products :: Package ZenRelations :: Module ImportRM |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRelations.ImportRM

      1  #! /usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  # ##########################################################################
      4  #
      5  # This program is part of Zenoss Core, an open source monitoring platform.
      6  # Copyright (C) 2009, Zenoss Inc.
      7  #
      8  # This program is free software; you can redistribute it and/or modify it
      9  # under the terms of the GNU General Public License version 2 as published by
     10  # the Free Software Foundation.
     11  #
     12  # For complete information please visit: http://www.zenoss.com/oss/
     13  #
     14  # ##########################################################################
     15
     16  __doc__ = """ImportRM
     17  Import RelationshipManager objects into a Zope database
     18  This provides support methods used by the Python xml.sax library to
     19  parse and construct an XML document.
     20
     21  Descriptions of the XML document format can be found in the
     22  Developers Guide.
     23  """
     24  import Globals
     25  import sys
     26  import os
     27  import types
     28  import transaction
     29  import zope.component
     30  from zope.event import notify
     31  from DateTime import DateTime
     32  from xml.sax import make_parser, saxutils, SAXParseException
     33  from xml.sax.handler import ContentHandler
     34
     35  from Acquisition import aq_base
     36  from zExceptions import NotFound
     37  from OFS.PropertyManager import PropertyManager
     38
     39  from Products.ZenUtils.ZCmdBase import ZCmdBase
     40  from Products.ZenUtils.Utils import importClass
     41  from Products.ZenUtils.Utils import getObjByPath
     42
     43  from Products.ZenModel.interfaces import IZenDocProvider
     44  from Products.ZenRelations.Exceptions import *
     45  from Products.Zuul.catalog.events import IndexingEvent
     46
     47  _STRING_PROPERTY_TYPES = ( 'string', 'text', 'password' )
     48
     49
     50
     51 -class ImportRM(ZCmdBase, ContentHandler):
     52      """
     53      Wrapper module to interface between Zope and the Python SAX XML library.
     54  The xml.sax.parse() calls different routines depending on what it finds.
     55
     56  A simple example of a valid XML file can be found in the objects.xml file
     57  for a ZenPack.
     58
     59   <?xml version="1.0"?>
     60   <objects>
     61     <!-- ('', 'zport', 'dmd', 'Devices', 'rrdTemplates', 'HelloWorld') -->
     62     <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'>
     63       <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property>
     64     <tomanycont id='datasources'>
     65       <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'>
     66         <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property>
     67         <property type="boolean" id="enabled" mode="w" > True </property>
     68         <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property>
     69         <property type="int" id="severity" mode="w" > 3 </property>
     70         <property type="int" id="cycletime" mode="w" > 300 </property>
     71         <property type="boolean" id="usessh" mode="w" > False </property>
     72       <tomanycont id='datapoints'>
     73         <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'>
     74             <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property>
     75             <property type="boolean" id="isrow" mode="w" > True </property>
     76         </object>
     77       </tomanycont>
     78    </object>
     79
     80   <!--    snip  -->
     81
     82   </objects>
     83      """
     84      rootpath = ''
     85      skipobj = 0
     86
     87 -    def __init__(self, noopts=0, app=None, keeproot=False):
     88          """
     89          Initializer
     90
     91          @param noopts: don't use sys.argv for command-line options
     92          @type noopts: boolean
     93          @param app: app
     94          @type app: object
     95          @param keeproot: keeproot
     96          @type keeproot: boolean
     97          """
     98          ZCmdBase.__init__(self, noopts, app, keeproot)
     99          ContentHandler.__init__(self)
    100
    101 -    def context(self):
    102          """
    103          Return the bottom object in the stack
    104
    105          @return:
    106          @rtype: object
    107          """
    108          return self.objstack[-1]
    109
    110 -    def cleanattrs(self, attrs):
    111          """
    112          Convert all attributes to string values
    113
    114          @param attrs: (key,val) pairs
    115          @type attrs: list
    116          @return: same list, but with all values as strings
    117          @rtype: list
    118          """
    119          myattrs = {}
    120          for (key, val) in attrs.items():
    121              myattrs[key] = str(val)
    122          return myattrs
    123
    124 -    def startElement(self, name, attrs):
    125          """
    126          Function called when the parser finds the starting element
    127          in the XML file.
    128
    129          @param name: name of the element
    130          @type name: string
    131          @param attrs: list of (key, value) tuples
    132          @type attrs: list
    133          """
    134          ignoredElements = [ 'objects' ]
    135          attrs = self.cleanattrs(attrs)
    136          if self.skipobj > 0:
    137              self.skipobj += 1
    138              return
    139
    140          self.log.debug('tag %s, context %s, line %s'  % (
    141               name, self.context().id, self._locator.getLineNumber() ))
    142
    143          if name == 'object':
    144
    145              obj = self.createObject(attrs)
    146
    147              if obj is None:
    148                  formattedAttrs = ''
    149                  for key, value in attrs.items():
    150                      formattedAttrs += '  * %s: %s\n' % (key, value)
    151                  raise Exception('Unable to create object using the following '
    152                          'attributes:\n%s' % formattedAttrs)
    153
    154              if not self.options.noindex and hasattr(aq_base(obj),
    155                      'reIndex') and not self.rootpath:
    156                  self.rootpath = obj.getPrimaryId()
    157
    158              self.objstack.append(obj)
    159
    160          elif name == 'tomanycont' or name == 'tomany':
    161              nextobj = self.context()._getOb(attrs['id'], None)
    162              if nextobj is None:
    163                  self.skipobj = 1
    164                  return
    165              else:
    166                  self.objstack.append(nextobj)
    167          elif name == 'toone':
    168              relname = attrs.get('id')
    169              self.log.debug('toone %s, on object %s', relname,
    170                             self.context().id)
    171              rel = getattr(aq_base(self.context()), relname, None)
    172              if rel is None:
    173                  return
    174              objid = attrs.get('objid')
    175              self.addLink(rel, objid)
    176          elif name == 'link':
    177              self.addLink(self.context(), attrs['objid'])
    178          elif name == 'property':
    179              self.curattrs = attrs
    180          elif name in ignoredElements:
    181              pass
    182          else:
    183              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    184
    185 -    def endElement(self, name):
    186          """
    187          Function called when the parser finds the starting element
    188          in the XML file.
    189
    190          @param name: name of the ending element
    191          @type name: string
    192          """
    193          ignoredElements = [ 'toone', 'link' ]
    194          if self.skipobj > 0:
    195              self.skipobj -= 1
    196              return
    197
    198          noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0
    199
    200          if name in ('object', 'tomany', 'tomanycont'):
    201              obj = self.objstack.pop()
    202              notify(IndexingEvent(obj))
    203              if hasattr(aq_base(obj), 'index_object'):
    204                 obj.index_object()
    205              if self.rootpath == obj.getPrimaryId():
    206                  self.log.info('Calling reIndex %s', obj.getPrimaryId())
    207                  obj.reIndex()
    208                  self.rootpath = ''
    209              if (not noIncrementalCommit and
    210                  not self.objectnumber % self.options.chunk_size):
    211                  self.log.debug("Committing a batch of %s objects" %
    212                                 self.options.chunk_size)
    213                  self.commit()
    214
    215          elif name == 'objects': # ie end of the file
    216              self.log.info('End loading objects')
    217              self.log.info('Processing links')
    218              self.processLinks()
    219              if not self.options.noCommit:
    220                  self.commit()
    221                  self.log.info('Loaded %d objects into the ZODB database'
    222                             % self.objectnumber)
    223              else:
    224                  self.log.info('Would have created %d objects in the ZODB database'
    225                             % self.objectnumber)
    226
    227          elif name == 'property':
    228              self.setProperty(self.context(), self.curattrs,
    229                               self.charvalue)
    230              # We've closed off a tag, so now we need to re-initialize
    231              # the area that stores the contents of elements
    232              self.charvalue = ''
    233
    234          elif name in ignoredElements:
    235              pass
    236          else:
    237              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    238
    239 -    def characters(self, chars):
    240          """
    241          Called by xml.sax.parse() with data found in an element
    242          eg <object>my characters stuff</object>
    243
    244          Note that this can be called character by character.
    245
    246          @param chars: chars
    247          @type chars: string
    248          """
    249          self.charvalue += saxutils.unescape(chars)
    250
    251 -    def createObject(self, attrs):
    252          """
    253          Create an object and set it into its container
    254
    255          @param attrs: attrs
    256          @type attrs: string
    257          @return: newly created object
    258          @rtype: object
    259          """
    260          # Does the object exist already?
    261          id = attrs.get('id')
    262          obj = None
    263          try:
    264              if id.startswith('/'):
    265                  obj = getObjByPath(self.app, id)
    266              else:
    267                  obj = self.context()._getOb(id)
    268          except (KeyError, AttributeError, NotFound):
    269              pass
    270
    271          if obj is None:
    272              klass = importClass(attrs.get('module'), attrs.get('class'))
    273              if id.find('/') > -1:
    274                  (contextpath, id) = os.path.split(id)
    275                  try:
    276                      pathobj = getObjByPath(self.context(), contextpath)
    277                  except (KeyError, AttributeError, NotFound):
    278                      self.log.warn( "Unable to find context path %s (line %s ?) for %s" % (
    279                          contextpath, self._locator.getLineNumber(), id ))
    280                      if not self.options.noCommit:
    281                          self.log.warn( "Not committing any changes" )
    282                          self.options.noCommit = True
    283                      return None
    284                  self.objstack.append(pathobj)
    285              self.log.debug('Building instance %s of class %s',id,klass.__name__)
    286              obj = klass(id)
    287              self.context()._setObject(obj.id, obj)
    288              obj = self.context()._getOb(obj.id)
    289              self.objectnumber += 1
    290              if self.objectnumber % 5000 == 0:
    291                  transaction.savepoint()
    292              self.log.debug('Added object %s to database'
    293                              % obj.getPrimaryId())
    294          else:
    295              self.log.debug('Object %s already exists -- skipping' % id)
    296          return obj
    297
    298 -    def setZendoc(self, obj, value):
    299          zendocObj = zope.component.queryAdapter(obj, IZenDocProvider)
    300          if zendocObj is not None:
    301              zendocObj.setZendoc( value )
    302          elif value:
    303              self.log.warn('zendoc property could not be set to' +
    304                            ' %s on object %s' % ( value, obj.id ) )
    305
    306 -    def setProperty(self, obj, attrs, value):
    307          """
    308          Set the value of a property on an object.
    309
    310          @param obj: obj
    311          @type obj: string
    312          @param attrs: attrs
    313          @type attrs: string
    314          @param value: value
    315          @type value: string
    316          @return:
    317          @rtype:
    318          """
    319          name = attrs.get('id')
    320          proptype = attrs.get('type')
    321          setter = attrs.get('setter', None)
    322          self.log.debug('Setting object %s attr %s type %s value %s'
    323                          % (obj.id, name, proptype, value))
    324          linenum = self._locator.getLineNumber()
    325
    326          # Sanity check the value
    327          value = value.strip()
    328          try:
    329              value = str(value)
    330          except UnicodeEncodeError:
    331              self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \
    332               ' str(%s) for object %s attribute %s -- ignoring' % (
    333                             obj.id, name, proptype, value))
    334
    335          if name == 'zendoc':
    336              return self.setZendoc( obj, value )
    337
    338          # Guess at how to interpret the value given the property type
    339          if proptype == 'selection':
    340              try:
    341                  firstElement = getattr(obj, name)[0]
    342                  if type(firstElement) in types.StringTypes:
    343                      proptype = 'string'
    344              except (TypeError, IndexError):
    345                  self.log.warn("Error at line %s when trying to " % linenum + \
    346                      " use (%s) as the value for object %s attribute %s -- assuming a string"
    347                                 % (obj.id, name, proptype, value))
    348                  proptype = 'string'
    349
    350          if proptype == 'date':
    351              try:
    352                  value = float(value)
    353              except ValueError:
    354                  pass
    355              value = DateTime(value)
    356
    357          elif proptype not in _STRING_PROPERTY_TYPES:
    358              try:
    359                  value = eval(value)
    360              except NameError:
    361                  self.log.exception( 'Error trying to evaluate %s', value )
    362                  raise
    363              except SyntaxError:
    364                  self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % (
    365                       linenum, value) )
    366
    367          # Actually use the value
    368          if not obj.hasProperty(name):
    369              obj._setProperty(name, value, type=proptype, setter=setter)
    370          else:
    371              obj._updateProperty(name, value)
    372
    373 -    def addLink(self, rel, objid):
    374          """
    375          Build list of links to form after all objects have been created
    376          make sure that we don't add other side of a bidirectional relation
    377
    378          @param rel: relationship object
    379          @type rel: relation object
    380          @param objid: objid
    381          @type objid: string
    382          """
    383          self.links.append((rel.getPrimaryId(), objid))
    384
    385 -    def processLinks(self):
    386          """
    387          Walk through all the links that we saved and link them up
    388          """
    389          for (relid, objid) in self.links:
    390              try:
    391                  self.log.debug('Linking relation %s to object %s',
    392                                 relid, objid)
    393                  rel = getObjByPath(self.app, relid)
    394                  obj = getObjByPath(self.app, objid)
    395                  if not rel.hasobject(obj):
    396                      rel.addRelation(obj)
    397              except:
    398                  self.log.critical('Failed linking relation %s to object %s' % (
    399                                    relid, objid))
    400
    401 -    def buildOptions(self):
    402          """
    403          Command-line options specific to this command
    404          """
    405          ZCmdBase.buildOptions(self)
    406          self.parser.add_option('-i', '--infile', dest='infile',
    407                                 help='Input file for import. Default is stdin'
    408                                 )
    409          self.parser.add_option('--noindex', dest='noindex',
    410                                 action='store_true', default=False,
    411                                 help='Do not try to index the freshly loaded objects.'
    412                                 )
    413          self.parser.add_option('--chunksize', dest='chunk_size',
    414                                 help='Number of objects to commit at a time.',
    415                                 type='int',
    416                                 default=100
    417                                 )
    418          self.parser.add_option(
    419              '-n',
    420              '--noCommit',
    421              dest='noCommit',
    422              action='store_true',
    423              default=0,
    424              help='Do not store changes to the DMD (ie for debugging purposes)',
    425              )
    426
    427 -    def loadObjectFromXML(self, xmlfile=''):
    428          """
    429          This method can be used to load data for the root of Zenoss (default
    430          behavior) or it can be used to operate on a specific point in the
    431          Zenoss hierarchy (ZODB).
    432
    433          Upon loading the XML file to be processed, the content of the XML file
    434          is handled by the methods in this class when called by xml.sax.parse().
    435
    436          Reads from a file if xmlfile is specified, otherwise reads
    437          from the command-line option --infile.  If no files are found from
    438          any of these places, read from standard input.
    439
    440          @param xmlfile: Name of XML file to load, or file-like object
    441          @type xmlfile: string or file-like object
    442          """
    443          self.objstack = [self.app]
    444          self.links = []
    445          self.objectnumber = 0
    446          self.charvalue = ''
    447          if xmlfile and type(xmlfile) in types.StringTypes:
    448              self.infile = open(xmlfile)
    449          elif hasattr(xmlfile, 'read'):
    450              self.infile = xmlfile
    451          elif self.options.infile:
    452              self.infile = open(self.options.infile)
    453          else:
    454              self.infile = sys.stdin
    455          parser = make_parser()
    456          parser.setContentHandler(self)
    457          try:
    458              parser.parse(self.infile)
    459          except SAXParseException, ex:
    460              self.log.error("XML parse error at line %d column %d: %s",
    461                     ex.getLineNumber(), ex.getColumnNumber(),
    462                     ex.getMessage())
    463          finally:
    464              self.infile.close()
    465
    466 -    def loadDatabase(self):
    467          """
    468          The default behavior of loadObjectFromXML() will be to use the Zope
    469          app object, and thus operatate on the whole of Zenoss.
    470          """
    471          self.loadObjectFromXML()
    472
    473 -    def commit(self):
    474          """
    475          Wrapper around the Zope database commit()
    476          """
    477          trans = transaction.get()
    478          trans.note('Import from file %s using %s'
    479                      % (self.options.infile, self.__class__.__name__))
    480          trans.commit()
    481          if hasattr(self, 'connection'):
    482              # It's safe to call syncdb()
    483              self.syncdb()
    484
    485 -class SpoofedOptions(object):
    486      """
    487      SpoofedOptions
    488      """
    489
    490 -    def __init__(self):
    491          self.infile = ''
    492          self.noCommit = True
    493          self.noindex = True
    494
    495
    496 -class NoLoginImportRM(ImportRM):
    497      """
    498      An ImportRM that does not call the __init__ method on ZCmdBase
    499      """
    500
    501 -    def __init__(self, app):
    502          """
    503          Initializer
    504
    505          @param app: app
    506          @type app: string
    507          """
    508          self.app = app
    509          ContentHandler.__init__(self)
    510          import logging
    511          self.log = logging.getLogger('zen.ImportRM')
    512          self.options = SpoofedOptions()
    513
    514
    515  if __name__ == '__main__':
    516      im = ImportRM()
    517      im.loadDatabase()
    518