Package Products :: Package ZenRelations :: Module ImportRM |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRelations.ImportRM

      1  #! /usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  # ##########################################################################
      4  #
      5  # This program is part of Zenoss Core, an open source monitoring platform.
      6  # Copyright (C) 2009, Zenoss Inc.
      7  #
      8  # This program is free software; you can redistribute it and/or modify it
      9  # under the terms of the GNU General Public License version 2 as published by
     10  # the Free Software Foundation.
     11  #
     12  # For complete information please visit: http://www.zenoss.com/oss/
     13  #
     14  # ##########################################################################
     15
     16  __doc__ = """ImportRM
     17  Import RelationshipManager objects into a Zope database
     18  This provides support methods used by the Python xml.sax library to
     19  parse and construct an XML document.
     20
     21  Descriptions of the XML document format can be found in the
     22  Developers Guide.
     23  """
     24
     25  import sys
     26  import os
     27  import types
     28  import transaction
     29  from DateTime import DateTime
     30  from xml.sax import make_parser, saxutils
     31  from xml.sax.handler import ContentHandler
     32
     33  from Acquisition import aq_base
     34  from zExceptions import NotFound
     35  from OFS.PropertyManager import PropertyManager
     36
     37  from Products.ZenUtils.ZCmdBase import ZCmdBase
     38  from Products.ZenUtils.Utils import importClass
     39  from Products.ZenUtils.Utils import getObjByPath
     40
     41  from Products.ZenRelations.Exceptions import *
     42
     43  _STRING_PROPERTY_TYPES = ( 'string', 'text', 'password' )
     44
     45 -class ImportRM(ZCmdBase, ContentHandler):
     46      """
     47      Wrapper module to interface between Zope and the Python SAX XML library.
     48  The xml.sax.parse() calls different routines depending on what it finds.
     49
     50  A simple example of a valid XML file can be found in the objects.xml file
     51  for a ZenPack.
     52
     53   <?xml version="1.0"?>
     54   <objects>
     55     <!-- ('', 'zport', 'dmd', 'Devices', 'rrdTemplates', 'HelloWorld') -->
     56     <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'>
     57       <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property>
     58     <tomanycont id='datasources'>
     59       <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'>
     60         <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property>
     61         <property type="boolean" id="enabled" mode="w" > True </property>
     62         <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property>
     63         <property type="int" id="severity" mode="w" > 3 </property>
     64         <property type="int" id="cycletime" mode="w" > 300 </property>
     65         <property type="boolean" id="usessh" mode="w" > False </property>
     66       <tomanycont id='datapoints'>
     67         <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'>
     68             <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property>
     69             <property type="boolean" id="isrow" mode="w" > True </property>
     70         </object>
     71       </tomanycont>
     72    </object>
     73
     74   <!--    snip  -->
     75
     76   </objects>
     77      """
     78      rootpath = ''
     79      skipobj = 0
     80
     81 -    def __init__(self, noopts=0, app=None, keeproot=False):
     82          """
     83          Initializer
     84
     85          @param noopts: don't use sys.argv for command-line options
     86          @type noopts: boolean
     87          @param app: app
     88          @type app: object
     89          @param keeproot: keeproot
     90          @type keeproot: boolean
     91          """
     92          ZCmdBase.__init__(self, noopts, app, keeproot)
     93          ContentHandler.__init__(self)
     94
     95 -    def context(self):
     96          """
     97          Return the bottom object in the stack
     98
     99          @return:
    100          @rtype: object
    101          """
    102          return self.objstack[-1]
    103
    104 -    def cleanattrs(self, attrs):
    105          """
    106          Convert all attributes to string values
    107
    108          @param attrs: (key,val) pairs
    109          @type attrs: list
    110          @return: same list, but with all values as strings
    111          @rtype: list
    112          """
    113          myattrs = {}
    114          for (key, val) in attrs.items():
    115              myattrs[key] = str(val)
    116          return myattrs
    117
    118 -    def startElement(self, name, attrs):
    119          """
    120          Function called when the parser finds the starting element
    121          in the XML file.
    122
    123          @param name: name of the element
    124          @type name: string
    125          @param attrs: list of (key, value) tuples
    126          @type attrs: list
    127          """
    128          ignoredElements = [ 'objects' ]
    129          attrs = self.cleanattrs(attrs)
    130          if self.skipobj > 0:
    131              self.skipobj += 1
    132              return
    133
    134          self.log.debug('tag %s, context %s, line %s'  % (
    135               name, self.context().id, self._locator.getLineNumber() ))
    136
    137          if name == 'object':
    138
    139              obj = self.createObject(attrs)
    140
    141              if obj is None:
    142                  formattedAttrs = ''
    143                  for key, value in attrs.items():
    144                      formattedAttrs += '  * %s: %s\n' % (key, value)
    145                  raise Exception('Unable to create object using the following '
    146                          'attributes:\n%s' % formattedAttrs)
    147
    148              if not self.options.noindex and hasattr(aq_base(obj),
    149                      'reIndex') and not self.rootpath:
    150                  self.rootpath = obj.getPrimaryId()
    151
    152              self.objstack.append(obj)
    153
    154          elif name == 'tomanycont' or name == 'tomany':
    155              nextobj = self.context()._getOb(attrs['id'], None)
    156              if nextobj is None:
    157                  self.skipobj = 1
    158                  return
    159              else:
    160                  self.objstack.append(nextobj)
    161          elif name == 'toone':
    162              relname = attrs.get('id')
    163              self.log.debug('toone %s, on object %s', relname,
    164                             self.context().id)
    165              rel = getattr(aq_base(self.context()), relname, None)
    166              if rel is None:
    167                  return
    168              objid = attrs.get('objid')
    169              self.addLink(rel, objid)
    170          elif name == 'link':
    171              self.addLink(self.context(), attrs['objid'])
    172          elif name == 'property':
    173              self.curattrs = attrs
    174          elif name in ignoredElements:
    175              pass
    176          else:
    177              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    178
    179 -    def endElement(self, name):
    180          """
    181          Function called when the parser finds the starting element
    182          in the XML file.
    183
    184          @param name: name of the ending element
    185          @type name: string
    186          """
    187          ignoredElements = [ 'toone', 'link' ]
    188          if self.skipobj > 0:
    189              self.skipobj -= 1
    190              return
    191
    192          if name in ('object', 'tomany', 'tomanycont'):
    193              obj = self.objstack.pop()
    194              if hasattr(aq_base(obj), 'index_object'):
    195                  obj.index_object()
    196              if self.rootpath == obj.getPrimaryId():
    197                  self.log.info('Calling reIndex %s', obj.getPrimaryId())
    198                  obj.reIndex()
    199                  self.rootpath = ''
    200
    201          elif name == 'objects': # ie end of the file
    202              self.log.info('End loading objects')
    203              self.log.info('Processing links')
    204              self.processLinks()
    205              if not self.options.noCommit:
    206                  self.commit()
    207                  self.log.info('Loaded %d objects into the ZODB database'
    208                             % self.objectnumber)
    209              else:
    210                  self.log.info('Would have created %d objects in the ZODB database'
    211                             % self.objectnumber)
    212
    213          elif name == 'property':
    214              self.setProperty(self.context(), self.curattrs,
    215                               self.charvalue)
    216              # We've closed off a tag, so now we need to re-initialize
    217              # the area that stores the contents of elements
    218              self.charvalue = ''
    219
    220          elif name in ignoredElements:
    221              pass
    222          else:
    223              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    224
    225 -    def characters(self, chars):
    226          """
    227          Called by xml.sax.parse() with data found in an element
    228          eg <object>my characters stuff</object>
    229
    230          Note that this can be called character by character.
    231
    232          @param chars: chars
    233          @type chars: string
    234          """
    235          self.charvalue += saxutils.unescape(chars)
    236
    237 -    def createObject(self, attrs):
    238          """
    239          Create an object and set it into its container
    240
    241          @param attrs: attrs
    242          @type attrs: string
    243          @return: newly created object
    244          @rtype: object
    245          """
    246          # Does the object exist already?
    247          id = attrs.get('id')
    248          obj = None
    249          try:
    250              if id.startswith('/'):
    251                  obj = getObjByPath(self.app, id)
    252              else:
    253                  obj = self.context()._getOb(id)
    254          except (KeyError, AttributeError, NotFound):
    255              pass
    256
    257          if obj is None:
    258              klass = importClass(attrs.get('module'), attrs.get('class'))
    259              if id.find('/') > -1:
    260                  (contextpath, id) = os.path.split(id)
    261                  try:
    262                      pathobj = getObjByPath(self.context(), contextpath)
    263                  except (KeyError, AttributeError, NotFound):
    264                      self.log.warn( "Unable to find context path %s (line %s ?) for %s" % (
    265                          contextpath, self._locator.getLineNumber(), id ))
    266                      if not self.options.noCommit:
    267                          self.log.warn( "Not committing any changes" )
    268                          self.options.noCommit = True
    269                      return None
    270                  self.objstack.append(pathobj)
    271              obj = klass(id)
    272              self.context()._setObject(obj.id, obj)
    273              obj = self.context()._getOb(obj.id)
    274              self.objectnumber += 1
    275              if self.objectnumber % 5000 == 0:
    276                  transaction.savepoint()
    277              self.log.debug('Added object %s to database'
    278                              % obj.getPrimaryId())
    279          else:
    280              self.log.debug('Object %s already exists -- skipping' % id)
    281          return obj
    282
    283 -    def setProperty(self, obj, attrs, value):
    284          """
    285          Set the value of a property on an object.
    286
    287          @param obj: obj
    288          @type obj: string
    289          @param attrs: attrs
    290          @type attrs: string
    291          @param value: value
    292          @type value: string
    293          @return:
    294          @rtype:
    295          """
    296          name = attrs.get('id')
    297          proptype = attrs.get('type')
    298          setter = attrs.get('setter', None)
    299          self.log.debug('Setting object %s attr %s type %s value %s'
    300                          % (obj.id, name, proptype, value))
    301          linenum = self._locator.getLineNumber()
    302
    303          # Sanity check the value
    304          value = value.strip()
    305          try:
    306              value = str(value)
    307          except UnicodeEncodeError:
    308              self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \
    309               ' str(%s) for object %s attribute %s -- ignoring' % (
    310                             obj.id, name, proptype, value))
    311
    312          # Guess at how to interpret the value given the property type
    313          if proptype == 'selection':
    314              try:
    315                  firstElement = getattr(obj, name)[0]
    316                  if type(firstElement) in types.StringTypes:
    317                      proptype = 'string'
    318              except (TypeError, IndexError):
    319                  self.log.warn("Error at line %s when trying to " % linenum + \
    320                      " use (%s) as the value for object %s attribute %s -- assuming a string"
    321                                 % (obj.id, name, proptype, value))
    322                  proptype = 'string'
    323
    324          if proptype == 'date':
    325              try:
    326                  value = float(value)
    327              except ValueError:
    328                  pass
    329              value = DateTime(value)
    330
    331          elif proptype not in _STRING_PROPERTY_TYPES:
    332              try:
    333                  value = eval(value)
    334              except NameError:
    335                  self.log.exception( 'Error trying to evaluate %s', value )
    336                  raise
    337              except SyntaxError:
    338                  self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % (
    339                       linenum, value) )
    340
    341          # Actually use the value
    342          if not obj.hasProperty(name):
    343              obj._setProperty(name, value, type=proptype, setter=setter)
    344          else:
    345              obj._updateProperty(name, value)
    346
    347 -    def addLink(self, rel, objid):
    348          """
    349          Build list of links to form after all objects have been created
    350          make sure that we don't add other side of a bidirectional relation
    351
    352          @param rel: relationship object
    353          @type rel: relation object
    354          @param objid: objid
    355          @type objid: string
    356          """
    357          self.links.append((rel.getPrimaryId(), objid))
    358
    359 -    def processLinks(self):
    360          """
    361          Walk through all the links that we saved and link them up
    362          """
    363          for (relid, objid) in self.links:
    364              try:
    365                  self.log.debug('Linking relation %s to object %s',
    366                                 relid, objid)
    367                  rel = getObjByPath(self.app, relid)
    368                  obj = getObjByPath(self.app, objid)
    369                  if not rel.hasobject(obj):
    370                      rel.addRelation(obj)
    371              except:
    372                  self.log.critical('Failed linking relation %s to object %s' % (
    373                                    relid, objid))
    374
    375 -    def buildOptions(self):
    376          """
    377          Command-line options specific to this command
    378          """
    379          ZCmdBase.buildOptions(self)
    380          self.parser.add_option('-i', '--infile', dest='infile',
    381                                 help='Input file for import. Default is stdin'
    382                                 )
    383          self.parser.add_option('--noindex', dest='noindex',
    384                                 action='store_true', default=False,
    385                                 help='Do not try to index the freshly loaded objects.'
    386                                 )
    387          self.parser.add_option(
    388              '-n',
    389              '--noCommit',
    390              dest='noCommit',
    391              action='store_true',
    392              default=0,
    393              help='Do not store changes to the DMD (ie for debugging purposes)',
    394              )
    395
    396 -    def loadObjectFromXML(self, xmlfile=''):
    397          """
    398          This method can be used to load data for the root of Zenoss (default
    399          behavior) or it can be used to operate on a specific point in the
    400          Zenoss hierarchy (ZODB).
    401
    402          Upon loading the XML file to be processed, the content of the XML file
    403          is handled by the methods in this class when called by xml.sax.parse().
    404
    405          Reads from a file if xmlfile is specified, otherwise reads
    406          from the command-line option --infile.  If no files are found from
    407          any of these places, read from standard input.
    408
    409          @param xmlfile: Name of XML file to load, or file-like object
    410          @type xmlfile: string or file-like object
    411          """
    412          self.objstack = [self.app]
    413          self.links = []
    414          self.objectnumber = 0
    415          self.charvalue = ''
    416          if xmlfile and type(xmlfile) in types.StringTypes:
    417              self.infile = open(xmlfile)
    418          elif hasattr(xmlfile, 'read'):
    419              self.infile = xmlfile
    420          elif self.options.infile:
    421              self.infile = open(self.options.infile)
    422          else:
    423              self.infile = sys.stdin
    424          parser = make_parser()
    425          parser.setContentHandler(self)
    426          try:
    427              parser.parse(self.infile)
    428          finally:
    429              self.infile.close()
    430
    431 -    def loadDatabase(self):
    432          """
    433          The default behavior of loadObjectFromXML() will be to use the Zope
    434          app object, and thus operatate on the whole of Zenoss.
    435          """
    436          self.loadObjectFromXML()
    437
    438 -    def commit(self):
    439          """
    440          Wrapper around the Zope database commit()
    441          """
    442          trans = transaction.get()
    443          trans.note('Import from file %s using %s'
    444                      % (self.options.infile, self.__class__.__name__))
    445          trans.commit()
    446
    447
    448 -class SpoofedOptions(object):
    449      """
    450      SpoofedOptions
    451      """
    452
    453 -    def __init__(self):
    454          self.infile = ''
    455          self.noCommit = True
    456          self.noindex = True
    457
    458
    459 -class NoLoginImportRM(ImportRM):
    460      """
    461      An ImportRM that does not call the __init__ method on ZCmdBase
    462      """
    463
    464 -    def __init__(self, app):
    465          """
    466          Initializer
    467
    468          @param app: app
    469          @type app: string
    470          """
    471          self.app = app
    472          ContentHandler.__init__(self)
    473          import logging
    474          self.log = logging.getLogger('zen.ImportRM')
    475          self.options = SpoofedOptions()
    476
    477
    478  if __name__ == '__main__':
    479      im = ImportRM()
    480      im.loadDatabase()
    481