Package Products :: Package ZenRelations :: Module ImportRM |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRelations.ImportRM

      1  #! /usr/bin/env python
      2  ##############################################################################
      3  #
      4  # Copyright (C) Zenoss, Inc. 2009, all rights reserved.
      5  #
      6  # This content is made available according to terms specified in
      7  # License.zenoss under the directory where your Zenoss product is installed.
      8  #
      9  ##############################################################################
     10
     11
     12  __doc__ = """ImportRM
     13  Import RelationshipManager objects into a Zope database
     14  This provides support methods used by the Python xml.sax library to
     15  parse and construct an XML document.
     16
     17  Descriptions of the XML document format can be found in the
     18  Developers Guide.
     19  """
     20  import Globals
     21  import sys
     22  import os
     23  import transaction
     24  import zope.component
     25  from zope.event import notify
     26  from DateTime import DateTime
     27  from xml.sax import make_parser, saxutils, SAXParseException
     28  from xml.sax.handler import ContentHandler
     29
     30  from Acquisition import aq_base
     31  from zExceptions import NotFound
     32  from OFS.PropertyManager import PropertyManager
     33
     34  from Products.ZenUtils.ZCmdBase import ZCmdBase
     35  from Products.ZenUtils.Utils import importClass
     36  from Products.ZenUtils.Utils import getObjByPath
     37
     38  from Products.ZenModel.interfaces import IZenDocProvider
     39  from Products.ZenRelations.Exceptions import *
     40  from Products.Zuul.catalog.events import IndexingEvent
     41
     42  _STRING_PROPERTY_TYPES = ( 'string', 'text', 'password' )
     43
     44
     45
     46 -class ImportRM(ZCmdBase, ContentHandler):
     47      """
     48      Wrapper module to interface between Zope and the Python SAX XML library.
     49  The xml.sax.parse() calls different routines depending on what it finds.
     50
     51  A simple example of a valid XML file can be found in the objects.xml file
     52  for a ZenPack.
     53
     54   <?xml version="1.0"?>
     55   <objects>
     56     <!-- ('', 'zport', 'dmd', 'Devices', 'rrdTemplates', 'HelloWorld') -->
     57     <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'>
     58       <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property>
     59     <tomanycont id='datasources'>
     60       <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'>
     61         <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property>
     62         <property type="boolean" id="enabled" mode="w" > True </property>
     63         <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property>
     64         <property type="int" id="severity" mode="w" > 3 </property>
     65         <property type="int" id="cycletime" mode="w" > 300 </property>
     66         <property type="boolean" id="usessh" mode="w" > False </property>
     67       <tomanycont id='datapoints'>
     68         <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'>
     69             <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property>
     70             <property type="boolean" id="isrow" mode="w" > True </property>
     71         </object>
     72       </tomanycont>
     73    </object>
     74
     75   <!--    snip  -->
     76
     77   </objects>
     78      """
     79      rootpath = ''
     80      skipobj = 0
     81
     82 -    def __init__(self, noopts=0, app=None, keeproot=False):
     83          """
     84          Initializer
     85
     86          @param noopts: don't use sys.argv for command-line options
     87          @type noopts: boolean
     88          @param app: app
     89          @type app: object
     90          @param keeproot: keeproot
     91          @type keeproot: boolean
     92          """
     93          ZCmdBase.__init__(self, noopts, app, keeproot)
     94          ContentHandler.__init__(self)
     95
     96 -    def context(self):
     97          """
     98          Return the bottom object in the stack
     99
    100          @return:
    101          @rtype: object
    102          """
    103          return self.objstack[-1]
    104
    105 -    def cleanattrs(self, attrs):
    106          """
    107          Convert all attributes to string values
    108
    109          @param attrs: (key,val) pairs
    110          @type attrs: list
    111          @return: same list, but with all values as strings
    112          @rtype: list
    113          """
    114          myattrs = {}
    115          for (key, val) in attrs.items():
    116              myattrs[key] = str(val)
    117          return myattrs
    118
    119 -    def startElement(self, name, attrs):
    120          """
    121          Function called when the parser finds the starting element
    122          in the XML file.
    123
    124          @param name: name of the element
    125          @type name: string
    126          @param attrs: list of (key, value) tuples
    127          @type attrs: list
    128          """
    129          ignoredElements = [ 'objects' ]
    130          attrs = self.cleanattrs(attrs)
    131          if self.skipobj > 0:
    132              self.skipobj += 1
    133              return
    134
    135          self.log.debug('tag %s, context %s, line %s'  % (
    136               name, self.context().id, self._locator.getLineNumber() ))
    137
    138          if name == 'object':
    139              if attrs.get('class') == 'Device':
    140                  devId = attrs['id'].split('/')[-1]
    141                  dev = self.dmd.Devices.findDeviceByIdOrIp(devId)
    142                  if dev:
    143                      msg = 'The device %s already exists on this system! (Line %s)' % \
    144                                      (devId, self._locator.getLineNumber())
    145                      raise Exception(msg)
    146
    147              if attrs.get('class') == 'IpAddress':
    148                  ipAddress = attrs['id']
    149                  dev = self.dmd.Devices.findDeviceByIdOrIp(ipAddress)
    150                  if dev:
    151                      msg = 'The IP address %s already exists on this system! (Line %s)' % \
    152                                  (ipAddress, self._locator.getLineNumber())
    153                      raise Exception(msg)
    154
    155              obj = self.createObject(attrs)
    156              if obj is None:
    157                  formattedAttrs = ''
    158                  for key, value in attrs.items():
    159                      formattedAttrs += '  * %s: %s\n' % (key, value)
    160                  raise Exception('Unable to create object using the following '
    161                          'attributes:\n%s' % formattedAttrs)
    162
    163              if not self.options.noindex and hasattr(aq_base(obj),
    164                      'reIndex') and not self.rootpath:
    165                  self.rootpath = obj.getPrimaryId()
    166
    167              self.objstack.append(obj)
    168
    169          elif name == 'tomanycont' or name == 'tomany':
    170              nextobj = self.context()._getOb(attrs['id'], None)
    171              if nextobj is None:
    172                  self.skipobj = 1
    173                  return
    174              else:
    175                  self.objstack.append(nextobj)
    176          elif name == 'toone':
    177              relname = attrs.get('id')
    178              self.log.debug('toone %s, on object %s', relname,
    179                             self.context().id)
    180              rel = getattr(aq_base(self.context()), relname, None)
    181              if rel is None:
    182                  return
    183              objid = attrs.get('objid')
    184              self.addLink(rel, objid)
    185          elif name == 'link':
    186              self.addLink(self.context(), attrs['objid'])
    187          elif name == 'property':
    188              self.curattrs = attrs
    189          elif name in ignoredElements:
    190              pass
    191          else:
    192              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    193
    194 -    def endElement(self, name):
    195          """
    196          Function called when the parser finds the starting element
    197          in the XML file.
    198
    199          @param name: name of the ending element
    200          @type name: string
    201          """
    202          ignoredElements = [ 'toone', 'link' ]
    203          if self.skipobj > 0:
    204              self.skipobj -= 1
    205              return
    206
    207          noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0
    208
    209          if name in ('object', 'tomany', 'tomanycont'):
    210              obj = self.objstack.pop()
    211              notify(IndexingEvent(obj))
    212              if hasattr(aq_base(obj), 'index_object'):
    213                 obj.index_object()
    214              if self.rootpath == obj.getPrimaryId():
    215                  self.log.info('Calling reIndex %s', obj.getPrimaryId())
    216                  obj.reIndex()
    217                  self.rootpath = ''
    218              if (not noIncrementalCommit and
    219                  not self.objectnumber % self.options.chunk_size):
    220                  self.log.debug("Committing a batch of %s objects" %
    221                                 self.options.chunk_size)
    222                  self.commit()
    223
    224          elif name == 'objects': # ie end of the file
    225              self.log.info('End loading objects')
    226              self.log.info('Processing links')
    227              self.processLinks()
    228              if not self.options.noCommit:
    229                  self.commit()
    230                  self.log.info('Loaded %d objects into the ZODB database'
    231                             % self.objectnumber)
    232              else:
    233                  self.log.info('Would have created %d objects in the ZODB database'
    234                             % self.objectnumber)
    235
    236          elif name == 'property':
    237              self.setProperty(self.context(), self.curattrs,
    238                               self.charvalue)
    239              # We've closed off a tag, so now we need to re-initialize
    240              # the area that stores the contents of elements
    241              self.charvalue = ''
    242
    243          elif name in ignoredElements:
    244              pass
    245          else:
    246              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    247
    248 -    def characters(self, chars):
    249          """
    250          Called by xml.sax.parse() with data found in an element
    251          eg <object>my characters stuff</object>
    252
    253          Note that this can be called character by character.
    254
    255          @param chars: chars
    256          @type chars: string
    257          """
    258          self.charvalue += saxutils.unescape(chars)
    259
    260 -    def createObject(self, attrs):
    261          """
    262          Create an object and set it into its container
    263
    264          @param attrs: attrs
    265          @type attrs: string
    266          @return: newly created object
    267          @rtype: object
    268          """
    269          # Does the object exist already?
    270          id = attrs.get('id')
    271          obj = None
    272          try:
    273              if id.startswith('/'):
    274                  obj = getObjByPath(self.app, id)
    275              else:
    276                  obj = self.context()._getOb(id)
    277          except (KeyError, AttributeError, NotFound):
    278              pass
    279
    280          if obj is None:
    281              klass = importClass(attrs.get('module'), attrs.get('class'))
    282              if id.find('/') > -1:
    283                  (contextpath, id) = os.path.split(id)
    284                  try:
    285                      pathobj = getObjByPath(self.context(), contextpath)
    286                  except (KeyError, AttributeError, NotFound):
    287                      self.log.warn( "Unable to find context path %s (line %s ?) for %s" % (
    288                          contextpath, self._locator.getLineNumber(), id ))
    289                      if not self.options.noCommit:
    290                          self.log.warn( "Not committing any changes" )
    291                          self.options.noCommit = True
    292                      return None
    293                  self.objstack.append(pathobj)
    294              self.log.debug('Building instance %s of class %s',id,klass.__name__)
    295              try:
    296                  if klass.__name__ == 'AdministrativeRole':
    297                      user = [x for x in self.dmd.ZenUsers.objectValues() if x.id == id]
    298                      if user:
    299                          obj = klass(user[0], self.context().device())
    300                      else:
    301                          msg = "No AdminRole user %s exists (line %s)" % (
    302                                         id, self._locator.getLineNumber())
    303                          self.log.error(msg)
    304                          raise Exception(msg)
    305                  else:
    306                      obj = klass(id)
    307              except TypeError, ex:
    308                  # This happens when the constructor expects more arguments
    309                  self.log.exception("Unable to build %s instance of class %s (line %s)",
    310                                     id, klass.__name__, self._locator.getLineNumber())
    311                  raise
    312              self.context()._setObject(obj.id, obj)
    313              obj = self.context()._getOb(obj.id)
    314              self.objectnumber += 1
    315              self.log.debug('Added object %s to database'
    316                              % obj.getPrimaryId())
    317          else:
    318              self.log.debug('Object %s already exists -- skipping' % id)
    319          return obj
    320
    321 -    def setZendoc(self, obj, value):
    322          zendocObj = zope.component.queryAdapter(obj, IZenDocProvider)
    323          if zendocObj is not None:
    324              zendocObj.setZendoc( value )
    325          elif value:
    326              self.log.warn('zendoc property could not be set to' +
    327                            ' %s on object %s' % ( value, obj.id ) )
    328
    329 -    def setProperty(self, obj, attrs, value):
    330          """
    331          Set the value of a property on an object.
    332
    333          @param obj: obj
    334          @type obj: string
    335          @param attrs: attrs
    336          @type attrs: string
    337          @param value: value
    338          @type value: string
    339          @return:
    340          @rtype:
    341          """
    342          name = attrs.get('id')
    343          proptype = attrs.get('type')
    344          setter = attrs.get('setter', None)
    345          self.log.debug('Setting object %s attr %s type %s value %s'
    346                          % (obj.id, name, proptype, value))
    347          linenum = self._locator.getLineNumber()
    348
    349          # Sanity check the value
    350          value = value.strip()
    351          try:
    352              value = str(value)
    353          except UnicodeEncodeError:
    354              self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \
    355               ' str(%s) for object %s attribute %s -- ignoring' % (
    356                             obj.id, name, proptype, value))
    357
    358          if name == 'zendoc':
    359              return self.setZendoc( obj, value )
    360
    361          # Guess at how to interpret the value given the property type
    362          if proptype == 'selection':
    363              try:
    364                  firstElement = getattr(obj, name)[0]
    365                  if isinstance(firstElement, basestring):
    366                      proptype = 'string'
    367              except (TypeError, IndexError):
    368                  self.log.warn("Error at line %s when trying to " % linenum + \
    369                      " use (%s) as the value for object %s attribute %s -- assuming a string"
    370                                 % (obj.id, name, proptype, value))
    371                  proptype = 'string'
    372
    373          if proptype == 'date':
    374              try:
    375                  value = float(value)
    376              except ValueError:
    377                  pass
    378              value = DateTime(value)
    379
    380          elif proptype not in _STRING_PROPERTY_TYPES:
    381              try:
    382                  value = eval(value)
    383              except NameError:
    384                  self.log.warn('Error trying to evaluate %s at line %s',
    385                                    value, linenum)
    386              except SyntaxError:
    387                  self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % (
    388                       linenum, value) )
    389
    390          # Actually use the value
    391          if not obj.hasProperty(name):
    392              obj._setProperty(name, value, type=proptype, setter=setter)
    393          else:
    394              obj._updateProperty(name, value)
    395
    396 -    def addLink(self, rel, objid):
    397          """
    398          Build list of links to form after all objects have been created
    399          make sure that we don't add other side of a bidirectional relation
    400
    401          @param rel: relationship object
    402          @type rel: relation object
    403          @param objid: objid
    404          @type objid: string
    405          """
    406          self.links.append((rel.getPrimaryId(), objid))
    407
    408 -    def processLinks(self):
    409          """
    410          Walk through all the links that we saved and link them up
    411          """
    412          for (relid, objid) in self.links:
    413              try:
    414                  self.log.debug('Linking relation %s to object %s',
    415                                 relid, objid)
    416                  rel = getObjByPath(self.app, relid)
    417                  obj = getObjByPath(self.app, objid)
    418                  if not rel.hasobject(obj):
    419                      rel.addRelation(obj)
    420              except:
    421                  self.log.critical('Failed linking relation %s to object %s' % (
    422                                    relid, objid))
    423
    424 -    def buildOptions(self):
    425          """
    426          Command-line options specific to this command
    427          """
    428          ZCmdBase.buildOptions(self)
    429          self.parser.add_option('-i', '--infile', dest='infile',
    430                                 help='Input file for import. Default is stdin'
    431                                 )
    432          self.parser.add_option('--noindex', dest='noindex',
    433                                 action='store_true', default=False,
    434                                 help='Do not try to index the freshly loaded objects.'
    435                                 )
    436          self.parser.add_option('--chunksize', dest='chunk_size',
    437                                 help='Number of objects to commit at a time.',
    438                                 type='int',
    439                                 default=100
    440                                 )
    441          self.parser.add_option(
    442              '-n',
    443              '--noCommit',
    444              dest='noCommit',
    445              action='store_true',
    446              default=0,
    447              help='Do not store changes to the DMD (ie for debugging purposes)',
    448              )
    449
    450 -    def loadObjectFromXML(self, xmlfile=''):
    451          """
    452          This method can be used to load data for the root of Zenoss (default
    453          behavior) or it can be used to operate on a specific point in the
    454          Zenoss hierarchy (ZODB).
    455
    456          Upon loading the XML file to be processed, the content of the XML file
    457          is handled by the methods in this class when called by xml.sax.parse().
    458
    459          Reads from a file if xmlfile is specified, otherwise reads
    460          from the command-line option --infile.  If no files are found from
    461          any of these places, read from standard input.
    462
    463          @param xmlfile: Name of XML file to load, or file-like object
    464          @type xmlfile: string or file-like object
    465          """
    466          self.objstack = [self.app]
    467          self.links = []
    468          self.objectnumber = 0
    469          self.charvalue = ''
    470          if xmlfile and isinstance(xmlfile, basestring):
    471              self.infile = open(xmlfile)
    472          elif hasattr(xmlfile, 'read'):
    473              self.infile = xmlfile
    474          elif self.options.infile:
    475              self.infile = open(self.options.infile)
    476          else:
    477              self.infile = sys.stdin
    478          parser = make_parser()
    479          parser.setContentHandler(self)
    480          try:
    481              parser.parse(self.infile)
    482          except SAXParseException, ex:
    483              self.log.error("XML parse error at line %d column %d: %s",
    484                     ex.getLineNumber(), ex.getColumnNumber(),
    485                     ex.getMessage())
    486          finally:
    487              self.infile.close()
    488
    489 -    def loadDatabase(self):
    490          """
    491          The default behavior of loadObjectFromXML() will be to use the Zope
    492          app object, and thus operatate on the whole of Zenoss.
    493          """
    494          self.loadObjectFromXML()
    495
    496 -    def commit(self):
    497          """
    498          Wrapper around the Zope database commit()
    499          """
    500          trans = transaction.get()
    501          trans.note('Import from file %s using %s'
    502                      % (self.options.infile, self.__class__.__name__))
    503          trans.commit()
    504          if hasattr(self, 'connection'):
    505              # It's safe to call syncdb()
    506              self.syncdb()
    507
    508 -class SpoofedOptions(object):
    509      """
    510      SpoofedOptions
    511      """
    512
    513 -    def __init__(self):
    514          self.infile = ''
    515          self.noCommit = True
    516          self.noindex = True
    517          self.dataroot = '/zport/dmd'
    518
    519
    520 -class NoLoginImportRM(ImportRM):
    521      """
    522      An ImportRM that does not call the __init__ method on ZCmdBase
    523      """
    524
    525 -    def __init__(self, app):
    526          """
    527          Initializer
    528
    529          @param app: app
    530          @type app: string
    531          """
    532          import Products.ZenossStartup
    533          from Products.Five import zcml
    534          zcml.load_site()
    535          import logging
    536          self.log = logging.getLogger('zen.ImportRM')
    537          self.app = app
    538          ContentHandler.__init__(self)
    539          self.options = SpoofedOptions()
    540          self.dataroot = None
    541          self.getDataRoot()
    542
    543
    544  if __name__ == '__main__':
    545      im = ImportRM()
    546      im.loadDatabase()
    547