Package Products :: Package ZenRelations :: Module ImportRM |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenRelations.ImportRM

      1  #! /usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  # ##########################################################################
      4  #
      5  # This program is part of Zenoss Core, an open source monitoring platform.
      6  # Copyright (C) 2009, Zenoss Inc.
      7  #
      8  # This program is free software; you can redistribute it and/or modify it
      9  # under the terms of the GNU General Public License version 2 or (at your
     10  # option) any later version as published by the Free Software Foundation.
     11  #
     12  # For complete information please visit: http://www.zenoss.com/oss/
     13  #
     14  # ##########################################################################
     15
     16  __doc__ = """ImportRM
     17  Import RelationshipManager objects into a Zope database
     18  This provides support methods used by the Python xml.sax library to
     19  parse and construct an XML document.
     20
     21  Descriptions of the XML document format can be found in the
     22  Developers Guide.
     23  """
     24  import Globals
     25  import sys
     26  import os
     27  import transaction
     28  import zope.component
     29  from zope.event import notify
     30  from DateTime import DateTime
     31  from xml.sax import make_parser, saxutils, SAXParseException
     32  from xml.sax.handler import ContentHandler
     33
     34  from Acquisition import aq_base
     35  from zExceptions import NotFound
     36  from OFS.PropertyManager import PropertyManager
     37
     38  from Products.ZenUtils.ZCmdBase import ZCmdBase
     39  from Products.ZenUtils.Utils import importClass
     40  from Products.ZenUtils.Utils import getObjByPath
     41
     42  from Products.ZenModel.interfaces import IZenDocProvider
     43  from Products.ZenRelations.Exceptions import *
     44  from Products.Zuul.catalog.events import IndexingEvent
     45
     46  _STRING_PROPERTY_TYPES = ( 'string', 'text', 'password' )
     47
     48
     49
     50 -class ImportRM(ZCmdBase, ContentHandler):
     51      """
     52      Wrapper module to interface between Zope and the Python SAX XML library.
     53  The xml.sax.parse() calls different routines depending on what it finds.
     54
     55  A simple example of a valid XML file can be found in the objects.xml file
     56  for a ZenPack.
     57
     58   <?xml version="1.0"?>
     59   <objects>
     60     <!-- ('', 'zport', 'dmd', 'Devices', 'rrdTemplates', 'HelloWorld') -->
     61     <object id='/zport/dmd/Devices/rrdTemplates/HelloWorld' module='Products.ZenModel.RRDTemplate' class='RRDTemplate'>
     62       <property type="text" id="description" mode="w" > This is the glorious description that shows up when we click on our RRD template </property>
     63     <tomanycont id='datasources'>
     64       <object id='hello' module='Products.ZenModel.BasicDataSource' class='BasicDataSource'>
     65         <property select_variable="sourcetypes" type="selection" id="sourcetype" mode="w" > SNMP </property>
     66         <property type="boolean" id="enabled" mode="w" > True </property>
     67         <property type="string" id="eventClass" mode="w" > /Cmd/Fail </property>
     68         <property type="int" id="severity" mode="w" > 3 </property>
     69         <property type="int" id="cycletime" mode="w" > 300 </property>
     70         <property type="boolean" id="usessh" mode="w" > False </property>
     71       <tomanycont id='datapoints'>
     72         <object id='hello' module='Products.ZenModel.RRDDataPoint' class='RRDDataPoint'>
     73             <property select_variable="rrdtypes" type="selection" id="rrdtype" mode="w" > GAUGE </property>
     74             <property type="boolean" id="isrow" mode="w" > True </property>
     75         </object>
     76       </tomanycont>
     77    </object>
     78
     79   <!--    snip  -->
     80
     81   </objects>
     82      """
     83      rootpath = ''
     84      skipobj = 0
     85
     86 -    def __init__(self, noopts=0, app=None, keeproot=False):
     87          """
     88          Initializer
     89
     90          @param noopts: don't use sys.argv for command-line options
     91          @type noopts: boolean
     92          @param app: app
     93          @type app: object
     94          @param keeproot: keeproot
     95          @type keeproot: boolean
     96          """
     97          ZCmdBase.__init__(self, noopts, app, keeproot)
     98          ContentHandler.__init__(self)
     99
    100 -    def context(self):
    101          """
    102          Return the bottom object in the stack
    103
    104          @return:
    105          @rtype: object
    106          """
    107          return self.objstack[-1]
    108
    109 -    def cleanattrs(self, attrs):
    110          """
    111          Convert all attributes to string values
    112
    113          @param attrs: (key,val) pairs
    114          @type attrs: list
    115          @return: same list, but with all values as strings
    116          @rtype: list
    117          """
    118          myattrs = {}
    119          for (key, val) in attrs.items():
    120              myattrs[key] = str(val)
    121          return myattrs
    122
    123 -    def startElement(self, name, attrs):
    124          """
    125          Function called when the parser finds the starting element
    126          in the XML file.
    127
    128          @param name: name of the element
    129          @type name: string
    130          @param attrs: list of (key, value) tuples
    131          @type attrs: list
    132          """
    133          ignoredElements = [ 'objects' ]
    134          attrs = self.cleanattrs(attrs)
    135          if self.skipobj > 0:
    136              self.skipobj += 1
    137              return
    138
    139          self.log.debug('tag %s, context %s, line %s'  % (
    140               name, self.context().id, self._locator.getLineNumber() ))
    141
    142          if name == 'object':
    143              if attrs.get('class') == 'Device':
    144                  devId = attrs['id'].split('/')[-1]
    145                  dev = self.dmd.Devices.findDeviceByIdOrIp(devId)
    146                  if dev:
    147                      msg = 'The device %s already exists on this system! (Line %s)' % \
    148                                      (devId, self._locator.getLineNumber())
    149                      raise Exception(msg)
    150
    151              if attrs.get('class') == 'IpAddress':
    152                  ipAddress = attrs['id']
    153                  dev = self.dmd.Devices.findDeviceByIdOrIp(ipAddress)
    154                  if dev:
    155                      msg = 'The IP address %s already exists on this system! (Line %s)' % \
    156                                  (ipAddress, self._locator.getLineNumber())
    157                      raise Exception(msg)
    158
    159              obj = self.createObject(attrs)
    160              if obj is None:
    161                  formattedAttrs = ''
    162                  for key, value in attrs.items():
    163                      formattedAttrs += '  * %s: %s\n' % (key, value)
    164                  raise Exception('Unable to create object using the following '
    165                          'attributes:\n%s' % formattedAttrs)
    166
    167              if not self.options.noindex and hasattr(aq_base(obj),
    168                      'reIndex') and not self.rootpath:
    169                  self.rootpath = obj.getPrimaryId()
    170
    171              self.objstack.append(obj)
    172
    173          elif name == 'tomanycont' or name == 'tomany':
    174              nextobj = self.context()._getOb(attrs['id'], None)
    175              if nextobj is None:
    176                  self.skipobj = 1
    177                  return
    178              else:
    179                  self.objstack.append(nextobj)
    180          elif name == 'toone':
    181              relname = attrs.get('id')
    182              self.log.debug('toone %s, on object %s', relname,
    183                             self.context().id)
    184              rel = getattr(aq_base(self.context()), relname, None)
    185              if rel is None:
    186                  return
    187              objid = attrs.get('objid')
    188              self.addLink(rel, objid)
    189          elif name == 'link':
    190              self.addLink(self.context(), attrs['objid'])
    191          elif name == 'property':
    192              self.curattrs = attrs
    193          elif name in ignoredElements:
    194              pass
    195          else:
    196              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    197
    198 -    def endElement(self, name):
    199          """
    200          Function called when the parser finds the starting element
    201          in the XML file.
    202
    203          @param name: name of the ending element
    204          @type name: string
    205          """
    206          ignoredElements = [ 'toone', 'link' ]
    207          if self.skipobj > 0:
    208              self.skipobj -= 1
    209              return
    210
    211          noIncrementalCommit = self.options.noCommit or self.options.chunk_size==0
    212
    213          if name in ('object', 'tomany', 'tomanycont'):
    214              obj = self.objstack.pop()
    215              notify(IndexingEvent(obj))
    216              if hasattr(aq_base(obj), 'index_object'):
    217                 obj.index_object()
    218              if self.rootpath == obj.getPrimaryId():
    219                  self.log.info('Calling reIndex %s', obj.getPrimaryId())
    220                  obj.reIndex()
    221                  self.rootpath = ''
    222              if (not noIncrementalCommit and
    223                  not self.objectnumber % self.options.chunk_size):
    224                  self.log.debug("Committing a batch of %s objects" %
    225                                 self.options.chunk_size)
    226                  self.commit()
    227
    228          elif name == 'objects': # ie end of the file
    229              self.log.info('End loading objects')
    230              self.log.info('Processing links')
    231              self.processLinks()
    232              if not self.options.noCommit:
    233                  self.commit()
    234                  self.log.info('Loaded %d objects into the ZODB database'
    235                             % self.objectnumber)
    236              else:
    237                  self.log.info('Would have created %d objects in the ZODB database'
    238                             % self.objectnumber)
    239
    240          elif name == 'property':
    241              self.setProperty(self.context(), self.curattrs,
    242                               self.charvalue)
    243              # We've closed off a tag, so now we need to re-initialize
    244              # the area that stores the contents of elements
    245              self.charvalue = ''
    246
    247          elif name in ignoredElements:
    248              pass
    249          else:
    250              self.log.warning( "Ignoring an unknown XML element type: %s" % name )
    251
    252 -    def characters(self, chars):
    253          """
    254          Called by xml.sax.parse() with data found in an element
    255          eg <object>my characters stuff</object>
    256
    257          Note that this can be called character by character.
    258
    259          @param chars: chars
    260          @type chars: string
    261          """
    262          self.charvalue += saxutils.unescape(chars)
    263
    264 -    def createObject(self, attrs):
    265          """
    266          Create an object and set it into its container
    267
    268          @param attrs: attrs
    269          @type attrs: string
    270          @return: newly created object
    271          @rtype: object
    272          """
    273          # Does the object exist already?
    274          id = attrs.get('id')
    275          obj = None
    276          try:
    277              if id.startswith('/'):
    278                  obj = getObjByPath(self.app, id)
    279              else:
    280                  obj = self.context()._getOb(id)
    281          except (KeyError, AttributeError, NotFound):
    282              pass
    283
    284          if obj is None:
    285              klass = importClass(attrs.get('module'), attrs.get('class'))
    286              if id.find('/') > -1:
    287                  (contextpath, id) = os.path.split(id)
    288                  try:
    289                      pathobj = getObjByPath(self.context(), contextpath)
    290                  except (KeyError, AttributeError, NotFound):
    291                      self.log.warn( "Unable to find context path %s (line %s ?) for %s" % (
    292                          contextpath, self._locator.getLineNumber(), id ))
    293                      if not self.options.noCommit:
    294                          self.log.warn( "Not committing any changes" )
    295                          self.options.noCommit = True
    296                      return None
    297                  self.objstack.append(pathobj)
    298              self.log.debug('Building instance %s of class %s',id,klass.__name__)
    299              try:
    300                  if klass.__name__ == 'AdministrativeRole':
    301                      user = [x for x in self.dmd.ZenUsers.objectValues() if x.id == id]
    302                      if user:
    303                          obj = klass(user[0], self.context().device())
    304                      else:
    305                          msg = "No AdminRole user %s exists (line %s)" % (
    306                                         id, self._locator.getLineNumber())
    307                          self.log.error(msg)
    308                          raise Exception(msg)
    309                  else:
    310                      obj = klass(id)
    311              except TypeError, ex:
    312                  # This happens when the constructor expects more arguments
    313                  self.log.exception("Unable to build %s instance of class %s (line %s)",
    314                                     id, klass.__name__, self._locator.getLineNumber())
    315                  raise
    316              self.context()._setObject(obj.id, obj)
    317              obj = self.context()._getOb(obj.id)
    318              self.objectnumber += 1
    319              self.log.debug('Added object %s to database'
    320                              % obj.getPrimaryId())
    321          else:
    322              self.log.debug('Object %s already exists -- skipping' % id)
    323          return obj
    324
    325 -    def setZendoc(self, obj, value):
    326          zendocObj = zope.component.queryAdapter(obj, IZenDocProvider)
    327          if zendocObj is not None:
    328              zendocObj.setZendoc( value )
    329          elif value:
    330              self.log.warn('zendoc property could not be set to' +
    331                            ' %s on object %s' % ( value, obj.id ) )
    332
    333 -    def setProperty(self, obj, attrs, value):
    334          """
    335          Set the value of a property on an object.
    336
    337          @param obj: obj
    338          @type obj: string
    339          @param attrs: attrs
    340          @type attrs: string
    341          @param value: value
    342          @type value: string
    343          @return:
    344          @rtype:
    345          """
    346          name = attrs.get('id')
    347          proptype = attrs.get('type')
    348          setter = attrs.get('setter', None)
    349          self.log.debug('Setting object %s attr %s type %s value %s'
    350                          % (obj.id, name, proptype, value))
    351          linenum = self._locator.getLineNumber()
    352
    353          # Sanity check the value
    354          value = value.strip()
    355          try:
    356              value = str(value)
    357          except UnicodeEncodeError:
    358              self.log.warn('UnicodeEncodeError at line %s while attempting' % linenum + \
    359               ' str(%s) for object %s attribute %s -- ignoring' % (
    360                             obj.id, name, proptype, value))
    361
    362          if name == 'zendoc':
    363              return self.setZendoc( obj, value )
    364
    365          # Guess at how to interpret the value given the property type
    366          if proptype == 'selection':
    367              try:
    368                  firstElement = getattr(obj, name)[0]
    369                  if isinstance(firstElement, basestring):
    370                      proptype = 'string'
    371              except (TypeError, IndexError):
    372                  self.log.warn("Error at line %s when trying to " % linenum + \
    373                      " use (%s) as the value for object %s attribute %s -- assuming a string"
    374                                 % (obj.id, name, proptype, value))
    375                  proptype = 'string'
    376
    377          if proptype == 'date':
    378              try:
    379                  value = float(value)
    380              except ValueError:
    381                  pass
    382              value = DateTime(value)
    383
    384          elif proptype not in _STRING_PROPERTY_TYPES:
    385              try:
    386                  value = eval(value)
    387              except NameError:
    388                  self.log.critical('Error trying to evaluate %s at line %s',
    389                                    value, linenum)
    390                  raise
    391              except SyntaxError:
    392                  self.log.debug("Non-fatal SyntaxError at line %s while eval'ing '%s'" % (
    393                       linenum, value) )
    394
    395          # Actually use the value
    396          if not obj.hasProperty(name):
    397              obj._setProperty(name, value, type=proptype, setter=setter)
    398          else:
    399              obj._updateProperty(name, value)
    400
    401 -    def addLink(self, rel, objid):
    402          """
    403          Build list of links to form after all objects have been created
    404          make sure that we don't add other side of a bidirectional relation
    405
    406          @param rel: relationship object
    407          @type rel: relation object
    408          @param objid: objid
    409          @type objid: string
    410          """
    411          self.links.append((rel.getPrimaryId(), objid))
    412
    413 -    def processLinks(self):
    414          """
    415          Walk through all the links that we saved and link them up
    416          """
    417          for (relid, objid) in self.links:
    418              try:
    419                  self.log.debug('Linking relation %s to object %s',
    420                                 relid, objid)
    421                  rel = getObjByPath(self.app, relid)
    422                  obj = getObjByPath(self.app, objid)
    423                  if not rel.hasobject(obj):
    424                      rel.addRelation(obj)
    425              except:
    426                  self.log.critical('Failed linking relation %s to object %s' % (
    427                                    relid, objid))
    428
    429 -    def buildOptions(self):
    430          """
    431          Command-line options specific to this command
    432          """
    433          ZCmdBase.buildOptions(self)
    434          self.parser.add_option('-i', '--infile', dest='infile',
    435                                 help='Input file for import. Default is stdin'
    436                                 )
    437          self.parser.add_option('--noindex', dest='noindex',
    438                                 action='store_true', default=False,
    439                                 help='Do not try to index the freshly loaded objects.'
    440                                 )
    441          self.parser.add_option('--chunksize', dest='chunk_size',
    442                                 help='Number of objects to commit at a time.',
    443                                 type='int',
    444                                 default=100
    445                                 )
    446          self.parser.add_option(
    447              '-n',
    448              '--noCommit',
    449              dest='noCommit',
    450              action='store_true',
    451              default=0,
    452              help='Do not store changes to the DMD (ie for debugging purposes)',
    453              )
    454
    455 -    def loadObjectFromXML(self, xmlfile=''):
    456          """
    457          This method can be used to load data for the root of Zenoss (default
    458          behavior) or it can be used to operate on a specific point in the
    459          Zenoss hierarchy (ZODB).
    460
    461          Upon loading the XML file to be processed, the content of the XML file
    462          is handled by the methods in this class when called by xml.sax.parse().
    463
    464          Reads from a file if xmlfile is specified, otherwise reads
    465          from the command-line option --infile.  If no files are found from
    466          any of these places, read from standard input.
    467
    468          @param xmlfile: Name of XML file to load, or file-like object
    469          @type xmlfile: string or file-like object
    470          """
    471          self.objstack = [self.app]
    472          self.links = []
    473          self.objectnumber = 0
    474          self.charvalue = ''
    475          if xmlfile and isinstance(xmlfile, basestring):
    476              self.infile = open(xmlfile)
    477          elif hasattr(xmlfile, 'read'):
    478              self.infile = xmlfile
    479          elif self.options.infile:
    480              self.infile = open(self.options.infile)
    481          else:
    482              self.infile = sys.stdin
    483          parser = make_parser()
    484          parser.setContentHandler(self)
    485          try:
    486              parser.parse(self.infile)
    487          except SAXParseException, ex:
    488              self.log.error("XML parse error at line %d column %d: %s",
    489                     ex.getLineNumber(), ex.getColumnNumber(),
    490                     ex.getMessage())
    491          finally:
    492              self.infile.close()
    493
    494 -    def loadDatabase(self):
    495          """
    496          The default behavior of loadObjectFromXML() will be to use the Zope
    497          app object, and thus operatate on the whole of Zenoss.
    498          """
    499          self.loadObjectFromXML()
    500
    501 -    def commit(self):
    502          """
    503          Wrapper around the Zope database commit()
    504          """
    505          trans = transaction.get()
    506          trans.note('Import from file %s using %s'
    507                      % (self.options.infile, self.__class__.__name__))
    508          trans.commit()
    509          if hasattr(self, 'connection'):
    510              # It's safe to call syncdb()
    511              self.syncdb()
    512
    513 -class SpoofedOptions(object):
    514      """
    515      SpoofedOptions
    516      """
    517
    518 -    def __init__(self):
    519          self.infile = ''
    520          self.noCommit = True
    521          self.noindex = True
    522          self.dataroot = '/zport/dmd'
    523
    524
    525 -class NoLoginImportRM(ImportRM):
    526      """
    527      An ImportRM that does not call the __init__ method on ZCmdBase
    528      """
    529
    530 -    def __init__(self, app):
    531          """
    532          Initializer
    533
    534          @param app: app
    535          @type app: string
    536          """
    537          import Products.ZenossStartup
    538          from Products.Five import zcml
    539          zcml.load_site()
    540          import logging
    541          self.log = logging.getLogger('zen.ImportRM')
    542          self.app = app
    543          ContentHandler.__init__(self)
    544          self.options = SpoofedOptions()
    545          self.dataroot = None
    546          self.getDataRoot()
    547
    548
    549  if __name__ == '__main__':
    550      im = ImportRM()
    551      im.loadDatabase()
    552