Package Products :: Package ZenModel :: Module ZenPackLoader |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenModel.ZenPackLoader

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2007, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  __doc__='Base Classes for loading gunk in a ZenPack'
     12
     13  import Globals
     14  from Products.ZenReports.ReportLoader import ReportLoader
     15  from Products.ZenUtils.Utils import zenPath, binPath
     16  from Products.ZenUtils.guid.interfaces import IGUIDManager
     17  from Products.Zuul import getFacade
     18
     19  from zenoss.protocols.jsonformat import from_dict
     20  from zenoss.protocols.protobufs.zep_pb2 import EventDetailItemSet, EventDetailItem
     21  from zenoss.protocols.services import ServiceResponseError
     22
     23  import os
     24  import json
     25  import ConfigParser
     26  import subprocess
     27  import logging
     28  log = logging.getLogger('zen.ZPLoader')
     29
     30  CONFIG_FILE = 'about.txt'
     31  CONFIG_SECTION_ABOUT = 'about'
     32
     33 -def findFiles(pack, directory, filter=None, excludedDirs=('.svn',)):
     34      result = []
     35      if isinstance(pack, basestring):
     36          path = os.path.join(pack, directory)
     37      else:
     38          path = pack.path(directory)
     39      for p, ds, fs in os.walk(path):
     40          # Don't visit excluded directories - os.walk with topdown=True
     41          for excludedDir in excludedDirs:
     42              if excludedDir in ds:
     43                  ds.remove(excludedDir)
     44          if not os.path.split(p)[-1].startswith('.'):
     45              for f in fs:
     46                  if filter is None or filter(f):
     47                      result.append(os.path.join(p, f))
     48      return result
     49
     50 -def findDirectories(pack, directory):
     51      result = []
     52      for p, ds, fs in os.walk(pack.path(directory)):
     53          if not os.path.split(p)[-1].startswith('.'):
     54              for d in ds:
     55                  result.append(os.path.join(p, d))
     56      return result
     57
     58 -def branchAfter(filename, directory, prefix = ""):
     59      "return the branch after the given directory name"
     60      path = filename.split('/')
     61      return prefix + '/'.join(path[path.index(directory)+1:])
     62
     63
     64 -class ZenPackLoader:
     65
     66      name = "Set This Name"
     67
     68 -    def load(self, pack, app):
     69          """Load things from the ZenPack and put it
     70          into the app"""
     71
     72 -    def unload(self, pack, app, leaveObjects=False):
     73          """Remove things from Zenoss defined in the ZenPack"""
     74
     75 -    def list(self, pack, app):
     76          "List the items that would be loaded from the given (unpacked) ZenPack"
     77
     78 -    def upgrade(self, pack, app):
     79          "Run an upgrade on an existing pack"
     80
     81  from xml.sax import make_parser
     82
     83 -class ZPLObject(ZenPackLoader):
     84
     85      name = "Objects"
     86
     87 -    def load(self, pack, app):
     88          from Products.ZenRelations.ImportRM import ImportRM
     89          class AddToPack(ImportRM):
     90              def endElement(self, name):
     91                  if name == 'object':
     92                      obj = self.objstack[-1]
     93                      log.debug('Now adding %s', obj.getPrimaryUrlPath())
     94                      try:
     95                          obj.buildRelations()
     96                          obj.removeRelation('pack')
     97                          obj.addRelation('pack', pack)
     98                      except Exception, ex:
     99                          log.exception("Error adding pack to %s",
    100                                        obj.getPrimaryUrlPath())
    101
    102                  ImportRM.endElement(self, name)
    103          importer = AddToPack(noopts=True, app=app)
    104          importer.options.noindex = True
    105          for f in self.objectFiles(pack):
    106              log.info("Loading %s", f)
    107              importer.loadObjectFromXML(xmlfile=f)
    108
    109
    110 -    def parse(self, filename, handler):
    111          parser = make_parser()
    112          parser.setContentHandler(handler)
    113          parser.parse(open(filename))
    114
    115
    116 -    def unload(self, pack, app, leaveObjects=False):
    117          if leaveObjects:
    118              return
    119          from Products.ZenRelations.Exceptions import ObjectNotFound
    120          dmd = app.zport.dmd
    121          objs = sorted(pack.packables(), key=lambda x: x.getPrimaryPath(),
    122                          reverse=True)
    123          for obj in objs:
    124              path = obj.getPrimaryPath()
    125              path, id = path[:-1], path[-1]
    126              obj = dmd.getObjByPath(path)
    127              if len(path) > 3:           # leave /Services, /Devices, etc.
    128                  try:
    129                      try:
    130                          obj._delObject(id)
    131                      except ObjectNotFound:
    132                          obj._delOb(id)
    133                  except (AttributeError, KeyError), ex:
    134                      log.warning("Unable to remove %s on %s", id,
    135                                  '/'.join(path))
    136
    137 -    def list(self, pack, unused):
    138          return [obj.getPrimaryUrlPath() for obj in pack.packables()]
    139
    140
    141 -    def objectFiles(self, pack):
    142          def isXml(f): return f.endswith('.xml')
    143          return findFiles(pack, 'objects', isXml)
    144
    145
    146 -class ZPLReport(ZPLObject):
    147
    148      name = "Reports"
    149
    150 -    def load(self, pack, app):
    151          class HookReportLoader(ReportLoader):
    152              def loadFile(self, root, id, fullname):
    153                  rpt = ReportLoader.loadFile(self, root, id, fullname)
    154                  rpt.addRelation('pack', pack)
    155                  return rpt
    156          rl = HookReportLoader(noopts=True, app=app)
    157          rl.options.force = True
    158          rl.loadDirectory(pack.path('reports'))
    159
    160 -    def upgrade(self, pack, app):
    161          self.unload(pack, app)
    162          self.load(pack, app)
    163
    164 -    def list(self, pack, unused):
    165          return [branchAfter(r, 'reports') for r in findFiles(pack, 'reports')]
    166
    167
    168 -class ZPLDaemons(ZenPackLoader):
    169
    170      name = "Daemons"
    171
    172      extensionsToIgnore = ('.svn-base', '.pyc' '~')
    173 -    def filter(self, f):
    174          if 'zenexample' in f:
    175              return False
    176
    177          for ext in self.extensionsToIgnore:
    178              if f.endswith(ext):
    179                  return False
    180
    181          return True
    182
    183
    184 -    def binPath(self, daemon):
    185          return zenPath('bin', os.path.basename(daemon))
    186
    187
    188 -    def _genConfFile(self, pack):
    189          """
    190          Attempt to generate a conf file for any daemons. Wait for completion.
    191          """
    192          try:
    193              p = subprocess.Popen(binPath('create_sample_config.sh'),
    194                              stdout=subprocess.PIPE,
    195                              stderr=subprocess.PIPE,
    196                              cwd=pack.path())
    197              p.wait()
    198          except OSError:
    199              pass
    200
    201 -    def _updateConfFile(self, pack):
    202          """
    203          Update conf files for any daemons to account for logfile path
    204          differences on the localhost collector.
    205          """
    206          for fs in findFiles(pack, 'daemons', filter=self.filter):
    207              name = fs.rsplit('/', 1)[-1]
    208              logpath = pack.About._getLogPath(name).rsplit('/', 1)[0]
    209              if logpath != zenPath('log'):
    210                  try:
    211                      with open(zenPath('etc', '%s.conf' % name), 'a') as conf:
    212                          conf.write('logpath %s' % logpath)
    213                  except IOError:
    214                      # No conf file. Move on.
    215                      pass
    216
    217 -    def load(self, pack, unused):
    218          for fs in findFiles(pack, 'daemons', filter=self.filter):
    219              os.chmod(fs, 0755)
    220              path = self.binPath(fs)
    221              if os.path.lexists(path):
    222                  os.remove(path)
    223              os.symlink(fs, path)
    224          self._genConfFile(pack)
    225          self._updateConfFile(pack)
    226
    227
    228 -    def upgrade(self, pack, app):
    229          self.unload(pack, app)
    230          self.load(pack, app)
    231
    232
    233 -    def unload(self, pack, unused, leaveObjects=False):
    234          for fs in findFiles(pack, 'daemons', filter=self.filter):
    235              try:
    236                  os.remove(self.binPath(fs))
    237              except OSError:
    238                  pass
    239
    240 -    def list(self, pack, unused):
    241          return [branchAfter(d, 'daemons')
    242                  for d in findFiles(pack, 'daemons', filter=self.filter)]
    243
    244
    245 -class ZPLBin(ZenPackLoader):
    246
    247      name = "Bin"
    248
    249      extensionsToIgnore = ('.svn-base', '.pyc' '~')
    250 -    def filter(self, f):
    251          for ext in self.extensionsToIgnore:
    252              if f.endswith(ext):
    253                  return False
    254          return True
    255
    256 -    def load(self, pack, unused):
    257          for fs in findFiles(pack, 'bin', filter=self.filter):
    258              os.chmod(fs, 0755)
    259
    260 -    def upgrade(self, pack, app):
    261          self.unload(pack, app)
    262          self.load(pack, app)
    263
    264 -    def list(self, pack, unused):
    265          return [branchAfter(d, 'bin')
    266                  for d in findFiles(pack, 'bin', filter=self.filter)]
    267
    268
    269 -class ZPLLibExec(ZenPackLoader):
    270
    271      name = "LibExec"
    272
    273      extensionsToIgnore = ('.svn-base', '.pyc' '~')
    274 -    def filter(self, f):
    275          for ext in self.extensionsToIgnore:
    276              if f.endswith(ext):
    277                  return False
    278          return True
    279
    280 -    def load(self, pack, unused):
    281          for fs in findFiles(pack, 'libexec', filter=self.filter):
    282              os.chmod(fs, 0755)
    283
    284 -    def upgrade(self, pack, app):
    285          self.unload(pack, app)
    286          self.load(pack, app)
    287
    288 -    def list(self, pack, unused):
    289          return [branchAfter(d, 'libexec')
    290                  for d in findFiles(pack, 'libexec', filter=self.filter)]
    291
    292
    293 -class ZPLModelers(ZenPackLoader):
    294
    295      name = "Modeler Plugins"
    296
    297
    298 -    def list(self, pack, unused):
    299          return [branchAfter(d, 'plugins')
    300                  for d in findFiles(pack, 'modeler/plugins')]
    301
    302
    303 -class ZPLSkins(ZenPackLoader):
    304
    305      name = "Skins"
    306
    307
    308 -    def load(self, pack, app):
    309          from Products.ZenUtils.Skins import registerSkin
    310          skinsDir = pack.path('')
    311          if os.path.isdir(skinsDir):
    312              registerSkin(app.zport.dmd, skinsDir)
    313
    314
    315 -    def upgrade(self, pack, app):
    316          self.unload(pack, app)
    317          return self.load(pack, app)
    318
    319
    320 -    def unload(self, pack, app, leaveObjects=False):
    321          from Products.ZenUtils.Skins import unregisterSkin
    322          skinsDir = pack.path('')
    323          if os.path.isdir(skinsDir):
    324              unregisterSkin(app.zport.dmd, skinsDir)
    325
    326
    327 -    def list(self, pack, unused):
    328          return [branchAfter(d, 'skins') for d in findDirectories(pack, 'skins')]
    329
    330
    331 -class ZPLDataSources(ZenPackLoader):
    332
    333      name = "DataSources"
    334
    335
    336 -    def list(self, pack, unused):
    337          return [branchAfter(d, 'datasources')
    338                  for d in findFiles(pack, 'datasources',
    339                  lambda f: not f.endswith('.pyc') and f != '__init__.py')]
    340
    341
    342
    343 -class ZPLLibraries(ZenPackLoader):
    344
    345      name = "Libraries"
    346
    347
    348 -    def list(self, pack, unused):
    349          d = pack.path('lib')
    350          if os.path.isdir(d):
    351              return [l for l in os.listdir(d)]
    352          return []
    353
    354 -class ZPLAbout(ZenPackLoader):
    355
    356      name = "About"
    357
    358 -    def getAttributeValues(self, pack):
    359          about = pack.path(CONFIG_FILE)
    360          result = []
    361          if os.path.exists(about):
    362              parser = ConfigParser.SafeConfigParser()
    363              parser.read(about)
    364              result = []
    365              for key, value in parser.items(CONFIG_SECTION_ABOUT):
    366                  try:
    367                      value = eval(value)
    368                  except:
    369                      # Blanket exception catchers like this are evil.
    370                      pass
    371                  result.append((key, value))
    372          return result
    373
    374
    375 -    def load(self, pack, unused):
    376          for name, value in self.getAttributeValues(pack):
    377              setattr(pack, name, value)
    378
    379
    380 -    def upgrade(self, pack, app):
    381          self.load(pack, app)
    382
    383
    384 -    def list(self, pack, unused):
    385          return [('%s %s' % av) for av in self.getAttributeValues(pack)]
    386
    387 -class ZPTriggerAction(ZenPackLoader):
    388
    389      name = "Triggers and Actions"
    390
    391 -    def load(self, pack, app):
    392          """
    393          Load Notifications and Triggers from an actions.json file
    394
    395          Given a JSON-formatted configuration located at {zenpack}/actions/actions.json,
    396          create or update triggers and notifications specific to this zenpack.
    397          When creating or updating, the object is first checked to see whether or
    398          not an object exists with the configured guid for notifications or uuid
    399          for triggers. If an object is not found, one will be created. During
    400          creation, care is taken with regard to the id - integer suffixes will be
    401          appended to try to create a unique id. If we do not find a unique id after
    402          100 tries, an error will occur. When updating an object, care is taken
    403          to not change the name as it may have since been altered by the user (or
    404          by this loader adding a suffix).
    405
    406          """
    407          log.debug("ZPTriggerAction: load")
    408          import Products.Zuul as Zuul
    409          from Products.Zuul.facades import ObjectNotFoundException
    410
    411          tf = Zuul.getFacade('triggers', app.dmd)
    412          guidManager = IGUIDManager(app)
    413
    414          for conf in findFiles(pack, 'zep',lambda f: f == 'actions.json'):
    415
    416              import json
    417              data = json.load(open(conf, "r"))
    418              log.debug("DATA IS: %s" % data)
    419
    420              triggers = data.get('triggers', [])
    421              notifications = data.get('notifications', [])
    422
    423
    424              tf.synchronize()
    425              all_names = set(t['name'] for t in tf.getTriggerList())
    426
    427              for trigger_conf in triggers:
    428
    429                  existing_trigger = guidManager.getObject(trigger_conf['uuid'])
    430
    431                  if existing_trigger:
    432                      trigger_data = tf.getTrigger(trigger_conf['uuid'])
    433                      trigger_conf['name'] = trigger_data['name']
    434
    435                      log.info('Existing trigger found, updating: %s' % trigger_conf['name'])
    436                      tf.updateTrigger(**trigger_conf)
    437
    438                  else:
    439
    440                      test_name = trigger_conf['name']
    441                      for x in xrange(1,101):
    442                          if test_name in all_names:
    443                              test_name = '%s_%d' % (trigger_conf['name'], x)
    444                          else:
    445                              log.debug('Found unused trigger name: %s' % test_name)
    446                              break
    447                      else:
    448                          # if we didn't find a unique name
    449                          raise Exception('Could not find unique name for trigger: "%s".' % trigger_conf['name'])
    450
    451                      log.info('Creating trigger: %s' % test_name)
    452                      tf.createTrigger(test_name, uuid=trigger_conf['uuid'], rule=trigger_conf['rule'])
    453
    454
    455              for notification_conf in notifications:
    456
    457                  existing_notification = guidManager.getObject(str(notification_conf['guid']))
    458
    459                  if existing_notification:
    460                      log.info("Existing notification found, updating: %s" % existing_notification.id)
    461
    462                      subscriptions = set(existing_notification.subscriptions + notification_conf['subscriptions'])
    463                      notification_conf['uid'] = '/zport/dmd/NotificationSubscriptions/%s' % existing_notification.id
    464                      notification_conf['subscriptions'] = list(subscriptions)
    465                      notification_conf['name'] = existing_notification.id
    466                      tf.updateNotification(**notification_conf)
    467                  else:
    468
    469
    470                      test_id = notification_conf['id']
    471                      for x in xrange(1,101):
    472                          test_uid = '/zport/dmd/NotificationSubscriptions/%s' % test_id
    473
    474                          try:
    475                              tf.getNotification(test_uid)
    476                          except ObjectNotFoundException:
    477                              break
    478
    479                          test_id = '%s_%d' % (notification_conf['id'], x)
    480                      else:
    481                          # if we didn't find a unique name
    482                          raise Exception('Could not find unique name for notification: "%s".' % notification_conf['id'])
    483
    484                      log.info('Creating notification: %s' % test_id)
    485                      tf.createNotification(str(test_id), notification_conf['action'], notification_conf['guid'])
    486
    487                      notification_conf['uid'] = '/zport/dmd/NotificationSubscriptions/%s' % test_id
    488                      tf.updateNotification(**notification_conf)
    489
    490
    491 -    def _getTriggerGuid(self, facade, name):
    492          triggers = facade.getTriggers()
    493          guid = None
    494          for trigger in triggers:
    495              if trigger['name'] == name:
    496                  guid = trigger['uuid']
    497                  break
    498          if not guid:
    499              guid = facade.addTrigger(name)
    500          return guid
    501
    502 -    def unload(self, pack, app, leaveObjects=False):
    503          """Remove things from Zenoss defined in the ZenPack"""
    504          log.debug("ZPTriggerAction: unload")
    505
    506 -    def list(self, pack, app):
    507          "List the items that would be loaded from the given (unpacked) ZenPack"
    508          log.debug("ZPTriggerAction: list")
    509
    510 -    def upgrade(self, pack, app):
    511          "Run an upgrade on an existing pack"
    512          log.debug("ZPTriggerAction: upgrade")
    513
    514
    515 -class ZPZep(ZenPackLoader):
    516
    517      name = "ZEP"
    518
    519 -    def _data(self, conf):
    520          data = {}
    521          try:
    522              with open(conf, "r") as configFile:
    523                  data = json.load(configFile)
    524          except IOError, e:
    525              # this file doesn't exist in this zenpack.
    526              log.debug("File could not be opened for reading: %s" % conf)
    527              pass
    528          return data
    529
    530 -    def _prepare(self, pack, app):
    531          """
    532          Load in the Zep configuration file which should be located here:
    533          $ZENPACK/zep/zep.json
    534          """
    535
    536          self.handlers = (EventDetailItemHandler(), )
    537          p = pack.path('zep')
    538          confFile = os.path.join(p, 'zep.json')
    539          data = self._data(confFile)
    540
    541          return data
    542
    543 -    def load(self, pack, app):
    544          data = self._prepare(pack, app)
    545          for handler in self.handlers:
    546              handler.load(data)
    547
    548 -    def unload(self, pack, app, leaveObjects=False):
    549          data = self._prepare(pack, app)
    550          for handler in self.handlers:
    551              handler.unload(data, leaveObjects)
    552
    553 -    def list(self, pack, app):
    554          data = self._prepare(pack, app)
    555          info = []
    556          for handler in self.handlers:
    557              info.extend(handler.list(data))
    558
    559 -    def upgrade(self, pack, app):
    560          data = self._prepare(pack, app)
    561          for handler in self.handlers:
    562              handler.upgrade(data)
    563
    564
    565
    566 -class EventDetailItemHandler(object):
    567      key = 'EventDetailItem'
    568
    569 -    def load(self, configData):
    570          """
    571          configData is a json dict. This is the entire config structure.
    572          """
    573          if configData:
    574              self.zep = getFacade('zep')
    575              items = configData.get(EventDetailItemHandler.key, [])
    576              detailItemSet = from_dict(EventDetailItemSet, dict(
    577                  details = items
    578              ))
    579              self.zep.addIndexedDetails(detailItemSet)
    580
    581 -    def list(self, configData):
    582          if configData:
    583              self.zep = getFacade('zep')
    584              items = configData.get(EventDetailItemHandler.key, [])
    585              info = []
    586              for item in items:
    587                  info.append("Would be adding the following detail to be indexed by ZEP: %s" % item.key)
    588              return info
    589
    590 -    def unload(self, configData, leaveObjects):
    591          if not leaveObjects and configData:
    592              self.zep = getFacade('zep')
    593              items = configData.get(EventDetailItemHandler.key, [])
    594              for item in items:
    595                  log.info("Removing the following currently indexed detail by ZEP: %s" % item['key'])
    596                  try:
    597                      self.zep.removeIndexedDetail(item['key'])
    598                  except ServiceResponseError as e:
    599                      if e.status == 404:
    600                          log.debug('Indexed detail was previously removed from ZEP')
    601                      else:
    602                          log.warning("Failed to remove indexed detail: %s", e.message)
    603
    604
    605 -    def upgrade(self, configData):
    606          if configData:
    607              self.zep = getFacade('zep')
    608              items = configData.get(EventDetailItemHandler.key, [])
    609              for item in items:
    610                  log.info("Upgrading the following to be indexed by ZEP: %s" % item)
    611                  detailItem = from_dict(EventDetailItem, item)
    612                  self.zep.updateIndexedDetailItem(detailItem)
    613