Package Products :: Package ZenUtils :: Module ZenPackCmd |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenPackCmd

       1  ###########################################################################
       2  #
       3  # This program is part of Zenoss Core, an open source monitoring platform.
       4  # Copyright (C) 2008, Zenoss Inc.
       5  #
       6  # This program is free software; you can redistribute it and/or modify it
       7  # under the terms of the GNU General Public License version 2 or (at your
       8  # option) any later version as published by the Free Software Foundation.
       9  #
      10  # For complete information please visit: http://www.zenoss.com/oss/
      11  #
      12  ###########################################################################
      13
      14  __doc__ = "Manage ZenPacks"
      15
      16  import Globals
      17  from ZODB.transact import transact
      18  from Products.ZenUtils.ZenScriptBase import ZenScriptBase
      19  from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, getObjByPath
      20
      21  from Products.ZenModel.ZenPack import ZenPackException, \
      22                                          ZenPackNotFoundException, \
      23                                          ZenPackNeedMigrateException
      24  from Products.ZenModel.ZenPack import ZenPackDependentsException
      25  from Products.ZenModel.ZenPack import ZenPack
      26  from Products.ZenUtils.PkgResources import pkg_resources
      27  from Products.Zuul.utils import CatalogLoggingFilter
      28  import Products.ZenModel.ZenPackLoader as ZPL
      29  import zenpack as oldzenpack
      30  import transaction
      31  import os, sys
      32  import shutil
      33  import string
      34  import tempfile
      35  import subprocess
      36  import socket
      37  import logging
      38  import zExceptions
      39
      40
      41  log = logging.getLogger('zen.ZenPackCMD')
      42
      43  #import zenpacksupport
      44
      45  FQDN = socket.getfqdn()
      46
      47  ZEN_PACK_INDEX_URL = ''
      48
      49  # All ZenPack eggs have to define exactly one entry point in this group.
      50  ZENPACK_ENTRY_POINT = 'zenoss.zenpacks'
      51
      52  ########################################
      53  #   ZenPack Creation
      54  ########################################
      55
      56 -def CreateZenPack(zpId, prevZenPackName=''):
      57      """
      58      Create the zenpack in the filesystem.
      59      The zenpack is not installed in Zenoss, it is simply created in
      60      the $ZENHOME/ZenPacks directory.  Usually this should be followed
      61      with a "zenpack install" call.
      62      zpId should already be valid, scrubbed value.
      63      prevZenPackName is written to PREV_ZENPACK_NAME in setup.py.
      64      """
      65      parts = zpId.split('.')
      66
      67      # Copy template to $ZENHOME/ZenPacks
      68      srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate')
      69      devDir = zenPath('ZenPacks')
      70      if not os.path.exists(devDir):
      71          os.mkdir(devDir, 0750)
      72      destDir = os.path.join(devDir, zpId)
      73      shutil.copytree(srcDir, destDir, symlinks=False)
      74      os.system('find %s -name .svn | xargs rm -rf' % destDir)
      75
      76      # Write setup.py
      77      packages = []
      78      for i in range(len(parts)):
      79          packages.append('.'.join(parts[:i+1]))
      80      mapping = dict(
      81          NAME = zpId,
      82          VERSION = '1.0.0',
      83          AUTHOR = '',
      84          LICENSE = '',
      85          NAMESPACE_PACKAGES = packages[:-1],
      86          PACKAGES = packages,
      87          INSTALL_REQUIRES = [],
      88          COMPAT_ZENOSS_VERS = '',
      89          PREV_ZENPACK_NAME = prevZenPackName,
      90          )
      91      WriteSetup(os.path.join(destDir, 'setup.py'), mapping)
      92
      93      # Create subdirectories
      94      base = destDir
      95      for part in parts[:-1]:
      96          base = os.path.join(base, part)
      97          os.mkdir(base)
      98          f = open(os.path.join(base, '__init__.py'), 'w')
      99          f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
     100          f.close()
     101      base = os.path.join(base, parts[-1])
     102      shutil.move(os.path.join(destDir, 'CONTENT'), base)
     103
     104      return destDir
     105
     106
     107 -def WriteSetup(setupPath, values):
     108      """
     109      """
     110      f = file(setupPath, 'r')
     111      lines = f.readlines()
     112      f.close()
     113
     114      newLines = []
     115      for i, line in enumerate(lines):
     116          if line.startswith('STOP_REPLACEMENTS'):
     117              newLines += lines[i:]
     118              break
     119          key = line.split('=')[0].strip()
     120          if key in values:
     121              value = values[key]
     122              if isinstance(value, basestring):
     123                  fmt = '%s = "%s"\n'
     124              else:
     125                  fmt = '%s = %s\n'
     126              newLines.append(fmt % (key, value))
     127          else:
     128              newLines.append(line)
     129
     130      f = file(setupPath, 'w')
     131      f.writelines(newLines)
     132      f.close()
     133
     134
     135 -def CanCreateZenPack(dmd, zpId):
     136      """
     137      Return tuple (bool, string) where first element is true if a new zenpack
     138      can be created with the given info and false if not.  If first element
     139      is True then the second part of the tuple contains the scrubbed ZenPack id.
     140      If the first part is False then the second contains an explanatory
     141      message.
     142      """
     143      # Check if id and package looks reasonable
     144      (allowable, idOrMsg) = ScrubZenPackId(zpId)
     145      if allowable:
     146          zpId = idOrMsg
     147      else:
     148          return (False, idOrMsg)
     149
     150      # Is the id already in use?
     151      if dmd:
     152          if zpId in dmd.ZenPackManager.packs.objectIds():
     153              return (False, 'A ZenPack named %s already exists.' % zpId)
     154
     155      # Is there another zenpack in the way?
     156      # Now that zenpacks are created in $ZENHOME/ZenPacks instead of
     157      # $ZENHOME/ZenPackDev this may no longer be necessary because a
     158      # zp in the way should be installed and caught by the already in use
     159      # check above.
     160      if os.path.exists(zenPath('ZenPacks', zpId)):
     161          return (False, 'A directory named %s already exists' % zpId +
     162                          ' in $ZENHOME/ZenPacks.  Use a different name'
     163                          ' or remove that directory.')
     164
     165      return (True, idOrMsg)
     166
     167
     168 -def ScrubZenPackId(name):
     169      """
     170      If the given name conforms to ZenPack naming rules, or can easily be
     171      modified to do so, then return (True, scrubbedName) where scrubbedName
     172      is either name or a slightly modified name.  If the given name does
     173      not conform to naming rules and we can't easily modify it to do so
     174      then return (False, errorMsg) where errorMsg describes why name
     175      is unacceptable.
     176      """
     177      parts = name.split('.')
     178
     179      # Remove leading dots, trailing dots, adjacent dots and strip whitespace
     180      # from each part
     181      parts = [p.strip() for p in parts]
     182      parts = [p for p in parts if p]
     183
     184      # Add/fix leading 'ZenPacks'
     185      if parts[0] != 'ZenPacks':
     186          if parts[0].lower() == 'zenpacks':
     187              parts[0] = 'ZenPacks'
     188          else:
     189              parts.insert(0, 'ZenPacks')
     190
     191      # Must be at least 3 parts
     192      if len(parts) < 3:
     193          return (False, 'ZenPack names must contain at least three package '
     194                  'names separated by periods.')
     195
     196      # Each part must start with a letter
     197      for p in parts:
     198          if p[0] not in string.letters:
     199              return (False, 'Each package name must start with a letter.')
     200
     201      # Only letters, numbers and underscores in each part
     202      allowable = string.letters + string.digits + '_'
     203      for p in parts:
     204          for c in p:
     205              if c not in allowable:
     206                  return (False, 'Package names may only contain letters, '
     207                          'numbers and underscores.')
     208
     209      return (True, '.'.join(parts))
     210
     211
     212  ########################################
     213  #   ZenPack Installation
     214  ########################################
     215
     216 -class NonCriticalInstallError(Exception):
     217 -    def __init__(self, message):
     218          Exception.__init__(self, message)
     219          self.message = message
     220
     221 -def InstallEggAndZenPack(dmd, eggPath, link=False,
     222                              filesOnly=False, sendEvent=True,
     223                              previousVersion=None, forceRunExternal=False):
     224      """
     225      Installs the given egg, instantiates the ZenPack, installs in
     226      dmd.ZenPackManager.packs, and runs the zenpacks's install method.
     227      Returns a list of ZenPacks that were installed.
     228      """
     229      zenPacks = []
     230      nonCriticalErrorEncountered = False
     231      try:
     232          zpDists = InstallEgg(dmd, eggPath, link=link)
     233          for d in zpDists:
     234              try:
     235                  zp = InstallDistAsZenPack(dmd,
     236                                            d,
     237                                            eggPath,
     238                                            link,
     239                                            filesOnly=filesOnly,
     240                                            previousVersion=previousVersion,
     241                                            forceRunExternal=forceRunExternal)
     242                  zenPacks.append(zp)
     243              except NonCriticalInstallError, ex:
     244                  nonCriticalErrorEncountered = True
     245                  if sendEvent:
     246                      ZPEvent(dmd, 3, ex.message)
     247      except:
     248          if sendEvent:
     249              ZPEvent(dmd, 4, 'Error installing ZenPack %s' % eggPath,
     250                  '%s: %s' % sys.exc_info()[:2])
     251          raise
     252      if sendEvent:
     253          zenPackIds = [zp.id for zp in zenPacks]
     254          if zenPackIds:
     255              ZPEvent(dmd, 2, 'Installed ZenPacks %s' % ','.join(zenPackIds))
     256          elif not nonCriticalErrorEncountered:
     257              ZPEvent(dmd, 4, 'Unable to install %s' % eggPath)
     258      return zenPacks
     259
     260
     261 -def InstallEgg(dmd, eggPath, link=False):
     262      """
     263      Install the given egg and add to the current working set.
     264      This does not install the egg as a ZenPack.
     265      Return a list of distributions that should be installed as ZenPacks.
     266      """
     267      eggPath = os.path.abspath(eggPath)
     268      zenPackDir = zenPath('ZenPacks')
     269      eggInZenPacksDir = eggPath.startswith(zenPackDir + '/')
     270
     271      # Make sure $ZENHOME/ZenPacks exists
     272      CreateZenPacksDir()
     273
     274      # Install the egg
     275      if link:
     276          cmd = ('%s setup.py develop ' % binPath('python') +
     277                  '--site-dirs=%s ' % zenPackDir +
     278                  '-d %s' % zenPackDir)
     279          p = subprocess.Popen(cmd,
     280                              stdout=subprocess.PIPE,
     281                              stderr=subprocess.PIPE,
     282                              shell=True,
     283                              cwd=eggPath)
     284          out, err = p.communicate()
     285          p.wait()
     286          if p.returncode:
     287              DoEasyUninstall(eggPath)
     288              raise ZenPackException('Error installing the egg (%s): %s' %
     289                                  (p.returncode, err))
     290          zpDists = AddDistToWorkingSet(eggPath)
     291      else:
     292          try:
     293              zpDists = DoEasyInstall(eggPath)
     294          except:
     295              DoEasyUninstall(eggPath)
     296              raise
     297          # cmd = 'easy_install --always-unzip --site-dirs=%s -d %s %s' % (
     298          #             zenPackDir,
     299          #             zenPackDir,
     300          #             eggPath)
     301          # p = subprocess.Popen(cmd,
     302          #                     stdout=subprocess.PIPE,
     303          #                     stderr=subprocess.PIPE,
     304          #                     shell=True)
     305          # p.wait()
     306          # eggName = os.path.split(eggPath)[1]
     307          # eggPath = os.path.join(zenPackDir, eggName)
     308      return zpDists
     309
     310
     311  # def GetZenPackNamesFromEggPath(eggPath):
     312  #     """
     313  #     Given a path to a ZenPack egg (installed or not) return the
     314  #     name of the ZenPack it contains.
     315  #     """
     316  #     zpNames = []
     317  #     for d in pkg_resources.find_distributions(eggPath)
     318  #         if d.project_name.startswith('ZenPacks.'):
     319  #             zpNames.append(d.project_name)
     320  #     return zpNames
     321
     322
     323 -def InstallDistAsZenPack(dmd, dist, eggPath, link=False, filesOnly=False,
     324                           previousVersion=None, forceRunExternal=False):
     325      """
     326      Given an installed dist, install it into Zenoss as a ZenPack.
     327      Return the ZenPack instance.
     328      """
     329      @transact
     330      def transactional_actions():
     331          # Instantiate ZenPack
     332          entryMap = pkg_resources.get_entry_map(dist, ZENPACK_ENTRY_POINT)
     333          if not entryMap or len(entryMap) > 1:
     334              raise ZenPackException('A ZenPack egg must contain exactly one'
     335                      ' zenoss.zenpacks entry point.  This egg appears to contain'
     336                      ' %s such entry points.' % len(entryMap))
     337          packName, packEntry = entryMap.items()[0]
     338          runExternalZenpack = True
     339          #if zenpack with same name exists we can't load both modules
     340          #installing new egg zenpack will be done in a sub process
     341          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     342          if existing:
     343              log.info("Previous ZenPack exists with same name %s" % packName)
     344          if filesOnly or not existing:
     345              #running files only or zenpack by same name doesn't already exists
     346              # so no need to install the zenpack in an external process
     347              runExternalZenpack = False
     348              module = packEntry.load()
     349              if hasattr(module, 'ZenPack'):
     350                  zenPack = module.ZenPack(packName)
     351              else:
     352                  zenPack = ZenPack(packName)
     353              zenPack.eggPack = True
     354              CopyMetaDataToZenPackObject(dist, zenPack)
     355              if filesOnly:
     356                  for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
     357                      loader.load(zenPack, None)
     358
     359
     360          if not filesOnly:
     361              # Look for an installed ZenPack to be upgraded.  In this case
     362              # upgraded means that it is removed before the new one is installed
     363              # but that its objects are not removed and the packables are
     364              # copied to the new instance.
     365              existing = dmd.ZenPackManager.packs._getOb(packName, None)
     366              if not existing and zenPack.prevZenPackName:
     367                  existing = dmd.ZenPackManager.packs._getOb(
     368                                      zenPack.prevZenPackName, None)
     369
     370              deferFileDeletion = False
     371              packables = []
     372              upgradingFrom = None
     373              if existing:
     374                  upgradingFrom = existing.version
     375                  for p in existing.packables():
     376                      packables.append(p)
     377                      existing.packables.removeRelation(p)
     378                  if existing.isEggPack():
     379                      forceNoFileDeletion = existing.eggPath() == dist.location
     380                      RemoveZenPack(dmd, existing.id,
     381                                      skipDepsCheck=True, leaveObjects=True,
     382                                      forceNoFileDeletion=forceNoFileDeletion,
     383                                      uninstallEgg=False)
     384                  else:
     385                      # Don't delete files, might still be needed for
     386                      # migrate scripts to be run below.
     387                      deferFileDeletion = True
     388                      oldzenpack.RemoveZenPack(dmd, existing.id,
     389                                      skipDepsCheck=True, leaveObjects=True,
     390                                      deleteFiles=False)
     391              if runExternalZenpack or forceRunExternal:
     392                  log.info("installing zenpack %s; launching process" % packName)
     393                  cmd = [binPath('zenpack')]
     394                  if link:
     395                      cmd += ["--link"]
     396                  cmd += ["--install", eggPath]
     397                  if upgradingFrom:
     398                      cmd += ['--previousversion', upgradingFrom]
     399
     400                  cmdStr = " ".join(cmd)
     401                  log.debug("launching sub process command: %s" % cmdStr)
     402                  p = subprocess.Popen(cmdStr,
     403                                  shell=True)
     404                  out, err = p.communicate()
     405                  p.wait()
     406                  if p.returncode:
     407                      raise ZenPackException('Error installing the egg (%s): %s' %
     408                                             (p.returncode, err))
     409                  dmd._p_jar.sync()
     410              else:
     411                  dmd.ZenPackManager.packs._setObject(packName, zenPack)
     412                  zenPack = dmd.ZenPackManager.packs._getOb(packName)
     413                  #hack because ZenPack.install is overridden by a lot of zenpacks
     414                  #so we can't change the signature of install to take the
     415                  #previousVerison
     416                  zenPack.prevZenPackVersion = previousVersion
     417                  zenPack.install(dmd)
     418                  zenPack.prevZenPackVersion = None
     419
     420              try:
     421                  zenPack = dmd.ZenPackManager.packs._getOb(packName)
     422                  for p in packables:
     423                      pId = p.getPrimaryId()
     424                      try:
     425                          # make sure packable still exists; could be deleted by a
     426                          # migrate
     427                          getObjByPath(dmd, pId)
     428                          log.debug("adding packable relation for id %s", pId)
     429                          zenPack.packables.addRelation(p)
     430                      except (KeyError, zExceptions.NotFound):
     431                          log.debug('did not find packable %s',pId)
     432              except AttributeError, e:
     433                  # If this happens in the child process or during the non-upgrade
     434                  # flow, reraise the exception
     435                  if not runExternalZenpack:
     436                      raise
     437
     438                  # This specific error will occur when the version of the ZenPack
     439                  # being installed subclasses Products.ZenModel.ZenPack, but the
     440                  # previous version of the ZenPack did not.
     441                  if str(e) == "'ZenPack' object has no attribute '__of__'":
     442                      zenPack = ZenPack(packName)
     443                  else:
     444                      # This is the signature error of class-loading issues
     445                      # during zenpack upgrade.  The final state should be okay,
     446                      # except that modified packables may be lost.
     447                      message = "There has been an error during the post-" + \
     448                                "installation steps for the zenpack %s.  In " + \
     449                                "most cases, no further action is required.  If " + \
     450                                "issues persist, please reinstall this zenpack."
     451                      message = message % packName
     452                      log.warning( message )
     453                      raise NonCriticalInstallError( message )
     454
     455              cleanupSkins(dmd)
     456              return zenPack, deferFileDeletion, existing
     457
     458      zenPack, deferFileDeletion, existing = transactional_actions()
     459
     460      if not filesOnly and deferFileDeletion:
     461          # We skipped deleting the existing files from filesystem
     462          # because maybe they'd be needed in migrate scripts.
     463          # Delete them now
     464          oldZpDir = zenPath('Products', existing.id)
     465          if os.path.islink(oldZpDir):
     466              os.remove(oldZpDir)
     467          else:
     468              shutil.rmtree(oldZpDir)
     469
     470      return zenPack
     471
     472
     473 -def DiscoverEggs(dmd, zenPackId):
     474      """
     475      Find installed eggs that provide a zenoss.zenpacks entry point.
     476      Return a list of distributions whose ZenPacks need to be installed
     477      or upgraded.  The list is sorted into the order in which this needs to
     478      happen.
     479      """
     480      # Create a set of all available zenoss.zenpack entries that aren't
     481      # already installed in zenoss or need to be upgraded in zenoss.
     482      entries = set()
     483      parse_version = pkg_resources.parse_version
     484      for entry in pkg_resources.iter_entry_points(ZENPACK_ENTRY_POINT):
     485          packName = entry.name
     486          packVers = entry.dist.version
     487          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     488          if existing and existing.isEggPack():
     489              # We use >= to allow migrate to be run on currently installed
     490              # zenpacks whose version has been changed or for whom new
     491              # migrates have been added.
     492              if parse_version(packVers) >= parse_version(existing.version):
     493                  entries.add(entry)
     494          else:
     495              entries.add(entry)
     496
     497      # Starting with the entry representing zenPackId create a list of
     498      # all entrypoints
     499
     500      # orderedEntries lists entries in the opposite order of that in which
     501      # they need to be installed.  This is simply for convenience of using
     502      # .append() in code below.
     503      orderedEntries = []
     504      entriesByName = dict((e.name, e) for e in entries)
     505
     506      def AddEntryAndProcessDeps(e):
     507          orderedEntries.append(e)
     508          for name in [r.project_name for r in e.dist.requires()]:
     509              if name in [e.name for e in orderedEntries]:
     510                  # This entry depends on something that we've already processed.
     511                  # This might be a circular dependency, might not be.
     512                  # We are just going to bail however.  This should be
     513                  # very unusual and the user can install deps first to work
     514                  # around.
     515                  raise ZenPackException('Unable to resolve ZenPack dependencies.'
     516                      ' Try installing dependencies first.')
     517              if name in entriesByName:
     518                  # The requirement is an entry that has not yet been processed
     519                  # here.  Add it to the list of entries to install/upgrade.
     520                  AddEntryAndProcessDeps(entriesByName[name])
     521              else:
     522                  # The requirement is not in the entries generated above.
     523                  # This either means that the dep is already installed (this
     524                  # is likely) or that easy_install missed something and the dep
     525                  # is not installed/available (this should be unlikely.)
     526                  pass
     527
     528      if zenPackId not in entriesByName:
     529          if zenPackId in dmd.ZenPackManager.packs.objectIds():
     530              return []
     531          else:
     532              raise ZenPackException('Unable to discover ZenPack named %s' %
     533                                      zenPackId)
     534      AddEntryAndProcessDeps(entriesByName[zenPackId])
     535      orderedEntries.reverse()
     536      return [e.dist for e in orderedEntries]
     537
     538
     539 -def AddDistToWorkingSet(distPath):
     540      """
     541      Given the path to a dist (an egg) add it to the current working set.
     542      This is basically a pkg_resources-friendly way of adding it to
     543      sys.path.
     544      Return a list of all distributions on distPath that appear to
     545      be ZenPacks.
     546      """
     547      zpDists = []
     548      for d in pkg_resources.find_distributions(distPath):
     549          pkg_resources.working_set.add(d)
     550          pkg_resources.require(d.project_name)
     551          if d.project_name.startswith('ZenPacks.'):
     552              zpDists.append(d)
     553      return zpDists
     554
     555
     556 -def ReadZenPackInfo(dist):
     557      """
     558      Return a dictionary containing the egg metadata
     559      """
     560      info = {}
     561      if dist.has_metadata('PKG-INFO'):
     562          lines = dist.get_metadata('PKG-INFO')
     563          for line in pkg_resources.yield_lines(lines):
     564              key, value = line.split(':', 1)
     565              info[key.strip()] = value.strip()
     566      if dist.has_metadata('zenpack_info'):
     567          lines = dist.get_metadata('zenpack_info')
     568          for line in pkg_resources.yield_lines(lines):
     569              key, value = line.split(':', 1)
     570              info[key.strip()] = value.strip()
     571      return info
     572
     573
     574 -def CopyMetaDataToZenPackObject(dist, pack):
     575      """
     576      Copy metadata type stuff from the distribution to the zp object.
     577      """
     578      # Version
     579      pack.version = dist.version
     580
     581      # Egg Info
     582      info = ReadZenPackInfo(dist)
     583      pack.author = info.get('Author', '')
     584      if pack.author == 'UNKNOWN':
     585          pack.author = ''
     586      pack.compatZenossVers = info.get('compatZenossVers', '')
     587      pack.prevZenPackName = info.get('prevZenPackName', '')
     588
     589      # Requires
     590      pack.dependencies = {}
     591      for r in dist.requires():
     592          name = r.project_name
     593          spec = str(r)[len(name):]
     594          pack.dependencies[name] = spec
     595
     596
     597 -def CreateZenPacksDir():
     598      """
     599      Make sure $ZENHOME/ZenPacks exists
     600      """
     601      zpDir = zenPath('ZenPacks')
     602      if not os.path.isdir(zpDir):
     603          os.mkdir(zpDir, 0750)
     604
     605
     606 -def DoEasyInstall(eggPath):
     607      """
     608      Use easy_install to install an egg from the filesystem.
     609      easy_install will install the egg, but does not install it into
     610      Zenoss as ZenPacks.
     611      Returns a list of distributions that were installed that appear
     612      to be ZenPacks.
     613      """
     614      from setuptools.command import easy_install
     615
     616      # Make sure $ZENHOME/ZenPacks exists
     617      CreateZenPacksDir()
     618
     619      # Create temp file for easy_install to write results to
     620      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     621      # eggPaths is a set of paths to eggs that were installed.  We need to
     622      # add them to the current workingset so we can discover their
     623      # entry points.
     624      eggPaths = set()
     625      try:
     626          # Execute the easy_install
     627          args = ['--site-dirs', zenPath('ZenPacks'),
     628              '-d', zenPath('ZenPacks'),
     629              # '-i', ZEN_PACK_INDEX_URL,
     630              '--allow-hosts', 'None',
     631              '--record', tempPath,
     632              '--quiet',
     633              eggPath]
     634          easy_install.main(args)
     635          # Collect the paths for eggs that were installed
     636          f = open(tempPath, 'r')
     637          marker = '.egg/'
     638          markerOffset = len(marker)-1
     639          for l in f.readlines():
     640              i = l.find(marker)
     641              if i > 0:
     642                  eggPaths.add(l[:i+markerOffset])
     643      finally:
     644          os.remove(tempPath)
     645      # Add any installed eggs to the current working set
     646      zpDists = []
     647      for path in eggPaths:
     648          zpDists += AddDistToWorkingSet(path)
     649      return zpDists
     650
     651
     652  ########################################
     653  #   Zenoss.Net
     654  ########################################
     655
     656
     657 -def FetchAndInstallZenPack(dmd, zenPackName, zenPackVersion='', sendEvent=True):
     658      """
     659      Fetch the named zenpack and all its dependencies and install them.
     660      Return a list of the ZenPacks that were installed.
     661      """
     662      zenPacks = []
     663      try:
     664          zpDists = FetchZenPack(zenPackName, zenPackVersion)
     665          for d in zpDists:
     666              zenPacks.append(InstallDistAsZenPack(dmd, d))
     667      except:
     668          if sendEvent:
     669              ZPEvent(dmd, 4, 'Failed to install ZenPack %s' % zenPackName,
     670                  '%s: %s' % sys.exc_info()[:2])
     671          raise
     672      if sendEvent:
     673          zenPackIds = [z.id for z in zenPacks]
     674          if zenPackIds:
     675              ZPEvent(dmd, 2, 'Installed ZenPacks: %s' % ', '.join(zenPackIds))
     676          if zenPackName not in zenPackIds:
     677              ZPEvent(dmd, 4, 'Unable to install ZenPack %s' % zenPackName)
     678      return zenPacks
     679
     680
     681 -def FetchZenPack(zenPackName, zenPackVersion=''):
     682      """
     683      Use easy_install to retrieve the given zenpack and any dependencies.
     684      easy_install will install the eggs, but does not install them into
     685      Zenoss as ZenPacks.
     686      Return a list of distributions just installed that appear to be
     687      ZenPacks.
     688
     689      NB: This should be refactored.  It shares most of its code with
     690      DoEasyInstall()
     691      """
     692      from setuptools.command import easy_install
     693
     694      # Make sure $ZENHOME/ZenPacks exists
     695      CreateZenPacksDir()
     696
     697      # Create temp file for easy_install to write results to
     698      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     699      # eggPaths is a set of paths to eggs that were installed.  We need to
     700      # add them to the current workingset so we can discover their
     701      # entry points.
     702      eggPaths = set()
     703      try:
     704          # Execute the easy_install
     705          args = ['--site-dirs', zenPath('ZenPacks'),
     706              '-d', zenPath('ZenPacks'),
     707              '-i', ZEN_PACK_INDEX_URL,
     708              '--allow-hosts', 'None',
     709              '--record', tempPath,
     710              '--quiet',
     711              zenPackName]
     712          easy_install.main(args)
     713          # Collect the paths for eggs that were installed
     714          f = open(tempPath, 'r')
     715          marker = '.egg/'
     716          markerOffset = len(marker)-1
     717          for l in f.readlines():
     718              i = l.find(marker)
     719              if i > 0:
     720                  eggPaths.add(l[:i+markerOffset])
     721      finally:
     722          os.remove(tempPath)
     723      # Add any installed eggs to the current working set
     724      zpDists = []
     725      for path in eggPaths:
     726          zpDists += AddDistToWorkingSet(path)
     727      return zpDists
     728
     729
     730 -def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):
     731      """
     732      Upload the specified zenpack to the given project.
     733      Project is a string of the form 'enterprise/myproject' or
     734      'community/otherproject'.
     735      """
     736      zp = dmd.ZenPackManager.packs._getOb(packName, None)
     737      if not zp:
     738          raise ZenPackException('No ZenPack named %s' % packName)
     739
     740      # Export the zenpack
     741      fileName = zp.manage_exportPack()
     742      filePath = zenPath('export', fileName)
     743
     744      # Login to Zenoss.net
     745      from DotNetCommunication import DotNetSession
     746      session = DotNetSession()
     747      userSettings = dmd.ZenUsers.getUserSettings()
     748      session.login(znetUser, znetPass)
     749
     750      # Upload
     751      zpFile = open(zenPath('export', fileName), 'r')
     752      try:
     753          response = session.open('%s/createRelease' % project.strip('/'), {
     754                  'description': description,
     755                  'fileStorage': zpFile,
     756                  })
     757      finally:
     758          zpFile.close()
     759      if response:
     760          result = response.read()
     761          if "'success':true" not in result:
     762              raise ZenPackException('Upload failed')
     763      else:
     764          raise ZenPackException('Failed to connect to Zenoss.net')
     765      return
     766
     767
     768  ########################################
     769  #   ZenPack Removal
     770  ########################################
     771
     772
     773 -def RemoveZenPack(dmd, packName, filesOnly=False, skipDepsCheck=False,
     774                      leaveObjects=False, sendEvent=True,
     775                      forceNoFileDeletion=False, uninstallEgg=True):
     776      """
     777      Remove the given ZenPack from Zenoss.
     778      Whether the ZenPack will be removed from the filesystem or not
     779      depends on the result of the ZenPack's shouldDeleteFilesOnRemoval method.
     780      """
     781      try:
     782          if filesOnly:
     783              skipDepsCheck = True
     784
     785          # Check for dependency implications here?
     786          if not skipDepsCheck:
     787              deps = GetDependents(dmd, packName)
     788              if deps:
     789                  raise ZenPackDependentsException('%s cannot be removed ' % packName +
     790                          'because it is required by %s' % ', '.join(deps))
     791
     792          if not filesOnly:
     793              # Fetch the zenpack, call its remove() and remove from packs
     794              zp = None
     795              try:
     796                  zp = dmd.ZenPackManager.packs._getOb(packName)
     797              except AttributeError, ex:
     798                  raise ZenPackNotFoundException('No ZenPack named %s is installed' %
     799                                                  packName)
     800              # If zencatalog hasn't finished yet, we get ugly messages that don't
     801              # mean anything. Hide them.
     802              logFilter = None
     803              if not getattr(dmd.zport, '_zencatalog_completed', False):
     804                  logFilter = CatalogLoggingFilter()
     805                  logging.getLogger('Zope.ZCatalog').addFilter(logFilter)
     806              try:
     807                  zp.remove(dmd, leaveObjects)
     808                  dmd.ZenPackManager.packs._delObject(packName)
     809                  transaction.commit()
     810              finally:
     811                  # Remove our logging filter so we don't hide anything important
     812                  if logFilter is not None:
     813                      logging.getLogger('Zope.ZCatalog').removeFilter(logFilter)
     814
     815          # Uninstall the egg and possibly delete it
     816          # If we can't find the distribution then apparently the zp egg itself is
     817          # missing.  Continue on with the removal and swallow the
     818          # DistributionNotFound exception
     819          try:
     820              dist = zp.getDistribution()
     821          except pkg_resources.DistributionNotFound:
     822              dist = None
     823          if dist:
     824              # Determine deleteFiles before develop -u gets called.  Once
     825              # it is called the egg has problems figuring out some of it's state.
     826              deleteFiles = zp.shouldDeleteFilesOnRemoval()
     827              if uninstallEgg:
     828                  if zp.isDevelopment():
     829                      zenPackDir = zenPath('ZenPacks')
     830                      cmd = ('%s setup.py develop -u '
     831                              % binPath('python') +
     832                              '--site-dirs=%s ' % zenPackDir +
     833                              '-d %s' % zenPackDir)
     834                      p = subprocess.Popen(cmd,
     835                                          stdout=subprocess.PIPE,
     836                                          stderr=subprocess.PIPE,
     837                                          shell=True,
     838                                          cwd=zp.eggPath())
     839                      out, err = p.communicate()
     840                      code = p.wait()
     841                      if code:
     842                          raise ZenPackException(err)
     843                  else:
     844                      DoEasyUninstall(packName)
     845              # elif cleanupEasyInstallPth:
     846              #     # Do we need to call easy_install -m here?  It causes problems
     847              #     # because it tries to install deps.  Cleanup easy-install.pth
     848              #     # ourselves instead.
     849              #     # We don't want to cleanup easy-install.pth when a newer
     850              #     # version of the egg has already been installed (when doing
     851              #     # an upgrade or installing in new location.)
     852              #     eggLink = './%s' % zp.eggName()
     853              #     CleanupEasyInstallPth(eggLink)
     854              if deleteFiles and not forceNoFileDeletion:
     855                  eggDir = zp.eggPath()
     856                  if os.path.islink(eggDir):
     857                      os.remove(eggDir)
     858                  else:
     859                      shutil.rmtree(eggDir)
     860          cleanupSkins(dmd)
     861          transaction.commit()
     862      except:
     863          if sendEvent:
     864              ZPEvent(dmd, 4, 'Error removing ZenPack %s' % packName,
     865                  '%s: %s' % sys.exc_info()[:2])
     866          raise
     867      if sendEvent:
     868          ZPEvent(dmd, 2, 'Removed ZenPack %s' % packName)
     869
     870
     871 -def DoEasyUninstall(name):
     872      """
     873      Execute the easy_install command to unlink the given egg.
     874      What this is really doing is switching the egg to be in
     875      multiple-version mode, however this is the first step in deleting
     876      an egg as described here:
     877      http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages
     878      """
     879      from setuptools.command import easy_install
     880      args = ['--site-dirs', zenPath('ZenPacks'),
     881          '-d', zenPath('ZenPacks'),
     882          #'--record', tempPath,
     883          '--quiet',
     884          '-m',
     885          name]
     886      easy_install.main(args)
     887
     888
     889 -def CanRemoveZenPacks(dmd, packNames):
     890      """
     891      Returns a tuple of (canRemove, otherDependents)
     892      canRemove is True if the listed zenPacks have no dependents not also
     893      listed in packNames, False otherwise.
     894      otherDependents is a list of zenpack names not in packNames that
     895      depend on one or more of the packs in packNames.
     896      """
     897      unhappy = set()
     898      for name in packNames:
     899          deps = GetDependents(dmd, name)
     900          unhappy.update(set(dep for dep in deps if dep not in packNames))
     901      return (not unhappy, list(unhappy))
     902
     903
     904  # def CleanupEasyInstallPth(eggLink):
     905  #     """
     906  #     Remove the entry for the given egg from the
     907  #     $ZENHOME/ZenPacks/easy-install.pth file.  If this entry is left
     908  #     in place in can cause problems during the next install of the same
     909  #     egg.
     910  #     """
     911  #     # Remove the path from easy-install.pth
     912  #     eggTail = os.path.split(eggLink)[1]
     913  #     easyPth = zenPath('ZenPacks', 'easy-install.pth')
     914  #     if os.path.isfile(easyPth):
     915  #         needToWrite = False
     916  #         newLines = []
     917  #         f = open(easyPth, 'r')
     918  #         for line in f:
     919  #             if os.path.split(line.strip())[1] == eggTail:
     920  #                 needToWrite = True
     921  #             else:
     922  #                 newLines.append(line)
     923  #         f.close()
     924  #         if needToWrite:
     925  #             f = open(easyPth, 'w')
     926  #             f.writelines(newLines)
     927  #             f.close()
     928
     929
     930 -def GetDependents(dmd, packName):
     931      """
     932      Return a list of installed ZenPack ids that list packName as a dependency
     933      """
     934      return [zp.id for zp in dmd.ZenPackManager.packs()
     935                  if zp.id != packName and packName in zp.dependencies]
     936
     937
     938  ########################################
     939  #   __main__, dispatching, etc
     940  ########################################
     941
     942
     943 -def ZPEvent(dmd, severity, summary, message=None):
     944      """
     945      Send an event to Zenoss.
     946      """
     947      dmd.ZenEventManager.sendEvent(dict(
     948          device=FQDN,
     949          eventClass='/Unknown',
     950          severity=severity,
     951          summary=summary,
     952          message=message))
     953
     954
     955 -class ZenPackCmd(ZenScriptBase):
     956      """
     957      Utilities for creating, installing, removing ZenPacks.
     958
     959      NOTE: Users will probably invoke zenpack from the command line, which
     960      runs zenpack.py rather than this file.  zenpack.py calls functions
     961      in this module when it detects that new-style (egg) ZenPacks are involved.
     962      The plan is that once support for old-style (non-egg) ZenPacks is dropped
     963      zenpack.py can go away and this will take its place.  Until then this
     964      script can be invoked directly via the zenpackcmd script if desired.
     965      Invoking this script directly has the benefit of slightly better
     966      progress/status output to stdout.
     967      """
     968
     969 -    def run(self):
     970          """
     971          Execute the user's request.
     972          """
     973
     974          self.connect()
     975          def PrintInstalled(installed, eggOnly=False):
     976              if installed:
     977                  if eggOnly:
     978                      names = [i['id'] for i in installed]
     979                      what = 'ZenPack egg'
     980                  else:
     981                      names = [i.id for i in installed]
     982                      what = 'ZenPack'
     983                  print('Installed %s%s: %s' % (
     984                          what,
     985                          len(names) != 1 and 's' or '',
     986                          ', '.join(names)))
     987              else:
     988                  print('No ZenPacks installed.')
     989
     990          if not getattr(self.dmd, 'ZenPackManager', None):
     991              raise ZenPackNeedMigrateException('Your Zenoss database appears'
     992                  ' to be out of date. Try running zenmigrate to update.')
     993          if self.options.eggOnly and self.options.eggPath:
     994              zpDists = InstallEgg(self.dmd, self.options.eggPath,
     995                                              link=self.options.link)
     996              PrintInstalled([{'id':d.project_name} for d in zpDists],
     997                              eggOnly=True)
     998          if self.options.eggPath:
     999              installed = InstallEggAndZenPack(
    1000                                  self.dmd, self.options.eggPath,
    1001                                  link=self.options.link,
    1002                                  filesOnly=self.options.filesOnly,
    1003                                  previousVersion= self.options.previousVersion)
    1004              PrintInstalled(installed)
    1005          elif self.options.fetch:
    1006              installed = FetchAndInstallZenPack(self.dmd, self.options.fetch)
    1007              PrintInstalled(installed)
    1008          elif self.options.upload:
    1009              return UploadZenPack(self.dmd, self.options.upload,
    1010                                      self.options.znetProject,
    1011                                      self.options.uploadDesc,
    1012                                      self.options.znetUser,
    1013                                      self.options.znetPass)
    1014          elif self.options.removePackName:
    1015              try:
    1016                  RemoveZenPack(self.dmd, self.options.removePackName)
    1017                  print('Removed ZenPack: %s' % self.options.removePackName)
    1018              except ZenPackNotFoundException, e:
    1019                  sys.stderr.write(str(e) + '\n')
    1020          elif self.options.list:
    1021              self.list()
    1022          else:
    1023              self.parser.print_help()
    1024
    1025
    1026 -    def buildOptions(self):
    1027          self.parser.add_option('--install',
    1028                                 dest='eggPath',
    1029                                 default=None,
    1030                                 help="name of the pack to install")
    1031  #        self.parser.add_option('--fetch',
    1032  #                               dest='fetch',
    1033  #                               default=None,
    1034  #                               help='Name of ZenPack to retrieve from '
    1035  #                                    'Zenoss.net and install.')
    1036  #        self.parser.add_option('--fetch-vers',
    1037  #                               dest='fetchVers',
    1038  #                               default=None,
    1039  #                               help='Use with --fetch to specify a version'
    1040  #                                    ' for the ZenPack to download and install.')
    1041  #        self.parser.add_option('--znet-user',
    1042  #                               dest='znetUser',
    1043  #                               default=None,
    1044  #                               help='Use with --fetch or --upload to specify'
    1045  #                                    ' your Zenoss.net username.')
    1046  #        self.parser.add_option('--znet-pass',
    1047  #                               dest='znetPass',
    1048  #                               default=None,
    1049  #                               help='Use with --fetch or --upload to specify'
    1050  #                                    ' your Zenoss.net password.')
    1051  #        self.parser.add_option('--upload',
    1052  #                               dest='upload',
    1053  #                               default=None,
    1054  #                               help='Name of ZenPack to upload to '
    1055  #                                    'Zenoss.net')
    1056  #        self.parser.add_option('--znet-project',
    1057  #                               dest='znetProject',
    1058  #                               default=None,
    1059  #                               help='Use with --upload to specify'
    1060  #                                    ' which Zenoss.net project to create'
    1061  #                                    ' a release on.')
    1062  #        self.parser.add_option('--upload-desc',
    1063  #                               dest='uploadDesc',
    1064  #                               default=None,
    1065  #                               help='Use with --upload to provide'
    1066  #                                    ' a description for the new release.')
    1067          self.parser.add_option('--link',
    1068                                 dest='link',
    1069                                 action='store_true',
    1070                                 default=False,
    1071                                 help='Install the ZenPack in its current '
    1072                                  'location, do not copy to $ZENHOME/ZenPacks. '
    1073                                  'Also mark ZenPack as editable. '
    1074                                  'This only works with source directories '
    1075                                  'containing setup.py files, not '
    1076                                  'egg files.')
    1077          self.parser.add_option('--remove',
    1078                                 dest='removePackName',
    1079                                 default=None,
    1080                                 help="name of the pack to remove")
    1081          self.parser.add_option('--leave-objects',
    1082                                 dest='leaveObjects',
    1083                                 default=False,
    1084                                 action='store_true',
    1085                                 help="When specified with --remove then objects"
    1086                                      ' provided by the ZenPack and those'
    1087                                      ' depending on the ZenPack are not deleted.'
    1088                                      ' This may result in broken objects in your'
    1089                                      ' database unless the ZenPack is'
    1090                                      ' reinstalled.')
    1091          self.parser.add_option('--files-only',
    1092                                 dest='filesOnly',
    1093                                 action="store_true",
    1094                                 default=False,
    1095                                 help='install onto filesystem but not into '
    1096                                          'zenoss')
    1097          self.parser.add_option('--previousversion',
    1098                                 dest='previousVersion',
    1099                                 default=None,
    1100                                 help="Previous version of the zenpack;"
    1101                                      ' used during upgrades')
    1102          self.parser.prog = "zenpack"
    1103          ZenScriptBase.buildOptions(self)
    1104
    1105
    1106  if __name__ == '__main__':
    1107      try:
    1108          zp = ZenPackCmd()
    1109          zp.run()
    1110      except ZenPackException, e:
    1111          sys.stderr.write('%s\n' % str(e))
    1112          sys.exit(-1)
    1113