Package Products :: Package ZenUtils :: Module ZenPackCmd |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenPackCmd

       1  ##############################################################################
       2  #
       3  # Copyright (C) Zenoss, Inc. 2008, all rights reserved.
       4  #
       5  # This content is made available according to terms specified in
       6  # License.zenoss under the directory where your Zenoss product is installed.
       7  #
       8  ##############################################################################
       9
      10
      11  __doc__ = "Manage ZenPacks"
      12
      13  import Globals
      14  from ZODB.transact import transact
      15  from Products.ZenUtils.ZenScriptBase import ZenScriptBase
      16  from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, getObjByPath
      17
      18  from Products.ZenModel import ZVersion
      19  from Products.ZenModel.ZenPack import ZenPackException, \
      20                                          ZenPackNotFoundException, \
      21                                          ZenPackNeedMigrateException
      22  from Products.ZenModel.ZenPack import ZenPackDependentsException
      23  from Products.ZenModel.ZenPack import ZenPack
      24  from Products.ZenUtils.PkgResources import pkg_resources
      25  from Products.Zuul.utils import CatalogLoggingFilter
      26  import Products.ZenModel.ZenPackLoader as ZPL
      27  import zenpack as oldzenpack
      28  import transaction
      29  import os, sys
      30  import shutil
      31  import string
      32  import tempfile
      33  import subprocess
      34  import socket
      35  import logging
      36  import zExceptions
      37
      38  from urlparse import urlparse
      39
      40
      41  log = logging.getLogger('zen.ZenPackCMD')
      42
      43  #import zenpacksupport
      44
      45  FQDN = socket.getfqdn()
      46
      47  ZENPACKS_BASE_URL = 'http://zenpacks.zenoss.com/pypi'
      48
      49  # All ZenPack eggs have to define exactly one entry point in this group.
      50  ZENPACK_ENTRY_POINT = 'zenoss.zenpacks'
      51
      52  ########################################
      53  #   ZenPack Creation
      54  ########################################
      55
      56 -def CreateZenPack(zpId, prevZenPackName=''):
      57      """
      58      Create the zenpack in the filesystem.
      59      The zenpack is not installed in Zenoss, it is simply created in
      60      the $ZENHOME/ZenPacks directory.  Usually this should be followed
      61      with a "zenpack install" call.
      62      zpId should already be valid, scrubbed value.
      63      prevZenPackName is written to PREV_ZENPACK_NAME in setup.py.
      64      """
      65      parts = zpId.split('.')
      66
      67      # Copy template to $ZENHOME/ZenPacks
      68      srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate')
      69      devDir = zenPath('ZenPacks')
      70      if not os.path.exists(devDir):
      71          os.mkdir(devDir, 0750)
      72      destDir = os.path.join(devDir, zpId)
      73      shutil.copytree(srcDir, destDir, symlinks=False)
      74      os.system('find %s -name .svn | xargs rm -rf' % destDir)
      75
      76      # Write setup.py
      77      packages = []
      78      for i in range(len(parts)):
      79          packages.append('.'.join(parts[:i+1]))
      80      mapping = dict(
      81          NAME = zpId,
      82          VERSION = '1.0.0',
      83          AUTHOR = '',
      84          LICENSE = '',
      85          NAMESPACE_PACKAGES = packages[:-1],
      86          PACKAGES = packages,
      87          INSTALL_REQUIRES = [],
      88          COMPAT_ZENOSS_VERS = '',
      89          PREV_ZENPACK_NAME = prevZenPackName,
      90          )
      91      WriteSetup(os.path.join(destDir, 'setup.py'), mapping)
      92
      93      # Create subdirectories
      94      base = destDir
      95      for part in parts[:-1]:
      96          base = os.path.join(base, part)
      97          os.mkdir(base)
      98          f = open(os.path.join(base, '__init__.py'), 'w')
      99          f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
     100          f.close()
     101      base = os.path.join(base, parts[-1])
     102      shutil.move(os.path.join(destDir, 'CONTENT'), base)
     103
     104      return destDir
     105
     106
     107 -def WriteSetup(setupPath, values):
     108      """
     109      """
     110      f = file(setupPath, 'r')
     111      lines = f.readlines()
     112      f.close()
     113
     114      newLines = []
     115      for i, line in enumerate(lines):
     116          if line.startswith('STOP_REPLACEMENTS'):
     117              newLines += lines[i:]
     118              break
     119          key = line.split('=')[0].strip()
     120          if key in values:
     121              value = values[key]
     122              if isinstance(value, basestring):
     123                  fmt = '%s = "%s"\n'
     124              else:
     125                  fmt = '%s = %s\n'
     126              newLines.append(fmt % (key, value))
     127          else:
     128              newLines.append(line)
     129
     130      f = file(setupPath, 'w')
     131      f.writelines(newLines)
     132      f.close()
     133
     134
     135 -def CanCreateZenPack(dmd, zpId):
     136      """
     137      Return tuple (bool, string) where first element is true if a new zenpack
     138      can be created with the given info and false if not.  If first element
     139      is True then the second part of the tuple contains the scrubbed ZenPack id.
     140      If the first part is False then the second contains an explanatory
     141      message.
     142      """
     143      # Check if id and package looks reasonable
     144      (allowable, idOrMsg) = ScrubZenPackId(zpId)
     145      if allowable:
     146          zpId = idOrMsg
     147      else:
     148          return (False, idOrMsg)
     149
     150      # Is the id already in use?
     151      if dmd:
     152          if zpId in dmd.ZenPackManager.packs.objectIds():
     153              return (False, 'A ZenPack named %s already exists.' % zpId)
     154
     155      # Is there another zenpack in the way?
     156      # Now that zenpacks are created in $ZENHOME/ZenPacks instead of
     157      # $ZENHOME/ZenPackDev this may no longer be necessary because a
     158      # zp in the way should be installed and caught by the already in use
     159      # check above.
     160      if os.path.exists(zenPath('ZenPacks', zpId)):
     161          return (False, 'A directory named %s already exists' % zpId +
     162                          ' in $ZENHOME/ZenPacks.  Use a different name'
     163                          ' or remove that directory.')
     164
     165      return (True, idOrMsg)
     166
     167
     168 -def ScrubZenPackId(name):
     169      """
     170      If the given name conforms to ZenPack naming rules, or can easily be
     171      modified to do so, then return (True, scrubbedName) where scrubbedName
     172      is either name or a slightly modified name.  If the given name does
     173      not conform to naming rules and we can't easily modify it to do so
     174      then return (False, errorMsg) where errorMsg describes why name
     175      is unacceptable.
     176      """
     177      parts = name.split('.')
     178
     179      # Remove leading dots, trailing dots, adjacent dots and strip whitespace
     180      # from each part
     181      parts = [p.strip() for p in parts]
     182      parts = [p for p in parts if p]
     183
     184      # Add/fix leading 'ZenPacks'
     185      if parts[0] != 'ZenPacks':
     186          if parts[0].lower() == 'zenpacks':
     187              parts[0] = 'ZenPacks'
     188          else:
     189              parts.insert(0, 'ZenPacks')
     190
     191      # Must be at least 3 parts
     192      if len(parts) < 3:
     193          return (False, 'ZenPack names must contain at least three package '
     194                  'names separated by periods.')
     195
     196      # Each part must start with a letter
     197      for p in parts:
     198          if p[0] not in string.letters:
     199              return (False, 'Each package name must start with a letter.')
     200
     201      # Only letters, numbers and underscores in each part
     202      allowable = string.letters + string.digits + '_'
     203      for p in parts:
     204          for c in p:
     205              if c not in allowable:
     206                  return (False, 'Package names may only contain letters, '
     207                          'numbers and underscores.')
     208
     209      return (True, '.'.join(parts))
     210
     211
     212  ########################################
     213  #   ZenPack Installation
     214  ########################################
     215
     216 -class NonCriticalInstallError(Exception):
     217 -    def __init__(self, message):
     218          Exception.__init__(self, message)
     219          self.message = message
     220
     221 -def InstallEggAndZenPack(dmd, eggPath, link=False,
     222                              filesOnly=False, sendEvent=True,
     223                              previousVersion=None, forceRunExternal=False,
     224                              fromUI=False):
     225      """
     226      Installs the given egg, instantiates the ZenPack, installs in
     227      dmd.ZenPackManager.packs, and runs the zenpacks's install method.
     228      Returns a list of ZenPacks that were installed.
     229      """
     230      zenPacks = []
     231      nonCriticalErrorEncountered = False
     232      try:
     233          zpDists = InstallEgg(dmd, eggPath, link=link)
     234          for d in zpDists:
     235              try:
     236                  zp = InstallDistAsZenPack(dmd,
     237                                            d,
     238                                            eggPath,
     239                                            link,
     240                                            filesOnly=filesOnly,
     241                                            previousVersion=previousVersion,
     242                                            forceRunExternal=forceRunExternal,
     243                                            fromUI=fromUI)
     244                  zenPacks.append(zp)
     245              except NonCriticalInstallError, ex:
     246                  nonCriticalErrorEncountered = True
     247                  if sendEvent:
     248                      ZPEvent(dmd, 3, ex.message)
     249      except Exception, e:
     250          # Get that exception out there in case it gets blown away by ZPEvent
     251          log.exception("Error installing ZenPack %s" % eggPath)
     252          if sendEvent:
     253              ZPEvent(dmd, 4, 'Error installing ZenPack %s' % eggPath,
     254                  '%s: %s' % sys.exc_info()[:2])
     255          # Don't just raise, because if ZPEvent blew away exception context
     256          # it'll be None, which is bad. This manipulates the stack to look like
     257          # this is the source of the exception, but we logged it above so no
     258          # info is lost.
     259          raise e
     260      if sendEvent:
     261          zenPackIds = [zp.id for zp in zenPacks]
     262          if zenPackIds:
     263              ZPEvent(dmd, 2, 'Installed ZenPacks %s' % ','.join(zenPackIds))
     264          elif not nonCriticalErrorEncountered:
     265              ZPEvent(dmd, 4, 'Unable to install %s' % eggPath)
     266      return zenPacks
     267
     268
     269 -def InstallEgg(dmd, eggPath, link=False):
     270      """
     271      Install the given egg and add to the current working set.
     272      This does not install the egg as a ZenPack.
     273      Return a list of distributions that should be installed as ZenPacks.
     274      """
     275      eggPath = os.path.abspath(eggPath)
     276      zenPackDir = zenPath('ZenPacks')
     277      eggInZenPacksDir = eggPath.startswith(zenPackDir + '/')
     278
     279      # Make sure $ZENHOME/ZenPacks exists
     280      CreateZenPacksDir()
     281
     282      # Install the egg
     283      if link:
     284          cmd = ('%s setup.py develop ' % binPath('python') +
     285                  '--site-dirs=%s ' % zenPackDir +
     286                  '-d %s' % zenPackDir)
     287          p = subprocess.Popen(cmd,
     288                              stdout=subprocess.PIPE,
     289                              stderr=subprocess.PIPE,
     290                              shell=True,
     291                              cwd=eggPath)
     292          out, err = p.communicate()
     293          p.wait()
     294          if p.returncode:
     295              DoEasyUninstall(eggPath)
     296              raise ZenPackException('Error installing the egg (%s): %s' %
     297                                  (p.returncode, err))
     298          zpDists = AddDistToWorkingSet(eggPath)
     299      else:
     300          try:
     301              zpDists = DoEasyInstall(eggPath)
     302          except:
     303              DoEasyUninstall(eggPath)
     304              raise
     305          # cmd = 'easy_install --always-unzip --site-dirs=%s -d %s %s' % (
     306          #             zenPackDir,
     307          #             zenPackDir,
     308          #             eggPath)
     309          # p = subprocess.Popen(cmd,
     310          #                     stdout=subprocess.PIPE,
     311          #                     stderr=subprocess.PIPE,
     312          #                     shell=True)
     313          # p.wait()
     314          # eggName = os.path.split(eggPath)[1]
     315          # eggPath = os.path.join(zenPackDir, eggName)
     316      return zpDists
     317
     318
     319  # def GetZenPackNamesFromEggPath(eggPath):
     320  #     """
     321  #     Given a path to a ZenPack egg (installed or not) return the
     322  #     name of the ZenPack it contains.
     323  #     """
     324  #     zpNames = []
     325  #     for d in pkg_resources.find_distributions(eggPath)
     326  #         if d.project_name.startswith('ZenPacks.'):
     327  #             zpNames.append(d.project_name)
     328  #     return zpNames
     329
     330
     331 -def InstallDistAsZenPack(dmd, dist, eggPath, link=False, filesOnly=False,
     332                           previousVersion=None, forceRunExternal=False,
     333                           fromUI=False):
     334      """
     335      Given an installed dist, install it into Zenoss as a ZenPack.
     336      Return the ZenPack instance.
     337      """
     338      @transact
     339      def transactional_actions():
     340          # Instantiate ZenPack
     341          entryMap = pkg_resources.get_entry_map(dist, ZENPACK_ENTRY_POINT)
     342          if not entryMap or len(entryMap) > 1:
     343              raise ZenPackException('A ZenPack egg must contain exactly one'
     344                      ' zenoss.zenpacks entry point.  This egg appears to contain'
     345                      ' %s such entry points.' % len(entryMap))
     346          packName, packEntry = entryMap.items()[0]
     347          runExternalZenpack = True
     348          #if zenpack with same name exists we can't load both modules
     349          #installing new egg zenpack will be done in a sub process
     350          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     351          if existing:
     352              log.info("Previous ZenPack exists with same name %s" % packName)
     353          if filesOnly or not existing:
     354              #running files only or zenpack by same name doesn't already exists
     355              # so no need to install the zenpack in an external process
     356              runExternalZenpack = False
     357              module = packEntry.load()
     358              if hasattr(module, 'ZenPack'):
     359                  zenPack = module.ZenPack(packName)
     360              else:
     361                  zenPack = ZenPack(packName)
     362              zenPack.eggPack = True
     363              CopyMetaDataToZenPackObject(dist, zenPack)
     364              if filesOnly:
     365                  for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
     366                      loader.load(zenPack, None)
     367              if fromUI and not zenPack.installableFromUI:
     368                  raise ZenPackException("This ZenPack cannot be installed through the UI.")
     369
     370
     371          if not filesOnly:
     372              # Look for an installed ZenPack to be upgraded.  In this case
     373              # upgraded means that it is removed before the new one is installed
     374              # but that its objects are not removed and the packables are
     375              # copied to the new instance.
     376              existing = dmd.ZenPackManager.packs._getOb(packName, None)
     377              if not existing and zenPack.prevZenPackName:
     378                  existing = dmd.ZenPackManager.packs._getOb(
     379                                      zenPack.prevZenPackName, None)
     380
     381              deferFileDeletion = False
     382              packables = []
     383              upgradingFrom = None
     384              if existing:
     385                  upgradingFrom = existing.version
     386                  for p in existing.packables():
     387                      packables.append(p)
     388                      existing.packables.removeRelation(p)
     389                  if existing.isEggPack():
     390                      forceNoFileDeletion = existing.eggPath() == dist.location
     391                      RemoveZenPack(dmd, existing.id,
     392                                      skipDepsCheck=True, leaveObjects=True,
     393                                      forceNoFileDeletion=forceNoFileDeletion,
     394                                      uninstallEgg=False)
     395                  else:
     396                      # Don't delete files, might still be needed for
     397                      # migrate scripts to be run below.
     398                      deferFileDeletion = True
     399                      oldzenpack.RemoveZenPack(dmd, existing.id,
     400                                      skipDepsCheck=True, leaveObjects=True,
     401                                      deleteFiles=False)
     402              if runExternalZenpack or forceRunExternal:
     403                  log.info("installing zenpack %s; launching process" % packName)
     404                  cmd = [binPath('zenpack')]
     405                  if link:
     406                      cmd += ["--link"]
     407                  cmd += ["--install", eggPath]
     408                  if upgradingFrom:
     409                      cmd += ['--previousversion', upgradingFrom]
     410                  if fromUI:
     411                      cmd += ["--fromui"]
     412
     413                  cmdStr = " ".join(cmd)
     414                  log.debug("launching sub process command: %s" % cmdStr)
     415                  p = subprocess.Popen(cmdStr,
     416                                  shell=True)
     417                  out, err = p.communicate()
     418                  p.wait()
     419                  if p.returncode:
     420                      raise ZenPackException('Error installing the egg (%s): %s' %
     421                                             (p.returncode, err))
     422                  dmd._p_jar.sync()
     423              else:
     424                  dmd.ZenPackManager.packs._setObject(packName, zenPack)
     425                  zenPack = dmd.ZenPackManager.packs._getOb(packName)
     426                  #hack because ZenPack.install is overridden by a lot of zenpacks
     427                  #so we can't change the signature of install to take the
     428                  #previousVerison
     429                  zenPack.prevZenPackVersion = previousVersion
     430                  zenPack.install(dmd)
     431                  zenPack.prevZenPackVersion = None
     432
     433              try:
     434                  zenPack = dmd.ZenPackManager.packs._getOb(packName)
     435                  for p in packables:
     436                      pId = p.getPrimaryId()
     437                      try:
     438                          # make sure packable still exists; could be deleted by a
     439                          # migrate
     440                          getObjByPath(dmd, pId)
     441                          log.debug("adding packable relation for id %s", pId)
     442                          zenPack.packables.addRelation(p)
     443                      except (KeyError, zExceptions.NotFound):
     444                          log.debug('did not find packable %s',pId)
     445              except AttributeError, e:
     446                  # If this happens in the child process or during the non-upgrade
     447                  # flow, reraise the exception
     448                  if not runExternalZenpack:
     449                      raise
     450
     451                  # This specific error will occur when the version of the ZenPack
     452                  # being installed subclasses Products.ZenModel.ZenPack, but the
     453                  # previous version of the ZenPack did not.
     454                  if str(e) == "'ZenPack' object has no attribute '__of__'":
     455                      zenPack = ZenPack(packName)
     456                  else:
     457                      # This is the signature error of class-loading issues
     458                      # during zenpack upgrade.  The final state should be okay,
     459                      # except that modified packables may be lost.
     460                      message = "There has been an error during the post-" + \
     461                                "installation steps for the zenpack %s.  In " + \
     462                                "most cases, no further action is required.  If " + \
     463                                "issues persist, please reinstall this zenpack."
     464                      message = message % packName
     465                      log.warning( message )
     466                      raise NonCriticalInstallError( message )
     467
     468              cleanupSkins(dmd)
     469              return zenPack, deferFileDeletion, existing
     470
     471      zenPack, deferFileDeletion, existing = transactional_actions()
     472
     473      if not filesOnly and deferFileDeletion:
     474          # We skipped deleting the existing files from filesystem
     475          # because maybe they'd be needed in migrate scripts.
     476          # Delete them now
     477          oldZpDir = zenPath('Products', existing.id)
     478          if os.path.islink(oldZpDir):
     479              os.remove(oldZpDir)
     480          else:
     481              shutil.rmtree(oldZpDir)
     482
     483      return zenPack
     484
     485
     486 -def DiscoverEggs(dmd, zenPackId):
     487      """
     488      Find installed eggs that provide a zenoss.zenpacks entry point.
     489      Return a list of distributions whose ZenPacks need to be installed
     490      or upgraded.  The list is sorted into the order in which this needs to
     491      happen.
     492      """
     493      # Create a set of all available zenoss.zenpack entries that aren't
     494      # already installed in zenoss or need to be upgraded in zenoss.
     495      entries = set()
     496      parse_version = pkg_resources.parse_version
     497      for entry in pkg_resources.iter_entry_points(ZENPACK_ENTRY_POINT):
     498          packName = entry.name
     499          packVers = entry.dist.version
     500          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     501          if existing and existing.isEggPack():
     502              # We use >= to allow migrate to be run on currently installed
     503              # zenpacks whose version has been changed or for whom new
     504              # migrates have been added.
     505              if parse_version(packVers) >= parse_version(existing.version):
     506                  entries.add(entry)
     507          else:
     508              entries.add(entry)
     509
     510      # Starting with the entry representing zenPackId create a list of
     511      # all entrypoints
     512
     513      # orderedEntries lists entries in the opposite order of that in which
     514      # they need to be installed.  This is simply for convenience of using
     515      # .append() in code below.
     516      orderedEntries = []
     517      entriesByName = dict((e.name, e) for e in entries)
     518
     519      def AddEntryAndProcessDeps(e):
     520          orderedEntries.append(e)
     521          for name in [r.project_name for r in e.dist.requires()]:
     522              if name in [e.name for e in orderedEntries]:
     523                  # This entry depends on something that we've already processed.
     524                  # This might be a circular dependency, might not be.
     525                  # We are just going to bail however.  This should be
     526                  # very unusual and the user can install deps first to work
     527                  # around.
     528                  raise ZenPackException('Unable to resolve ZenPack dependencies.'
     529                      ' Try installing dependencies first.')
     530              if name in entriesByName:
     531                  # The requirement is an entry that has not yet been processed
     532                  # here.  Add it to the list of entries to install/upgrade.
     533                  AddEntryAndProcessDeps(entriesByName[name])
     534              else:
     535                  # The requirement is not in the entries generated above.
     536                  # This either means that the dep is already installed (this
     537                  # is likely) or that easy_install missed something and the dep
     538                  # is not installed/available (this should be unlikely.)
     539                  pass
     540
     541      if zenPackId not in entriesByName:
     542          if zenPackId in dmd.ZenPackManager.packs.objectIds():
     543              return []
     544          else:
     545              raise ZenPackException('Unable to discover ZenPack named %s' %
     546                                      zenPackId)
     547      AddEntryAndProcessDeps(entriesByName[zenPackId])
     548      orderedEntries.reverse()
     549      return [e.dist for e in orderedEntries]
     550
     551
     552 -def AddDistToWorkingSet(distPath):
     553      """
     554      Given the path to a dist (an egg) add it to the current working set.
     555      This is basically a pkg_resources-friendly way of adding it to
     556      sys.path.
     557      Return a list of all distributions on distPath that appear to
     558      be ZenPacks.
     559      """
     560      zpDists = []
     561      for d in pkg_resources.find_distributions(distPath):
     562          pkg_resources.working_set.add(d)
     563          pkg_resources.require(d.project_name)
     564          if d.project_name.startswith('ZenPacks.'):
     565              zpDists.append(d)
     566      return zpDists
     567
     568
     569 -def ReadZenPackInfo(dist):
     570      """
     571      Return a dictionary containing the egg metadata
     572      """
     573      info = {}
     574      if dist.has_metadata('PKG-INFO'):
     575          lines = dist.get_metadata('PKG-INFO')
     576          for line in pkg_resources.yield_lines(lines):
     577              key, value = line.split(':', 1)
     578              info[key.strip()] = value.strip()
     579      if dist.has_metadata('zenpack_info'):
     580          lines = dist.get_metadata('zenpack_info')
     581          for line in pkg_resources.yield_lines(lines):
     582              key, value = line.split(':', 1)
     583              info[key.strip()] = value.strip()
     584      return info
     585
     586
     587 -def CopyMetaDataToZenPackObject(dist, pack):
     588      """
     589      Copy metadata type stuff from the distribution to the zp object.
     590      """
     591      # Version
     592      pack.version = dist.version
     593
     594      # Egg Info
     595      info = ReadZenPackInfo(dist)
     596      pack.author = info.get('Author', '')
     597      if pack.author == 'UNKNOWN':
     598          pack.author = ''
     599
     600      pack.license = info.get('License', '')
     601      if pack.license == 'UNKNOWN':
     602          pack.license = ''
     603
     604      pack.compatZenossVers = info.get('compatZenossVers', '')
     605      pack.prevZenPackName = info.get('prevZenPackName', '')
     606
     607      # Requires
     608      pack.dependencies = {}
     609      for r in dist.requires():
     610          name = r.project_name
     611          spec = str(r)[len(name):]
     612          pack.dependencies[name] = spec
     613
     614
     615 -def CreateZenPacksDir():
     616      """
     617      Make sure $ZENHOME/ZenPacks exists
     618      """
     619      zpDir = zenPath('ZenPacks')
     620      if not os.path.isdir(zpDir):
     621          os.mkdir(zpDir, 0750)
     622
     623
     624 -def DoEasyInstall(eggPath):
     625      """
     626      Use easy_install to install an egg from the filesystem.
     627      easy_install will install the egg, but does not install it into
     628      Zenoss as ZenPacks.
     629      Returns a list of distributions that were installed that appear
     630      to be ZenPacks.
     631      """
     632      from setuptools.command import easy_install
     633
     634      # Make sure $ZENHOME/ZenPacks exists
     635      CreateZenPacksDir()
     636
     637      # Create temp file for easy_install to write results to
     638      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     639      # eggPaths is a set of paths to eggs that were installed.  We need to
     640      # add them to the current workingset so we can discover their
     641      # entry points.
     642      eggPaths = set()
     643      try:
     644          # Execute the easy_install
     645          args = ['--site-dirs', zenPath('ZenPacks'),
     646              '-d', zenPath('ZenPacks'),
     647              '--allow-hosts', 'None',
     648              '--record', tempPath,
     649              '--quiet',
     650              eggPath]
     651          easy_install.main(args)
     652          # Collect the paths for eggs that were installed
     653          f = open(tempPath, 'r')
     654          marker = '.egg/'
     655          markerOffset = len(marker)-1
     656          for l in f.readlines():
     657              i = l.find(marker)
     658              if i > 0:
     659                  eggPaths.add(l[:i+markerOffset])
     660      finally:
     661          os.remove(tempPath)
     662      # Add any installed eggs to the current working set
     663      zpDists = []
     664      for path in eggPaths:
     665          zpDists += AddDistToWorkingSet(path)
     666      return zpDists
     667
     668
     669  ########################################
     670  #   Zenoss.Net
     671  ########################################
     672
     673
     674 -def FetchAndInstallZenPack(dmd, zenPackName, sendEvent=True):
     675      """
     676      Fetch the named zenpack and all its dependencies and install them.
     677      Return a list of the ZenPacks that were installed.
     678      """
     679      zenPacks = []
     680      try:
     681          zpDists = FetchZenPack(dmd, zenPackName)
     682          for d in zpDists:
     683              zenPacks.append(InstallDistAsZenPack(dmd, d, d.location))
     684      except Exception, ex:
     685          log.exception("Error fetching ZenPack %s" % zenPackName)
     686          if sendEvent:
     687              ZPEvent(dmd, 4, 'Failed to install ZenPack %s' % zenPackName,
     688                  '%s: %s' % sys.exc_info()[:2])
     689
     690          raise ex
     691      if sendEvent:
     692          zenPackIds = [z.id for z in zenPacks]
     693          if zenPackIds:
     694              ZPEvent(dmd, 2, 'Installed ZenPacks: %s' % ', '.join(zenPackIds))
     695          if zenPackName not in zenPackIds:
     696              ZPEvent(dmd, 4, 'Unable to install ZenPack %s' % zenPackName)
     697      return zenPacks
     698
     699
     700 -def FetchZenPack(dmd, zenPackName):
     701      """
     702      Use easy_install to retrieve the given zenpack and any dependencies.
     703      easy_install will install the eggs, but does not install them into
     704      Zenoss as ZenPacks.
     705      Return a list of distributions just installed that appear to be
     706      ZenPacks.
     707
     708      NB: This should be refactored.  It shares most of its code with
     709      DoEasyInstall()
     710      """
     711      from setuptools.command import easy_install
     712
     713      # Make sure $ZENHOME/ZenPacks exists
     714      CreateZenPacksDir()
     715
     716      # Create the proper package index URL.
     717      index_url = '%s/%s/%s/' % (
     718          ZENPACKS_BASE_URL, dmd.uuid, ZVersion.VERSION)
     719
     720      # Create temp file for easy_install to write results to
     721      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     722      # eggPaths is a set of paths to eggs that were installed.  We need to
     723      # add them to the current workingset so we can discover their
     724      # entry points.
     725      eggPaths = set()
     726      try:
     727          # Execute the easy_install
     728          args = [
     729              '--site-dirs', zenPath('ZenPacks'),
     730              '-d', zenPath('ZenPacks'),
     731              '-i', index_url,
     732              '--allow-hosts', urlparse(index_url).hostname,
     733              '--record', tempPath,
     734              '--quiet',
     735              zenPackName]
     736          easy_install.main(args)
     737          # Collect the paths for eggs that were installed
     738          f = open(tempPath, 'r')
     739          marker = '.egg/'
     740          markerOffset = len(marker) - 1
     741          for l in f.readlines():
     742              i = l.find(marker)
     743              if i > 0:
     744                  eggPaths.add(l[:i + markerOffset])
     745      finally:
     746          os.remove(tempPath)
     747      # Add any installed eggs to the current working set
     748      zpDists = []
     749      for path in eggPaths:
     750          zpDists += AddDistToWorkingSet(path)
     751      return zpDists
     752
     753
     754 -def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):
     755      """
     756      Upload the specified zenpack to the given project.
     757      Project is a string of the form 'enterprise/myproject' or
     758      'community/otherproject'.
     759      """
     760      zp = dmd.ZenPackManager.packs._getOb(packName, None)
     761      if not zp:
     762          raise ZenPackException('No ZenPack named %s' % packName)
     763
     764      # Export the zenpack
     765      fileName = zp.manage_exportPack()
     766      filePath = zenPath('export', fileName)
     767
     768      # Login to Zenoss.net
     769      from DotNetCommunication import DotNetSession
     770      session = DotNetSession()
     771      userSettings = dmd.ZenUsers.getUserSettings()
     772      session.login(znetUser, znetPass)
     773
     774      # Upload
     775      zpFile = open(zenPath('export', fileName), 'r')
     776      try:
     777          response = session.open('%s/createRelease' % project.strip('/'), {
     778                  'description': description,
     779                  'fileStorage': zpFile,
     780                  })
     781      finally:
     782          zpFile.close()
     783      if response:
     784          result = response.read()
     785          if "'success':true" not in result:
     786              raise ZenPackException('Upload failed')
     787      else:
     788          raise ZenPackException('Failed to connect to Zenoss.net')
     789      return
     790
     791
     792  ########################################
     793  #   ZenPack Removal
     794  ########################################
     795
     796
     797 -def RemoveZenPack(dmd, packName, filesOnly=False, skipDepsCheck=False,
     798                      leaveObjects=False, sendEvent=True,
     799                      forceNoFileDeletion=False, uninstallEgg=True):
     800      """
     801      Remove the given ZenPack from Zenoss.
     802      Whether the ZenPack will be removed from the filesystem or not
     803      depends on the result of the ZenPack's shouldDeleteFilesOnRemoval method.
     804      """
     805      try:
     806          if filesOnly:
     807              skipDepsCheck = True
     808
     809          # Check for dependency implications here?
     810          if not skipDepsCheck:
     811              deps = GetDependents(dmd, packName)
     812              if deps:
     813                  raise ZenPackDependentsException('%s cannot be removed ' % packName +
     814                          'because it is required by %s' % ', '.join(deps))
     815
     816          if not filesOnly:
     817              # Fetch the zenpack, call its remove() and remove from packs
     818              zp = None
     819              try:
     820                  zp = dmd.ZenPackManager.packs._getOb(packName)
     821              except AttributeError, ex:
     822                  raise ZenPackNotFoundException('No ZenPack named %s is installed' %
     823                                                  packName)
     824              # If zencatalog hasn't finished yet, we get ugly messages that don't
     825              # mean anything. Hide them.
     826              logFilter = None
     827              if not getattr(dmd.zport, '_zencatalog_completed', False):
     828                  logFilter = CatalogLoggingFilter()
     829                  logging.getLogger('Zope.ZCatalog').addFilter(logFilter)
     830              try:
     831                  zp.remove(dmd, leaveObjects)
     832                  dmd.ZenPackManager.packs._delObject(packName)
     833                  transaction.commit()
     834              finally:
     835                  # Remove our logging filter so we don't hide anything important
     836                  if logFilter is not None:
     837                      logging.getLogger('Zope.ZCatalog').removeFilter(logFilter)
     838
     839          # Uninstall the egg and possibly delete it
     840          # If we can't find the distribution then apparently the zp egg itself is
     841          # missing.  Continue on with the removal and swallow the
     842          # DistributionNotFound exception
     843          try:
     844              dist = zp.getDistribution()
     845          except pkg_resources.DistributionNotFound:
     846              dist = None
     847          if dist:
     848              # Determine deleteFiles before develop -u gets called.  Once
     849              # it is called the egg has problems figuring out some of it's state.
     850              deleteFiles = zp.shouldDeleteFilesOnRemoval()
     851              if uninstallEgg:
     852                  if zp.isDevelopment():
     853                      zenPackDir = zenPath('ZenPacks')
     854                      cmd = ('%s setup.py develop -u '
     855                              % binPath('python') +
     856                              '--site-dirs=%s ' % zenPackDir +
     857                              '-d %s' % zenPackDir)
     858                      p = subprocess.Popen(cmd,
     859                                          stdout=subprocess.PIPE,
     860                                          stderr=subprocess.PIPE,
     861                                          shell=True,
     862                                          cwd=zp.eggPath())
     863                      out, err = p.communicate()
     864                      code = p.wait()
     865                      if code:
     866                          raise ZenPackException(err)
     867                  else:
     868                      DoEasyUninstall(packName)
     869              # elif cleanupEasyInstallPth:
     870              #     # Do we need to call easy_install -m here?  It causes problems
     871              #     # because it tries to install deps.  Cleanup easy-install.pth
     872              #     # ourselves instead.
     873              #     # We don't want to cleanup easy-install.pth when a newer
     874              #     # version of the egg has already been installed (when doing
     875              #     # an upgrade or installing in new location.)
     876              #     eggLink = './%s' % zp.eggName()
     877              #     CleanupEasyInstallPth(eggLink)
     878              if deleteFiles and not forceNoFileDeletion:
     879                  eggDir = zp.eggPath()
     880                  if os.path.islink(eggDir):
     881                      os.remove(eggDir)
     882                  else:
     883                      shutil.rmtree(eggDir)
     884          cleanupSkins(dmd)
     885          transaction.commit()
     886      except ZenPackDependentsException, ex:
     887          log.error(ex)
     888      except Exception, ex:
     889          # Get that exception out there in case it gets blown away by ZPEvent
     890          log.exception("Error removing ZenPack %s" % packName)
     891          if sendEvent:
     892              ZPEvent(dmd, 4, 'Error removing ZenPack %s' % packName,
     893                  '%s: %s' % sys.exc_info()[:2])
     894
     895          # Don't just raise, because if ZPEvent blew away exception context
     896          # it'll be None, which is bad. This manipulates the stack to look like
     897          # this is the source of the exception, but we logged it above so no
     898          # info is lost.
     899          raise ex
     900      if sendEvent:
     901          ZPEvent(dmd, 2, 'Removed ZenPack %s' % packName)
     902
     903
     904 -def DoEasyUninstall(name):
     905      """
     906      Execute the easy_install command to unlink the given egg.
     907      What this is really doing is switching the egg to be in
     908      multiple-version mode, however this is the first step in deleting
     909      an egg as described here:
     910      http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages
     911      """
     912      from setuptools.command import easy_install
     913      args = ['--site-dirs', zenPath('ZenPacks'),
     914          '-d', zenPath('ZenPacks'),
     915          #'--record', tempPath,
     916          '--quiet',
     917          '-m',
     918          name]
     919      easy_install.main(args)
     920
     921
     922 -def CanRemoveZenPacks(dmd, packNames):
     923      """
     924      Returns a tuple of (canRemove, otherDependents)
     925      canRemove is True if the listed zenPacks have no dependents not also
     926      listed in packNames, False otherwise.
     927      otherDependents is a list of zenpack names not in packNames that
     928      depend on one or more of the packs in packNames.
     929      """
     930      unhappy = set()
     931      for name in packNames:
     932          deps = GetDependents(dmd, name)
     933          unhappy.update(set(dep for dep in deps if dep not in packNames))
     934      return (not unhappy, list(unhappy))
     935
     936
     937  # def CleanupEasyInstallPth(eggLink):
     938  #     """
     939  #     Remove the entry for the given egg from the
     940  #     $ZENHOME/ZenPacks/easy-install.pth file.  If this entry is left
     941  #     in place in can cause problems during the next install of the same
     942  #     egg.
     943  #     """
     944  #     # Remove the path from easy-install.pth
     945  #     eggTail = os.path.split(eggLink)[1]
     946  #     easyPth = zenPath('ZenPacks', 'easy-install.pth')
     947  #     if os.path.isfile(easyPth):
     948  #         needToWrite = False
     949  #         newLines = []
     950  #         f = open(easyPth, 'r')
     951  #         for line in f:
     952  #             if os.path.split(line.strip())[1] == eggTail:
     953  #                 needToWrite = True
     954  #             else:
     955  #                 newLines.append(line)
     956  #         f.close()
     957  #         if needToWrite:
     958  #             f = open(easyPth, 'w')
     959  #             f.writelines(newLines)
     960  #             f.close()
     961
     962
     963 -def GetDependents(dmd, packName):
     964      """
     965      Return a list of installed ZenPack ids that list packName as a dependency
     966      """
     967      return [zp.id for zp in dmd.ZenPackManager.packs()
     968                  if zp.id != packName and packName in zp.dependencies]
     969
     970
     971  ########################################
     972  #   __main__, dispatching, etc
     973  ########################################
     974
     975
     976 -def ZPEvent(dmd, severity, summary, message=None):
     977      """
     978      Send an event to Zenoss.
     979      """
     980      dmd.ZenEventManager.sendEvent(dict(
     981          device=FQDN,
     982          eventClass='/Unknown',
     983          severity=severity,
     984          summary=summary,
     985          message=message))
     986
     987
     988 -class ZenPackCmd(ZenScriptBase):
     989      """
     990      Utilities for creating, installing, removing ZenPacks.
     991
     992      NOTE: Users will probably invoke zenpack from the command line, which
     993      runs zenpack.py rather than this file.  zenpack.py calls functions
     994      in this module when it detects that new-style (egg) ZenPacks are involved.
     995      The plan is that once support for old-style (non-egg) ZenPacks is dropped
     996      zenpack.py can go away and this will take its place.  Until then this
     997      script can be invoked directly via the zenpackcmd script if desired.
     998      Invoking this script directly has the benefit of slightly better
     999      progress/status output to stdout.
    1000      """
    1001
    1002 -    def run(self):
    1003          """
    1004          Execute the user's request.
    1005          """
    1006
    1007          self.connect()
    1008          def PrintInstalled(installed, eggOnly=False):
    1009              if installed:
    1010                  if eggOnly:
    1011                      names = [i['id'] for i in installed]
    1012                      what = 'ZenPack egg'
    1013                  else:
    1014                      names = [i.id for i in installed]
    1015                      what = 'ZenPack'
    1016                  print('Installed %s%s: %s' % (
    1017                          what,
    1018                          len(names) != 1 and 's' or '',
    1019                          ', '.join(names)))
    1020              else:
    1021                  print('No ZenPacks installed.')
    1022
    1023          if not getattr(self.dmd, 'ZenPackManager', None):
    1024              raise ZenPackNeedMigrateException('Your Zenoss database appears'
    1025                  ' to be out of date. Try running zenmigrate to update.')
    1026          if self.options.eggOnly and self.options.eggPath:
    1027              zpDists = InstallEgg(self.dmd, self.options.eggPath,
    1028                                              link=self.options.link)
    1029              PrintInstalled([{'id':d.project_name} for d in zpDists],
    1030                              eggOnly=True)
    1031          if self.options.eggPath:
    1032              installed = InstallEggAndZenPack(
    1033                                  self.dmd, self.options.eggPath,
    1034                                  link=self.options.link,
    1035                                  filesOnly=self.options.filesOnly,
    1036                                  previousVersion= self.options.previousVersion)
    1037              PrintInstalled(installed)
    1038          elif self.options.fetch:
    1039              installed = FetchAndInstallZenPack(self.dmd, self.options.fetch)
    1040              PrintInstalled(installed)
    1041          elif self.options.upload:
    1042              return UploadZenPack(self.dmd, self.options.upload,
    1043                                      self.options.znetProject,
    1044                                      self.options.uploadDesc,
    1045                                      self.options.znetUser,
    1046                                      self.options.znetPass)
    1047          elif self.options.removePackName:
    1048              try:
    1049                  RemoveZenPack(self.dmd, self.options.removePackName)
    1050                  print('Removed ZenPack: %s' % self.options.removePackName)
    1051              except ZenPackNotFoundException, e:
    1052                  sys.stderr.write(str(e) + '\n')
    1053          elif self.options.list:
    1054              self.list()
    1055          else:
    1056              self.parser.print_help()
    1057
    1058
    1059 -    def buildOptions(self):
    1060          self.parser.add_option('--install',
    1061                                 dest='eggPath',
    1062                                 default=None,
    1063                                 help="name of the pack to install")
    1064  #        self.parser.add_option('--fetch',
    1065  #                               dest='fetch',
    1066  #                               default=None,
    1067  #                               help='Name of ZenPack to retrieve from '
    1068  #                                    'Zenoss.net and install.')
    1069  #        self.parser.add_option('--fetch-vers',
    1070  #                               dest='fetchVers',
    1071  #                               default=None,
    1072  #                               help='Use with --fetch to specify a version'
    1073  #                                    ' for the ZenPack to download and install.')
    1074  #        self.parser.add_option('--znet-user',
    1075  #                               dest='znetUser',
    1076  #                               default=None,
    1077  #                               help='Use with --fetch or --upload to specify'
    1078  #                                    ' your Zenoss.net username.')
    1079  #        self.parser.add_option('--znet-pass',
    1080  #                               dest='znetPass',
    1081  #                               default=None,
    1082  #                               help='Use with --fetch or --upload to specify'
    1083  #                                    ' your Zenoss.net password.')
    1084  #        self.parser.add_option('--upload',
    1085  #                               dest='upload',
    1086  #                               default=None,
    1087  #                               help='Name of ZenPack to upload to '
    1088  #                                    'Zenoss.net')
    1089  #        self.parser.add_option('--znet-project',
    1090  #                               dest='znetProject',
    1091  #                               default=None,
    1092  #                               help='Use with --upload to specify'
    1093  #                                    ' which Zenoss.net project to create'
    1094  #                                    ' a release on.')
    1095  #        self.parser.add_option('--upload-desc',
    1096  #                               dest='uploadDesc',
    1097  #                               default=None,
    1098  #                               help='Use with --upload to provide'
    1099  #                                    ' a description for the new release.')
    1100          self.parser.add_option('--link',
    1101                                 dest='link',
    1102                                 action='store_true',
    1103                                 default=False,
    1104                                 help='Install the ZenPack in its current '
    1105                                  'location, do not copy to $ZENHOME/ZenPacks. '
    1106                                  'Also mark ZenPack as editable. '
    1107                                  'This only works with source directories '
    1108                                  'containing setup.py files, not '
    1109                                  'egg files.')
    1110          self.parser.add_option('--remove',
    1111                                 dest='removePackName',
    1112                                 default=None,
    1113                                 help="name of the pack to remove")
    1114          self.parser.add_option('--leave-objects',
    1115                                 dest='leaveObjects',
    1116                                 default=False,
    1117                                 action='store_true',
    1118                                 help="When specified with --remove then objects"
    1119                                      ' provided by the ZenPack and those'
    1120                                      ' depending on the ZenPack are not deleted.'
    1121                                      ' This may result in broken objects in your'
    1122                                      ' database unless the ZenPack is'
    1123                                      ' reinstalled.')
    1124          self.parser.add_option('--files-only',
    1125                                 dest='filesOnly',
    1126                                 action="store_true",
    1127                                 default=False,
    1128                                 help='install onto filesystem but not into '
    1129                                          'zenoss')
    1130          self.parser.add_option('--previousversion',
    1131                                 dest='previousVersion',
    1132                                 default=None,
    1133                                 help="Previous version of the zenpack;"
    1134                                      ' used during upgrades')
    1135          self.parser.prog = "zenpack"
    1136          ZenScriptBase.buildOptions(self)
    1137
    1138
    1139  if __name__ == '__main__':
    1140      try:
    1141          zp = ZenPackCmd()
    1142          zp.run()
    1143      except ZenPackException, e:
    1144          sys.stderr.write('%s\n' % str(e))
    1145          sys.exit(-1)
    1146