Package Products :: Package ZenUtils :: Module ZenPackCmd |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenPackCmd

       1  ###########################################################################
       2  #
       3  # This program is part of Zenoss Core, an open source monitoring platform.
       4  # Copyright (C) 2008, Zenoss Inc.
       5  #
       6  # This program is free software; you can redistribute it and/or modify it
       7  # under the terms of the GNU General Public License version 2 as published by
       8  # the Free Software Foundation.
       9  #
      10  # For complete information please visit: http://www.zenoss.com/oss/
      11  #
      12  ###########################################################################
      13
      14  __doc__ = "Manage ZenPacks"
      15
      16  import Globals
      17  from Products.ZenUtils.ZenScriptBase import ZenScriptBase
      18  from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, getObjByPath
      19
      20  from Products.ZenModel.ZenPack import ZenPackException, \
      21                                          ZenPackNotFoundException, \
      22                                          ZenPackNeedMigrateException
      23  from Products.ZenModel.ZenPack import ZenPackDependentsException
      24  from Products.ZenModel.ZenPack import ZenPack
      25  from Products.ZenUtils.PkgResources import pkg_resources
      26  import Products.ZenModel.ZenPackLoader as ZPL
      27  import zenpack as oldzenpack
      28  import transaction
      29  import os, sys
      30  import shutil
      31  import string
      32  import tempfile
      33  import subprocess
      34  import socket
      35  import logging
      36  import zExceptions
      37
      38
      39  log = logging.getLogger('zen.ZenPackCMD')
      40
      41  #import zenpacksupport
      42
      43  FQDN = socket.getfqdn()
      44
      45  ZEN_PACK_INDEX_URL = ''
      46
      47  # All ZenPack eggs have to define exactly one entry point in this group.
      48  ZENPACK_ENTRY_POINT = 'zenoss.zenpacks'
      49
      50  ########################################
      51  #   ZenPack Creation
      52  ########################################
      53
      54 -def CreateZenPack(zpId, prevZenPackName=''):
      55      """
      56      Create the zenpack in the filesystem.
      57      The zenpack is not installed in Zenoss, it is simply created in
      58      the $ZENHOME/ZenPacks directory.  Usually this should be followed
      59      with a "zenpack install" call.
      60      zpId should already be valid, scrubbed value.
      61      prevZenPackName is written to PREV_ZENPACK_NAME in setup.py.
      62      """
      63      parts = zpId.split('.')
      64
      65      # Copy template to $ZENHOME/ZenPacks
      66      srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate')
      67      devDir = zenPath('ZenPacks')
      68      if not os.path.exists(devDir):
      69          os.mkdir(devDir, 0750)
      70      destDir = os.path.join(devDir, zpId)
      71      shutil.copytree(srcDir, destDir, symlinks=False)
      72      os.system('find %s -name .svn | xargs rm -rf' % destDir)
      73
      74      # Write setup.py
      75      packages = []
      76      for i in range(len(parts)):
      77          packages.append('.'.join(parts[:i+1]))
      78      mapping = dict(
      79          NAME = zpId,
      80          VERSION = '1.0',
      81          AUTHOR = '',
      82          LICENSE = '',
      83          NAMESPACE_PACKAGES = packages[:-1],
      84          PACKAGES = packages,
      85          INSTALL_REQUIRES = [],
      86          COMPAT_ZENOSS_VERS = '',
      87          PREV_ZENPACK_NAME = prevZenPackName,
      88          )
      89      WriteSetup(os.path.join(destDir, 'setup.py'), mapping)
      90
      91      # Create subdirectories
      92      base = destDir
      93      for part in parts[:-1]:
      94          base = os.path.join(base, part)
      95          os.mkdir(base)
      96          f = open(os.path.join(base, '__init__.py'), 'w')
      97          f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
      98          f.close()
      99      base = os.path.join(base, parts[-1])
     100      shutil.move(os.path.join(destDir, 'CONTENT'), base)
     101
     102      # Create the skins subdirs
     103      skinsDir = os.path.join(base, 'skins', zpId)
     104      os.makedirs(skinsDir)
     105
     106      # Stick a placeholder in the skins dir so that the egg will include
     107      # the dir even if empty.
     108      f = file(os.path.join(skinsDir, 'placeholder.txt'), 'w')
     109      f.close()
     110
     111      return destDir
     112
     113
     114 -def WriteSetup(setupPath, values):
     115      """
     116      """
     117      f = file(setupPath, 'r')
     118      lines = f.readlines()
     119      f.close()
     120
     121      newLines = []
     122      for i, line in enumerate(lines):
     123          if line.startswith('STOP_REPLACEMENTS'):
     124              newLines += lines[i:]
     125              break
     126          key = line.split('=')[0].strip()
     127          if values.has_key(key):
     128              value = values[key]
     129              if isinstance(value, basestring):
     130                  fmt = '%s = "%s"\n'
     131              else:
     132                  fmt = '%s = %s\n'
     133              newLines.append(fmt % (key, value))
     134          else:
     135              newLines.append(line)
     136
     137      f = file(setupPath, 'w')
     138      f.writelines(newLines)
     139      f.close()
     140
     141
     142 -def CanCreateZenPack(dmd, zpId):
     143      """
     144      Return tuple (bool, string) where first element is true if a new zenpack
     145      can be created with the given info and false if not.  If first element
     146      is True then the second part of the tuple contains the scrubbed ZenPack id.
     147      If the first part is False then the second contains an explanatory
     148      message.
     149      """
     150      # Check if id and package looks reasonable
     151      (allowable, idOrMsg) = ScrubZenPackId(zpId)
     152      if allowable:
     153          zpId = idOrMsg
     154      else:
     155          return (False, idOrMsg)
     156
     157      # Is the id already in use?
     158      if dmd:
     159          if zpId in dmd.ZenPackManager.packs.objectIds():
     160              return (False, 'A ZenPack named %s already exists.' % zpId)
     161
     162      # Is there another zenpack in the way?
     163      # Now that zenpacks are created in $ZENHOME/ZenPacks instead of
     164      # $ZENHOME/ZenPackDev this may no longer be necessary because a
     165      # zp in the way should be installed and caught by the already in use
     166      # check above.
     167      if os.path.exists(zenPath('ZenPacks', zpId)):
     168          return (False, 'A directory named %s already exists' % zpId +
     169                          ' in $ZENHOME/ZenPacks.  Use a different name'
     170                          ' or remove that directory.')
     171
     172      return (True, idOrMsg)
     173
     174
     175 -def ScrubZenPackId(name):
     176      """
     177      If the given name conforms to ZenPack naming rules, or can easily be
     178      modified to do so, then return (True, scrubbedName) where scrubbedName
     179      is either name or a slightly modified name.  If the given name does
     180      not conform to naming rules and we can't easily modify it to do so
     181      then return (False, errorMsg) where errorMsg describes why name
     182      is unacceptable.
     183      """
     184      parts = name.split('.')
     185
     186      # Remove leading dots, trailing dots, adjacent dots and strip whitespace
     187      # from each part
     188      parts = [p.strip() for p in parts]
     189      parts = [p for p in parts if p]
     190
     191      # Add/fix leading 'ZenPacks'
     192      if parts[0] != 'ZenPacks':
     193          if parts[0].lower() == 'zenpacks':
     194              parts[0] = 'ZenPacks'
     195          else:
     196              parts.insert(0, 'ZenPacks')
     197
     198      # Must be at least 3 parts
     199      if len(parts) < 3:
     200          return (False, 'ZenPack names must contain at least three package '
     201                  'names separated by periods.')
     202
     203      # Each part must start with a letter
     204      for p in parts:
     205          if p[0] not in string.letters:
     206              return (False, 'Each package name must start with a letter.')
     207
     208      # Only letters, numbers and underscores in each part
     209      allowable = string.letters + string.digits + '_'
     210      for p in parts:
     211          for c in p:
     212              if c not in allowable:
     213                  return (False, 'Package names may only contain letters, '
     214                          'numbers and underscores.')
     215
     216      return (True, '.'.join(parts))
     217
     218
     219  ########################################
     220  #   ZenPack Installation
     221  ########################################
     222
     223 -class NonCriticalInstallError(Exception):
     224 -    def __init__(self, message):
     225          Exception.__init__(self, message)
     226          self.message = message
     227
     228 -def InstallEggAndZenPack(dmd, eggPath, link=False,
     229                              filesOnly=False, sendEvent=True,
     230                              previousVersion=None):
     231      """
     232      Installs the given egg, instantiates the ZenPack, installs in
     233      dmd.ZenPackManager.packs, and runs the zenpacks's install method.
     234      Returns a list of ZenPacks that were installed.
     235      """
     236      zenPacks = []
     237      nonCriticalErrorEncountered = False
     238      try:
     239          zpDists = InstallEgg(dmd, eggPath, link=link)
     240          for d in zpDists:
     241              try:
     242                  zp = InstallDistAsZenPack(dmd,
     243                                            d,
     244                                            eggPath,
     245                                            link,
     246                                            filesOnly=filesOnly,
     247                                            previousVersion=\
     248                                              previousVersion)
     249                  zenPacks.append(zp)
     250              except NonCriticalInstallError, ex:
     251                  nonCriticalErrorEncountered = True
     252                  if sendEvent:
     253                      ZPEvent(dmd, 3, ex.message)
     254      except:
     255          if sendEvent:
     256              ZPEvent(dmd, 4, 'Error installing ZenPack %s' % eggPath,
     257                  '%s: %s' % sys.exc_info()[:2])
     258          raise
     259      if sendEvent:
     260          zenPackIds = [zp.id for zp in zenPacks]
     261          if zenPackIds:
     262              ZPEvent(dmd, 2, 'Installed ZenPacks %s' % ','.join(zenPackIds))
     263          elif not nonCriticalErrorEncountered:
     264              ZPEvent(dmd, 4, 'Unable to install %s' % eggPath)
     265      return zenPacks
     266
     267
     268 -def InstallEgg(dmd, eggPath, link=False):
     269      """
     270      Install the given egg and add to the current working set.
     271      This does not install the egg as a ZenPack.
     272      Return a list of distributions that should be installed as ZenPacks.
     273      """
     274      eggPath = os.path.abspath(eggPath)
     275      zenPackDir = zenPath('ZenPacks')
     276      eggInZenPacksDir = eggPath.startswith(zenPackDir + '/')
     277
     278      # Make sure $ZENHOME/ZenPacks exists
     279      CreateZenPacksDir()
     280
     281      # Install the egg
     282      if link:
     283          cmd = ('%s setup.py develop ' % binPath('python') +
     284                  '--site-dirs=%s ' % zenPackDir +
     285                  '-d %s' % zenPackDir)
     286          p = subprocess.Popen(cmd,
     287                              stdout=subprocess.PIPE,
     288                              stderr=subprocess.PIPE,
     289                              shell=True,
     290                              cwd=eggPath)
     291          out, err = p.communicate()
     292          p.wait()
     293          if p.returncode:
     294              raise ZenPackException('Error installing the egg (%s): %s' %
     295                                  (p.returncode, err))
     296          zpDists = AddDistToWorkingSet(eggPath)
     297      else:
     298          zpDists = DoEasyInstall(eggPath)
     299          # cmd = 'easy_install --always-unzip --site-dirs=%s -d %s %s' % (
     300          #             zenPackDir,
     301          #             zenPackDir,
     302          #             eggPath)
     303          # p = subprocess.Popen(cmd,
     304          #                     stdout=subprocess.PIPE,
     305          #                     stderr=subprocess.PIPE,
     306          #                     shell=True)
     307          # p.wait()
     308          # eggName = os.path.split(eggPath)[1]
     309          # eggPath = os.path.join(zenPackDir, eggName)
     310
     311      return zpDists
     312
     313
     314  # def GetZenPackNamesFromEggPath(eggPath):
     315  #     """
     316  #     Given a path to a ZenPack egg (installed or not) return the
     317  #     name of the ZenPack it contains.
     318  #     """
     319  #     zpNames = []
     320  #     for d in pkg_resources.find_distributions(eggPath)
     321  #         if d.project_name.startswith('ZenPacks.'):
     322  #             zpNames.append(d.project_name)
     323  #     return zpNames
     324
     325
     326 -def InstallDistAsZenPack(dmd, dist, eggPath, link=False, filesOnly=False,
     327                           previousVersion=None):
     328      """
     329      Given an installed dist, install it into Zenoss as a ZenPack.
     330      Return the ZenPack instance.
     331      """
     332      # Instantiate ZenPack
     333      entryMap = pkg_resources.get_entry_map(dist, ZENPACK_ENTRY_POINT)
     334      if not entryMap or len(entryMap) > 1:
     335          raise ZenPackException('A ZenPack egg must contain exactly one'
     336                  ' zenoss.zenpacks entry point.  This egg appears to contain'
     337                  ' %s such entry points.' % len(entryMap))
     338      packName, packEntry = entryMap.items()[0]
     339      zenPack = None;
     340      runExternalZenpack = True
     341      #if zenpack with same name exists we can't load both modules
     342      #installing new egg zenpack will be done in a sub process
     343      existing = dmd.ZenPackManager.packs._getOb(packName, None)
     344      if existing:
     345          log.info("Previous ZenPack exists with same name %s" % packName)
     346      if filesOnly or not existing:
     347          #running files only or zenpack by same name doesn't already exists
     348          # so no need to install the zenpack in an external process
     349          runExternalZenpack = False
     350          module = packEntry.load()
     351          if hasattr(module, 'ZenPack'):
     352              zenPack = module.ZenPack(packName)
     353          else:
     354              zenPack = ZenPack(packName)
     355          zenPack.eggPack = True
     356          CopyMetaDataToZenPackObject(dist, zenPack)
     357          if filesOnly:
     358              for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
     359                  loader.load(zenPack, None)
     360
     361
     362      if not filesOnly:
     363          # Look for an installed ZenPack to be upgraded.  In this case
     364          # upgraded means that it is removed before the new one is installed
     365          # but that its objects are not removed and the packables are
     366          # copied to the new instance.
     367          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     368          if not existing and zenPack.prevZenPackName:
     369              existing = dmd.ZenPackManager.packs._getOb(
     370                                  zenPack.prevZenPackName, None)
     371
     372          deferFileDeletion = False
     373          packables = []
     374          upgradingFrom = None
     375          if existing:
     376              upgradingFrom = existing.version
     377              for p in existing.packables():
     378                  packables.append(p)
     379                  existing.packables.removeRelation(p)
     380              if existing.isEggPack():
     381                  forceNoFileDeletion = existing.eggPath() == dist.location
     382                  RemoveZenPack(dmd, existing.id,
     383                                  skipDepsCheck=True, leaveObjects=True,
     384                                  forceNoFileDeletion=forceNoFileDeletion,
     385                                  uninstallEgg=False)
     386              else:
     387                  # Don't delete files, might still be needed for
     388                  # migrate scripts to be run below.
     389                  deferFileDeletion = True
     390                  oldzenpack.RemoveZenPack(dmd, existing.id,
     391                                  skipDepsCheck=True, leaveObjects=True,
     392                                  deleteFiles=False)
     393          if runExternalZenpack:
     394              log.info("installing zenpack %s; launching process" % packName)
     395              cmd = [binPath('zenpack')]
     396              if link:
     397                  cmd += ["--link"]
     398              cmd += ["--install", eggPath]
     399              if upgradingFrom:
     400                  cmd += ['--previousversion', upgradingFrom]
     401
     402              cmdStr = " ".join(cmd)
     403              log.debug("launching sub process command: %s" % cmdStr)
     404              p = subprocess.Popen(cmdStr,
     405                              shell=True)
     406              out, err = p.communicate()
     407              p.wait()
     408              if p.returncode:
     409                  raise ZenPackException('Error installing the egg (%s): %s' %
     410                                         (p.returncode, err))
     411              dmd._p_jar.sync()
     412          else:
     413              dmd.ZenPackManager.packs._setObject(packName, zenPack)
     414              zenPack = dmd.ZenPackManager.packs._getOb(packName)
     415              #hack because ZenPack.install is overridden by a lot of zenpacks
     416              #so we can't change the signature of install to take the
     417              #previousVerison
     418              zenPack.prevZenPackVersion = previousVersion
     419              zenPack.install(dmd)
     420              zenPack.prevZenPackVersion = None
     421
     422
     423          try:
     424              zenPack = dmd.ZenPackManager.packs._getOb(packName)
     425              for p in packables:
     426                  pId = p.getPrimaryId()
     427                  try:
     428                      # make sure packable still exists; could be deleted by a
     429                      # migrate
     430                      getObjByPath(dmd, pId)
     431                      log.debug("adding packable relation for id %s", pId)
     432                      zenPack.packables.addRelation(p)
     433                  except (KeyError, zExceptions.NotFound):
     434                      log.debug('did not find packable %s',pId)
     435          except AttributeError, e:
     436              # If this happens in the child process or during the non-upgrade
     437              # flow, reraise the exception
     438              if not runExternalZenpack:
     439                  raise
     440
     441              # This specific error will occur when the version of the ZenPack
     442              # being installed subclasses Products.ZenModel.ZenPack, but the
     443              # previous version of the ZenPack did not.
     444              if str(e) == "'ZenPack' object has no attribute '__of__'":
     445                  zenPack = ZenPack(packName)
     446              else:
     447                  # This is the signature error of class-loading issues
     448                  # during zenpack upgrade.  The final state should be okay,
     449                  # except that modified packables may be lost.
     450                  message = "There has been an error during the post-" + \
     451                            "installation steps for the zenpack %s.  In " + \
     452                            "most cases, no further action is required.  If " + \
     453                            "issues persist, please reinstall this zenpack."
     454                  message = message % packName
     455                  log.warning( message )
     456                  raise NonCriticalInstallError( message )
     457
     458          if deferFileDeletion:
     459              # We skipped deleting the existing files from filesystem
     460              # because maybe they'd be needed in migrate scripts.
     461              # Delete them now
     462              oldZpDir = zenPath('Products', existing.id)
     463              if os.path.islink(oldZpDir):
     464                  os.remove(oldZpDir)
     465              else:
     466                  shutil.rmtree(oldZpDir)
     467
     468      cleanupSkins(dmd)
     469      transaction.commit()
     470      return zenPack
     471
     472
     473 -def DiscoverEggs(dmd, zenPackId):
     474      """
     475      Find installed eggs that provide a zenoss.zenpacks entry point.
     476      Return a list of distributions whose ZenPacks need to be installed
     477      or upgraded.  The list is sorted into the order in which this needs to
     478      happen.
     479      """
     480      # Create a set of all available zenoss.zenpack entries that aren't
     481      # already installed in zenoss or need to be upgraded in zenoss.
     482      entries = set()
     483      parse_version = pkg_resources.parse_version
     484      for entry in pkg_resources.iter_entry_points(ZENPACK_ENTRY_POINT):
     485          packName = entry.name
     486          packVers = entry.dist.version
     487          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     488          if existing and existing.isEggPack():
     489              # We use >= to allow migrate to be run on currently installed
     490              # zenpacks whose version has been changed or for whom new
     491              # migrates have been added.
     492              if parse_version(packVers) >= parse_version(existing.version):
     493                  entries.add(entry)
     494          else:
     495              entries.add(entry)
     496
     497      # Starting with the entry representing zenPackId create a list of
     498      # all entrypoints
     499
     500      # orderedEntries lists entries in the opposite order of that in which
     501      # they need to be installed.  This is simply for convenience of using
     502      # .append() in code below.
     503      orderedEntries = []
     504      entriesByName = dict([(e.name, e) for e in entries])
     505
     506      def AddEntryAndProcessDeps(e):
     507          orderedEntries.append(e)
     508          for name in [r.project_name for r in e.dist.requires()]:
     509              if name in [e.name for e in orderedEntries]:
     510                  # This entry depends on something that we've already processed.
     511                  # This might be a circular dependency, might not be.
     512                  # We are just going to bail however.  This should be
     513                  # very unusual and the user can install deps first to work
     514                  # around.
     515                  raise ZenPackException('Unable to resolve ZenPack dependencies.'
     516                      ' Try installing dependencies first.')
     517              if name in entriesByName:
     518                  # The requirement is an entry that has not yet been processed
     519                  # here.  Add it to the list of entries to install/upgrade.
     520                  AddEntryAndProcessDeps(entriesByName[name])
     521              else:
     522                  # The requirement is not in the entries generated above.
     523                  # This either means that the dep is already installed (this
     524                  # is likely) or that easy_install missed something and the dep
     525                  # is not installed/available (this should be unlikely.)
     526                  pass
     527
     528      if zenPackId not in entriesByName:
     529          if zenPackId in dmd.ZenPackManager.packs.objectIds():
     530              return []
     531          else:
     532              raise ZenPackException('Unable to discover ZenPack named %s' %
     533                                      zenPackId)
     534      AddEntryAndProcessDeps(entriesByName[zenPackId])
     535      orderedEntries.reverse()
     536      return [e.dist for e in orderedEntries]
     537
     538
     539 -def AddDistToWorkingSet(distPath):
     540      """
     541      Given the path to a dist (an egg) add it to the current working set.
     542      This is basically a pkg_resources-friendly way of adding it to
     543      sys.path.
     544      Return a list of all distributions on distPath that appear to
     545      be ZenPacks.
     546      """
     547      zpDists = []
     548      for d in pkg_resources.find_distributions(distPath):
     549          pkg_resources.working_set.add(d)
     550          pkg_resources.require(d.project_name)
     551          if d.project_name.startswith('ZenPacks.'):
     552              zpDists.append(d)
     553      return zpDists
     554
     555
     556 -def ReadZenPackInfo(dist):
     557      """
     558      Return a dictionary containing the egg metadata
     559      """
     560      info = {}
     561      if dist.has_metadata('PKG-INFO'):
     562          lines = dist.get_metadata('PKG-INFO')
     563          for line in pkg_resources.yield_lines(lines):
     564              key, value = line.split(':', 1)
     565              info[key.strip()] = value.strip()
     566      if dist.has_metadata('zenpack_info'):
     567          lines = dist.get_metadata('zenpack_info')
     568          for line in pkg_resources.yield_lines(lines):
     569              key, value = line.split(':', 1)
     570              info[key.strip()] = value.strip()
     571      return info
     572
     573
     574 -def CopyMetaDataToZenPackObject(dist, pack):
     575      """
     576      Copy metadata type stuff from the distribution to the zp object.
     577      """
     578      # Version
     579      pack.version = dist.version
     580
     581      # Egg Info
     582      info = ReadZenPackInfo(dist)
     583      pack.author = info.get('Author', '')
     584      if pack.author == 'UNKNOWN':
     585          pack.author = ''
     586      pack.compatZenossVers = info.get('compatZenossVers', '')
     587      pack.prevZenPackName = info.get('prevZenPackName', '')
     588
     589      # Requires
     590      pack.dependencies = {}
     591      for r in dist.requires():
     592          name = r.project_name
     593          spec = str(r)[len(name):]
     594          pack.dependencies[name] = spec
     595
     596
     597 -def CreateZenPacksDir():
     598      """
     599      Make sure $ZENHOME/ZenPacks exists
     600      """
     601      zpDir = zenPath('ZenPacks')
     602      if not os.path.isdir(zpDir):
     603          os.mkdir(zpDir, 0750)
     604
     605
     606 -def DoEasyInstall(eggPath):
     607      """
     608      Use easy_install to install an egg from the filesystem.
     609      easy_install will install the egg, but does not install it into
     610      Zenoss as ZenPacks.
     611      Returns a list of distributions that were installed that appear
     612      to be ZenPacks.
     613      """
     614      from setuptools.command import easy_install
     615
     616      # Make sure $ZENHOME/ZenPacks exists
     617      CreateZenPacksDir()
     618
     619      # Create temp file for easy_install to write results to
     620      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     621      # eggPaths is a set of paths to eggs that were installed.  We need to
     622      # add them to the current workingset so we can discover their
     623      # entry points.
     624      eggPaths = set()
     625      try:
     626          # Execute the easy_install
     627          args = ['--site-dirs', zenPath('ZenPacks'),
     628              '-d', zenPath('ZenPacks'),
     629              # '-i', ZEN_PACK_INDEX_URL,
     630              '--allow-hosts', 'None',
     631              '--record', tempPath,
     632              '--quiet',
     633              eggPath]
     634          easy_install.main(args)
     635          # Collect the paths for eggs that were installed
     636          f = open(tempPath, 'r')
     637          marker = '.egg/'
     638          markerOffset = len(marker)-1
     639          for l in f.readlines():
     640              i = l.find(marker)
     641              if i > 0:
     642                  eggPaths.add(l[:i+markerOffset])
     643      finally:
     644          os.remove(tempPath)
     645      # Add any installed eggs to the current working set
     646      zpDists = []
     647      for path in eggPaths:
     648          zpDists += AddDistToWorkingSet(path)
     649      return zpDists
     650
     651
     652  ########################################
     653  #   Zenoss.Net
     654  ########################################
     655
     656
     657 -def FetchAndInstallZenPack(dmd, zenPackName, zenPackVersion='', sendEvent=True):
     658      """
     659      Fetch the named zenpack and all its dependencies and install them.
     660      Return a list of the ZenPacks that were installed.
     661      """
     662      zenPacks = []
     663      try:
     664          zpDists = FetchZenPack(zenPackName, zenPackVersion)
     665          for d in zpDists:
     666              zenPacks.append(InstallDistAsZenPack(dmd, d))
     667      except:
     668          if sendEvent:
     669              ZPEvent(dmd, 4, 'Failed to install ZenPack %s' % zenPackName,
     670                  '%s: %s' % sys.exc_info()[:2])
     671          raise
     672      if sendEvent:
     673          zenPackIds = [z.id for z in zenPacks]
     674          if zenPackIds:
     675              ZPEvent(dmd, 2, 'Installed ZenPacks: %s' % ', '.join(zenPackIds))
     676          if zenPackName not in zenPackIds:
     677              ZPEvent(dmd, 4, 'Unable to install ZenPack %s' % zenPackName)
     678      return zenPacks
     679
     680
     681 -def FetchZenPack(zenPackName, zenPackVersion=''):
     682      """
     683      Use easy_install to retrieve the given zenpack and any dependencies.
     684      easy_install will install the eggs, but does not install them into
     685      Zenoss as ZenPacks.
     686      Return a list of distributions just installed that appear to be
     687      ZenPacks.
     688
     689      NB: This should be refactored.  It shares most of its code with
     690      DoEasyInstall()
     691      """
     692      from setuptools.command import easy_install
     693
     694      # Make sure $ZENHOME/ZenPacks exists
     695      CreateZenPacksDir()
     696
     697      # Create temp file for easy_install to write results to
     698      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     699      # eggPaths is a set of paths to eggs that were installed.  We need to
     700      # add them to the current workingset so we can discover their
     701      # entry points.
     702      eggPaths = set()
     703      try:
     704          # Execute the easy_install
     705          args = ['--site-dirs', zenPath('ZenPacks'),
     706              '-d', zenPath('ZenPacks'),
     707              '-i', ZEN_PACK_INDEX_URL,
     708              '--allow-hosts', 'None',
     709              '--record', tempPath,
     710              '--quiet',
     711              zenPackName]
     712          easy_install.main(args)
     713          # Collect the paths for eggs that were installed
     714          f = open(tempPath, 'r')
     715          marker = '.egg/'
     716          markerOffset = len(marker)-1
     717          for l in f.readlines():
     718              i = l.find(marker)
     719              if i > 0:
     720                  eggPaths.add(l[:i+markerOffset])
     721      finally:
     722          os.remove(tempPath)
     723      # Add any installed eggs to the current working set
     724      zpDists = []
     725      for path in eggPaths:
     726          zpDists += AddDistToWorkingSet(path)
     727      return zpDists
     728
     729
     730 -def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):
     731      """
     732      Upload the specified zenpack to the given project.
     733      Project is a string of the form 'enterprise/myproject' or
     734      'community/otherproject'.
     735      """
     736      zp = dmd.ZenPackManager.packs._getOb(packName, None)
     737      if not zp:
     738          raise ZenPackException('No ZenPack named %s' % packName)
     739
     740      # Export the zenpack
     741      fileName = zp.manage_exportPack()
     742      filePath = zenPath('export', fileName)
     743
     744      # Login to Zenoss.net
     745      from DotNetCommunication import DotNetSession
     746      session = DotNetSession()
     747      userSettings = dmd.ZenUsers.getUserSettings()
     748      session.login(znetUser, znetPass)
     749
     750      # Upload
     751      zpFile = open(zenPath('export', fileName), 'r')
     752      try:
     753          response = session.open('%s/createRelease' % project.strip('/'), {
     754                  'description': description,
     755                  'fileStorage': zpFile,
     756                  })
     757      finally:
     758          zpFile.close()
     759      if response:
     760          result = response.read()
     761          if "'success':true" not in result:
     762              raise ZenPackException('Upload failed')
     763      else:
     764          raise ZenPackException('Failed to connect to Zenoss.net')
     765      return
     766
     767
     768  ########################################
     769  #   ZenPack Removal
     770  ########################################
     771
     772
     773 -def RemoveZenPack(dmd, packName, filesOnly=False, skipDepsCheck=False,
     774                      leaveObjects=False, sendEvent=True,
     775                      forceNoFileDeletion=False, uninstallEgg=True):
     776      """
     777      Remove the given ZenPack from Zenoss.
     778      Whether the ZenPack will be removed from the filesystem or not
     779      depends on the result of the ZenPack's shouldDeleteFilesOnRemoval method.
     780      """
     781      try:
     782          if filesOnly:
     783              skipDepsCheck = True
     784
     785          # Check for dependency implications here?
     786          if not skipDepsCheck:
     787              deps = GetDependents(dmd, packName)
     788              if deps:
     789                  raise ZenPackDependentsException('%s cannot be removed ' % packName +
     790                          'because it is required by %s' % ', '.join(deps))
     791
     792          if not filesOnly:
     793              # Fetch the zenpack, call its remove() and remove from packs
     794              zp = None
     795              try:
     796                  zp = dmd.ZenPackManager.packs._getOb(packName)
     797              except AttributeError, ex:
     798                  raise ZenPackNotFoundException('No ZenPack named %s is installed' %
     799                                                  packName)
     800              zp.remove(dmd, leaveObjects)
     801              dmd.ZenPackManager.packs._delObject(packName)
     802              transaction.commit()
     803
     804          # Uninstall the egg and possibly delete it
     805          # If we can't find the distribution then apparently the zp egg itself is
     806          # missing.  Continue on with the removal and swallow the
     807          # DistributionNotFound exception
     808          try:
     809              dist = zp.getDistribution()
     810          except pkg_resources.DistributionNotFound:
     811              dist = None
     812          if dist:
     813              # Determine deleteFiles before develop -u gets called.  Once
     814              # it is called the egg has problems figuring out some of it's state.
     815              deleteFiles = zp.shouldDeleteFilesOnRemoval()
     816              if uninstallEgg:
     817                  if zp.isDevelopment():
     818                      zenPackDir = zenPath('ZenPacks')
     819                      cmd = ('%s setup.py develop -u '
     820                              % binPath('python') +
     821                              '--site-dirs=%s ' % zenPackDir +
     822                              '-d %s' % zenPackDir)
     823                      p = subprocess.Popen(cmd,
     824                                          stdout=subprocess.PIPE,
     825                                          stderr=subprocess.PIPE,
     826                                          shell=True,
     827                                          cwd=zp.eggPath())
     828                      out, err = p.communicate()
     829                      code = p.wait()
     830                      if code:
     831                          raise ZenPackException(err)
     832                  else:
     833                      DoEasyUninstall(packName)
     834              # elif cleanupEasyInstallPth:
     835              #     # Do we need to call easy_install -m here?  It causes problems
     836              #     # because it tries to install deps.  Cleanup easy-install.pth
     837              #     # ourselves instead.
     838              #     # We don't want to cleanup easy-install.pth when a newer
     839              #     # version of the egg has already been installed (when doing
     840              #     # an upgrade or installing in new location.)
     841              #     eggLink = './%s' % zp.eggName()
     842              #     CleanupEasyInstallPth(eggLink)
     843              if deleteFiles and not forceNoFileDeletion:
     844                  eggDir = zp.eggPath()
     845                  if os.path.islink(eggDir):
     846                      os.remove(eggDir)
     847                  else:
     848                      shutil.rmtree(eggDir)
     849          cleanupSkins(dmd)
     850          transaction.commit()
     851      except:
     852          if sendEvent:
     853              ZPEvent(dmd, 4, 'Error removing ZenPack %s' % packName,
     854                  '%s: %s' % sys.exc_info()[:2])
     855          raise
     856      if sendEvent:
     857          ZPEvent(dmd, 2, 'Removed ZenPack %s' % packName)
     858
     859
     860 -def DoEasyUninstall(name):
     861      """
     862      Execute the easy_install command to unlink the given egg.
     863      What this is really doing is switching the egg to be in
     864      multiple-version mode, however this is the first step in deleting
     865      an egg as described here:
     866      http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages
     867      """
     868      from setuptools.command import easy_install
     869      args = ['--site-dirs', zenPath('ZenPacks'),
     870          '-d', zenPath('ZenPacks'),
     871          #'--record', tempPath,
     872          '--quiet',
     873          '-m',
     874          name]
     875      easy_install.main(args)
     876
     877
     878 -def CanRemoveZenPacks(dmd, packNames):
     879      """
     880      Returns a tuple of (canRemove, otherDependents)
     881      canRemove is True if the listed zenPacks have no dependents not also
     882      listed in packNames, False otherwise.
     883      otherDependents is a list of zenpack names not in packNames that
     884      depend on one or more of the packs in packNames.
     885      """
     886      unhappy = set()
     887      for name in packNames:
     888          deps = GetDependents(dmd, name)
     889          unhappy.update(set([dep for dep in deps if dep not in packNames]))
     890      return (not unhappy and True or False, list(unhappy))
     891
     892
     893  # def CleanupEasyInstallPth(eggLink):
     894  #     """
     895  #     Remove the entry for the given egg from the
     896  #     $ZENHOME/ZenPacks/easy-install.pth file.  If this entry is left
     897  #     in place in can cause problems during the next install of the same
     898  #     egg.
     899  #     """
     900  #     # Remove the path from easy-install.pth
     901  #     eggTail = os.path.split(eggLink)[1]
     902  #     easyPth = zenPath('ZenPacks', 'easy-install.pth')
     903  #     if os.path.isfile(easyPth):
     904  #         needToWrite = False
     905  #         newLines = []
     906  #         f = open(easyPth, 'r')
     907  #         for line in f:
     908  #             if os.path.split(line.strip())[1] == eggTail:
     909  #                 needToWrite = True
     910  #             else:
     911  #                 newLines.append(line)
     912  #         f.close()
     913  #         if needToWrite:
     914  #             f = open(easyPth, 'w')
     915  #             f.writelines(newLines)
     916  #             f.close()
     917
     918
     919 -def GetDependents(dmd, packName):
     920      """
     921      Return a list of installed ZenPack ids that list packName as a dependency
     922      """
     923      return [zp.id for zp in dmd.ZenPackManager.packs()
     924                  if zp.id != packName and zp.dependencies.has_key(packName)]
     925
     926
     927  ########################################
     928  #   __main__, dispatching, etc
     929  ########################################
     930
     931
     932 -def ZPEvent(dmd, severity, summary, message=None):
     933      """
     934      Send an event to Zenoss.
     935      """
     936      dmd.ZenEventManager.sendEvent(dict(
     937          device=FQDN,
     938          eventClass='/Unknown',
     939          severity=severity,
     940          summary=summary,
     941          message=message))
     942
     943
     944 -class ZenPackCmd(ZenScriptBase):
     945      """
     946      Utilities for creating, installing, removing ZenPacks.
     947
     948      NOTE: Users will probably invoke zenpack from the command line, which
     949      runs zenpack.py rather than this file.  zenpack.py calls functions
     950      in this module when it detects that new-style (egg) ZenPacks are involved.
     951      The plan is that once support for old-style (non-egg) ZenPacks is dropped
     952      zenpack.py can go away and this will take its place.  Until then this
     953      script can be invoked directly via the zenpackcmd script if desired.
     954      Invoking this script directly has the benefit of slightly better
     955      progress/status output to stdout.
     956      """
     957
     958 -    def run(self):
     959          """
     960          Execute the user's request.
     961          """
     962
     963          self.connect()
     964          def PrintInstalled(installed, eggOnly=False):
     965              if installed:
     966                  if eggOnly:
     967                      names = [i['id'] for i in installed]
     968                      what = 'ZenPack egg'
     969                  else:
     970                      names = [i.id for i in installed]
     971                      what = 'ZenPack'
     972                  print('Installed %s%s: %s' % (
     973                          what,
     974                          len(names) > 1 and 's' or '',
     975                          ', '.join(names)))
     976              else:
     977                  print('No ZenPacks installed.')
     978
     979          if not getattr(self.dmd, 'ZenPackManager', None):
     980              raise ZenPackNeedMigrateException('Your Zenoss database appears'
     981                  ' to be out of date. Try running zenmigrate to update.')
     982          if self.options.eggOnly and self.options.eggPath:
     983              zpDists = InstallEgg(self.dmd, self.options.eggPath,
     984                                              link=self.options.link)
     985              PrintInstalled([{'id':d.project_name} for d in zpDists],
     986                              eggOnly=True)
     987          if self.options.eggPath:
     988              installed = InstallEggAndZenPack(
     989                                  self.dmd, self.options.eggPath,
     990                                  link=self.options.link,
     991                                  filesOnly=self.options.filesOnly,
     992                                  previousVersion= self.options.previousVersion)
     993              PrintInstalled(installed)
     994          elif self.options.fetch:
     995              installed = FetchAndInstallZenPack(self.dmd, self.options.fetch)
     996              PrintInstalled(installed)
     997          elif self.options.upload:
     998              return UploadZenPack(self.dmd, self.options.upload,
     999                                      self.options.znetProject,
    1000                                      self.options.uploadDesc,
    1001                                      self.options.znetUser,
    1002                                      self.options.znetPass)
    1003          elif self.options.removePackName:
    1004              try:
    1005                  RemoveZenPack(self.dmd, self.options.removePackName)
    1006                  print('Removed ZenPack: %s' % self.options.removePackName)
    1007              except ZenPackNotFoundException, e:
    1008                  sys.stderr.write(str(e) + '\n')
    1009          elif self.options.list:
    1010              self.list()
    1011          else:
    1012              self.parser.print_help()
    1013
    1014
    1015 -    def buildOptions(self):
    1016          self.parser.add_option('--install',
    1017                                 dest='eggPath',
    1018                                 default=None,
    1019                                 help="name of the pack to install")
    1020  #        self.parser.add_option('--fetch',
    1021  #                               dest='fetch',
    1022  #                               default=None,
    1023  #                               help='Name of ZenPack to retrieve from '
    1024  #                                    'Zenoss.net and install.')
    1025  #        self.parser.add_option('--fetch-vers',
    1026  #                               dest='fetchVers',
    1027  #                               default=None,
    1028  #                               help='Use with --fetch to specify a version'
    1029  #                                    ' for the ZenPack to download and install.')
    1030  #        self.parser.add_option('--znet-user',
    1031  #                               dest='znetUser',
    1032  #                               default=None,
    1033  #                               help='Use with --fetch or --upload to specify'
    1034  #                                    ' your Zenoss.net username.')
    1035  #        self.parser.add_option('--znet-pass',
    1036  #                               dest='znetPass',
    1037  #                               default=None,
    1038  #                               help='Use with --fetch or --upload to specify'
    1039  #                                    ' your Zenoss.net password.')
    1040  #        self.parser.add_option('--upload',
    1041  #                               dest='upload',
    1042  #                               default=None,
    1043  #                               help='Name of ZenPack to upload to '
    1044  #                                    'Zenoss.net')
    1045  #        self.parser.add_option('--znet-project',
    1046  #                               dest='znetProject',
    1047  #                               default=None,
    1048  #                               help='Use with --upload to specify'
    1049  #                                    ' which Zenoss.net project to create'
    1050  #                                    ' a release on.')
    1051  #        self.parser.add_option('--upload-desc',
    1052  #                               dest='uploadDesc',
    1053  #                               default=None,
    1054  #                               help='Use with --upload to provide'
    1055  #                                    ' a description for the new release.')
    1056          self.parser.add_option('--link',
    1057                                 dest='link',
    1058                                 action='store_true',
    1059                                 default=False,
    1060                                 help='Install the ZenPack in its current '
    1061                                  'location, do not copy to $ZENHOME/ZenPacks. '
    1062                                  'Also mark ZenPack as editable. '
    1063                                  'This only works with source directories '
    1064                                  'containing setup.py files, not '
    1065                                  'egg files.')
    1066          self.parser.add_option('--remove',
    1067                                 dest='removePackName',
    1068                                 default=None,
    1069                                 help="name of the pack to remove")
    1070          self.parser.add_option('--leave-objects',
    1071                                 dest='leaveObjects',
    1072                                 default=False,
    1073                                 action='store_true',
    1074                                 help="When specified with --remove then objects"
    1075                                      ' provided by the ZenPack and those'
    1076                                      ' depending on the ZenPack are not deleted.'
    1077                                      ' This may result in broken objects in your'
    1078                                      ' database unless the ZenPack is'
    1079                                      ' reinstalled.')
    1080          self.parser.add_option('--files-only',
    1081                                 dest='filesOnly',
    1082                                 action="store_true",
    1083                                 default=False,
    1084                                 help='install onto filesystem but not into '
    1085                                          'zenoss')
    1086          self.parser.add_option('--previousversion',
    1087                                 dest='previousVersion',
    1088                                 default=None,
    1089                                 help="Previous version of the zenpack;"
    1090                                      ' used during upgrades')
    1091          ZenScriptBase.buildOptions(self)
    1092
    1093
    1094  if __name__ == '__main__':
    1095      try:
    1096          zp = ZenPackCmd()
    1097          zp.run()
    1098      except ZenPackException, e:
    1099          sys.stderr.write('%s\n' % str(e))
    1100          sys.exit(-1)
    1101