Package Products :: Package ZenUtils :: Module ZenPackCmd |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenPackCmd

       1  ###########################################################################
       2  #
       3  # This program is part of Zenoss Core, an open source monitoring platform.
       4  # Copyright (C) 2008, Zenoss Inc.
       5  #
       6  # This program is free software; you can redistribute it and/or modify it
       7  # under the terms of the GNU General Public License version 2 as published by
       8  # the Free Software Foundation.
       9  #
      10  # For complete information please visit: http://www.zenoss.com/oss/
      11  #
      12  ###########################################################################
      13
      14  __doc__ = "Manage ZenPacks"
      15
      16  import Globals
      17  from Products.ZenUtils.ZenScriptBase import ZenScriptBase
      18  from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, getObjByPath
      19
      20  from Products.ZenModel.ZenPack import ZenPackException, \
      21                                          ZenPackNotFoundException, \
      22                                          ZenPackNeedMigrateException
      23  from Products.ZenModel.ZenPack import ZenPackDependentsException
      24  from Products.ZenModel.ZenPack import ZenPack
      25  from Products.ZenUtils.PkgResources import pkg_resources
      26  from Products.Zuul.utils import CatalogLoggingFilter
      27  import Products.ZenModel.ZenPackLoader as ZPL
      28  import zenpack as oldzenpack
      29  import transaction
      30  import os, sys
      31  import shutil
      32  import string
      33  import tempfile
      34  import subprocess
      35  import socket
      36  import logging
      37  import zExceptions
      38
      39
      40  log = logging.getLogger('zen.ZenPackCMD')
      41
      42  #import zenpacksupport
      43
      44  FQDN = socket.getfqdn()
      45
      46  ZEN_PACK_INDEX_URL = ''
      47
      48  # All ZenPack eggs have to define exactly one entry point in this group.
      49  ZENPACK_ENTRY_POINT = 'zenoss.zenpacks'
      50
      51  ########################################
      52  #   ZenPack Creation
      53  ########################################
      54
      55 -def CreateZenPack(zpId, prevZenPackName=''):
      56      """
      57      Create the zenpack in the filesystem.
      58      The zenpack is not installed in Zenoss, it is simply created in
      59      the $ZENHOME/ZenPacks directory.  Usually this should be followed
      60      with a "zenpack install" call.
      61      zpId should already be valid, scrubbed value.
      62      prevZenPackName is written to PREV_ZENPACK_NAME in setup.py.
      63      """
      64      parts = zpId.split('.')
      65
      66      # Copy template to $ZENHOME/ZenPacks
      67      srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate')
      68      devDir = zenPath('ZenPacks')
      69      if not os.path.exists(devDir):
      70          os.mkdir(devDir, 0750)
      71      destDir = os.path.join(devDir, zpId)
      72      shutil.copytree(srcDir, destDir, symlinks=False)
      73      os.system('find %s -name .svn | xargs rm -rf' % destDir)
      74
      75      # Write setup.py
      76      packages = []
      77      for i in range(len(parts)):
      78          packages.append('.'.join(parts[:i+1]))
      79      mapping = dict(
      80          NAME = zpId,
      81          VERSION = '1.0',
      82          AUTHOR = '',
      83          LICENSE = '',
      84          NAMESPACE_PACKAGES = packages[:-1],
      85          PACKAGES = packages,
      86          INSTALL_REQUIRES = [],
      87          COMPAT_ZENOSS_VERS = '',
      88          PREV_ZENPACK_NAME = prevZenPackName,
      89          )
      90      WriteSetup(os.path.join(destDir, 'setup.py'), mapping)
      91
      92      # Create subdirectories
      93      base = destDir
      94      for part in parts[:-1]:
      95          base = os.path.join(base, part)
      96          os.mkdir(base)
      97          f = open(os.path.join(base, '__init__.py'), 'w')
      98          f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
      99          f.close()
     100      base = os.path.join(base, parts[-1])
     101      shutil.move(os.path.join(destDir, 'CONTENT'), base)
     102
     103      # Create the skins subdirs
     104      skinsDir = os.path.join(base, 'skins', zpId)
     105      os.makedirs(skinsDir)
     106
     107      # Stick a placeholder in the skins dir so that the egg will include
     108      # the dir even if empty.
     109      f = file(os.path.join(skinsDir, 'placeholder.txt'), 'w')
     110      f.close()
     111
     112      return destDir
     113
     114
     115 -def WriteSetup(setupPath, values):
     116      """
     117      """
     118      f = file(setupPath, 'r')
     119      lines = f.readlines()
     120      f.close()
     121
     122      newLines = []
     123      for i, line in enumerate(lines):
     124          if line.startswith('STOP_REPLACEMENTS'):
     125              newLines += lines[i:]
     126              break
     127          key = line.split('=')[0].strip()
     128          if values.has_key(key):
     129              value = values[key]
     130              if isinstance(value, basestring):
     131                  fmt = '%s = "%s"\n'
     132              else:
     133                  fmt = '%s = %s\n'
     134              newLines.append(fmt % (key, value))
     135          else:
     136              newLines.append(line)
     137
     138      f = file(setupPath, 'w')
     139      f.writelines(newLines)
     140      f.close()
     141
     142
     143 -def CanCreateZenPack(dmd, zpId):
     144      """
     145      Return tuple (bool, string) where first element is true if a new zenpack
     146      can be created with the given info and false if not.  If first element
     147      is True then the second part of the tuple contains the scrubbed ZenPack id.
     148      If the first part is False then the second contains an explanatory
     149      message.
     150      """
     151      # Check if id and package looks reasonable
     152      (allowable, idOrMsg) = ScrubZenPackId(zpId)
     153      if allowable:
     154          zpId = idOrMsg
     155      else:
     156          return (False, idOrMsg)
     157
     158      # Is the id already in use?
     159      if dmd:
     160          if zpId in dmd.ZenPackManager.packs.objectIds():
     161              return (False, 'A ZenPack named %s already exists.' % zpId)
     162
     163      # Is there another zenpack in the way?
     164      # Now that zenpacks are created in $ZENHOME/ZenPacks instead of
     165      # $ZENHOME/ZenPackDev this may no longer be necessary because a
     166      # zp in the way should be installed and caught by the already in use
     167      # check above.
     168      if os.path.exists(zenPath('ZenPacks', zpId)):
     169          return (False, 'A directory named %s already exists' % zpId +
     170                          ' in $ZENHOME/ZenPacks.  Use a different name'
     171                          ' or remove that directory.')
     172
     173      return (True, idOrMsg)
     174
     175
     176 -def ScrubZenPackId(name):
     177      """
     178      If the given name conforms to ZenPack naming rules, or can easily be
     179      modified to do so, then return (True, scrubbedName) where scrubbedName
     180      is either name or a slightly modified name.  If the given name does
     181      not conform to naming rules and we can't easily modify it to do so
     182      then return (False, errorMsg) where errorMsg describes why name
     183      is unacceptable.
     184      """
     185      parts = name.split('.')
     186
     187      # Remove leading dots, trailing dots, adjacent dots and strip whitespace
     188      # from each part
     189      parts = [p.strip() for p in parts]
     190      parts = [p for p in parts if p]
     191
     192      # Add/fix leading 'ZenPacks'
     193      if parts[0] != 'ZenPacks':
     194          if parts[0].lower() == 'zenpacks':
     195              parts[0] = 'ZenPacks'
     196          else:
     197              parts.insert(0, 'ZenPacks')
     198
     199      # Must be at least 3 parts
     200      if len(parts) < 3:
     201          return (False, 'ZenPack names must contain at least three package '
     202                  'names separated by periods.')
     203
     204      # Each part must start with a letter
     205      for p in parts:
     206          if p[0] not in string.letters:
     207              return (False, 'Each package name must start with a letter.')
     208
     209      # Only letters, numbers and underscores in each part
     210      allowable = string.letters + string.digits + '_'
     211      for p in parts:
     212          for c in p:
     213              if c not in allowable:
     214                  return (False, 'Package names may only contain letters, '
     215                          'numbers and underscores.')
     216
     217      return (True, '.'.join(parts))
     218
     219
     220  ########################################
     221  #   ZenPack Installation
     222  ########################################
     223
     224 -class NonCriticalInstallError(Exception):
     225 -    def __init__(self, message):
     226          Exception.__init__(self, message)
     227          self.message = message
     228
     229 -def InstallEggAndZenPack(dmd, eggPath, link=False,
     230                              filesOnly=False, sendEvent=True,
     231                              previousVersion=None):
     232      """
     233      Installs the given egg, instantiates the ZenPack, installs in
     234      dmd.ZenPackManager.packs, and runs the zenpacks's install method.
     235      Returns a list of ZenPacks that were installed.
     236      """
     237      zenPacks = []
     238      nonCriticalErrorEncountered = False
     239      try:
     240          zpDists = InstallEgg(dmd, eggPath, link=link)
     241          for d in zpDists:
     242              try:
     243                  zp = InstallDistAsZenPack(dmd,
     244                                            d,
     245                                            eggPath,
     246                                            link,
     247                                            filesOnly=filesOnly,
     248                                            previousVersion=\
     249                                              previousVersion)
     250                  zenPacks.append(zp)
     251              except NonCriticalInstallError, ex:
     252                  nonCriticalErrorEncountered = True
     253                  if sendEvent:
     254                      ZPEvent(dmd, 3, ex.message)
     255      except:
     256          if sendEvent:
     257              ZPEvent(dmd, 4, 'Error installing ZenPack %s' % eggPath,
     258                  '%s: %s' % sys.exc_info()[:2])
     259          raise
     260      if sendEvent:
     261          zenPackIds = [zp.id for zp in zenPacks]
     262          if zenPackIds:
     263              ZPEvent(dmd, 2, 'Installed ZenPacks %s' % ','.join(zenPackIds))
     264          elif not nonCriticalErrorEncountered:
     265              ZPEvent(dmd, 4, 'Unable to install %s' % eggPath)
     266      return zenPacks
     267
     268
     269 -def InstallEgg(dmd, eggPath, link=False):
     270      """
     271      Install the given egg and add to the current working set.
     272      This does not install the egg as a ZenPack.
     273      Return a list of distributions that should be installed as ZenPacks.
     274      """
     275      eggPath = os.path.abspath(eggPath)
     276      zenPackDir = zenPath('ZenPacks')
     277      eggInZenPacksDir = eggPath.startswith(zenPackDir + '/')
     278
     279      # Make sure $ZENHOME/ZenPacks exists
     280      CreateZenPacksDir()
     281
     282      # Install the egg
     283      if link:
     284          cmd = ('%s setup.py develop ' % binPath('python') +
     285                  '--site-dirs=%s ' % zenPackDir +
     286                  '-d %s' % zenPackDir)
     287          p = subprocess.Popen(cmd,
     288                              stdout=subprocess.PIPE,
     289                              stderr=subprocess.PIPE,
     290                              shell=True,
     291                              cwd=eggPath)
     292          out, err = p.communicate()
     293          p.wait()
     294          if p.returncode:
     295              DoEasyUninstall(eggPath)
     296              raise ZenPackException('Error installing the egg (%s): %s' %
     297                                  (p.returncode, err))
     298          zpDists = AddDistToWorkingSet(eggPath)
     299      else:
     300          try:
     301              zpDists = DoEasyInstall(eggPath)
     302          except:
     303              DoEasyUninstall(eggPath)
     304              raise
     305          # cmd = 'easy_install --always-unzip --site-dirs=%s -d %s %s' % (
     306          #             zenPackDir,
     307          #             zenPackDir,
     308          #             eggPath)
     309          # p = subprocess.Popen(cmd,
     310          #                     stdout=subprocess.PIPE,
     311          #                     stderr=subprocess.PIPE,
     312          #                     shell=True)
     313          # p.wait()
     314          # eggName = os.path.split(eggPath)[1]
     315          # eggPath = os.path.join(zenPackDir, eggName)
     316      return zpDists
     317
     318
     319  # def GetZenPackNamesFromEggPath(eggPath):
     320  #     """
     321  #     Given a path to a ZenPack egg (installed or not) return the
     322  #     name of the ZenPack it contains.
     323  #     """
     324  #     zpNames = []
     325  #     for d in pkg_resources.find_distributions(eggPath)
     326  #         if d.project_name.startswith('ZenPacks.'):
     327  #             zpNames.append(d.project_name)
     328  #     return zpNames
     329
     330
     331 -def InstallDistAsZenPack(dmd, dist, eggPath, link=False, filesOnly=False,
     332                           previousVersion=None):
     333      """
     334      Given an installed dist, install it into Zenoss as a ZenPack.
     335      Return the ZenPack instance.
     336      """
     337      # Instantiate ZenPack
     338      entryMap = pkg_resources.get_entry_map(dist, ZENPACK_ENTRY_POINT)
     339      if not entryMap or len(entryMap) > 1:
     340          raise ZenPackException('A ZenPack egg must contain exactly one'
     341                  ' zenoss.zenpacks entry point.  This egg appears to contain'
     342                  ' %s such entry points.' % len(entryMap))
     343      packName, packEntry = entryMap.items()[0]
     344      zenPack = None;
     345      runExternalZenpack = True
     346      #if zenpack with same name exists we can't load both modules
     347      #installing new egg zenpack will be done in a sub process
     348      existing = dmd.ZenPackManager.packs._getOb(packName, None)
     349      if existing:
     350          log.info("Previous ZenPack exists with same name %s" % packName)
     351      if filesOnly or not existing:
     352          #running files only or zenpack by same name doesn't already exists
     353          # so no need to install the zenpack in an external process
     354          runExternalZenpack = False
     355          module = packEntry.load()
     356          if hasattr(module, 'ZenPack'):
     357              zenPack = module.ZenPack(packName)
     358          else:
     359              zenPack = ZenPack(packName)
     360          zenPack.eggPack = True
     361          CopyMetaDataToZenPackObject(dist, zenPack)
     362          if filesOnly:
     363              for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
     364                  loader.load(zenPack, None)
     365
     366
     367      if not filesOnly:
     368          # Look for an installed ZenPack to be upgraded.  In this case
     369          # upgraded means that it is removed before the new one is installed
     370          # but that its objects are not removed and the packables are
     371          # copied to the new instance.
     372          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     373          if not existing and zenPack.prevZenPackName:
     374              existing = dmd.ZenPackManager.packs._getOb(
     375                                  zenPack.prevZenPackName, None)
     376
     377          deferFileDeletion = False
     378          packables = []
     379          upgradingFrom = None
     380          if existing:
     381              upgradingFrom = existing.version
     382              for p in existing.packables():
     383                  packables.append(p)
     384                  existing.packables.removeRelation(p)
     385              if existing.isEggPack():
     386                  forceNoFileDeletion = existing.eggPath() == dist.location
     387                  RemoveZenPack(dmd, existing.id,
     388                                  skipDepsCheck=True, leaveObjects=True,
     389                                  forceNoFileDeletion=forceNoFileDeletion,
     390                                  uninstallEgg=False)
     391              else:
     392                  # Don't delete files, might still be needed for
     393                  # migrate scripts to be run below.
     394                  deferFileDeletion = True
     395                  oldzenpack.RemoveZenPack(dmd, existing.id,
     396                                  skipDepsCheck=True, leaveObjects=True,
     397                                  deleteFiles=False)
     398          if runExternalZenpack:
     399              log.info("installing zenpack %s; launching process" % packName)
     400              cmd = [binPath('zenpack')]
     401              if link:
     402                  cmd += ["--link"]
     403              cmd += ["--install", eggPath]
     404              if upgradingFrom:
     405                  cmd += ['--previousversion', upgradingFrom]
     406
     407              cmdStr = " ".join(cmd)
     408              log.debug("launching sub process command: %s" % cmdStr)
     409              p = subprocess.Popen(cmdStr,
     410                              shell=True)
     411              out, err = p.communicate()
     412              p.wait()
     413              if p.returncode:
     414                  raise ZenPackException('Error installing the egg (%s): %s' %
     415                                         (p.returncode, err))
     416              dmd._p_jar.sync()
     417          else:
     418              dmd.ZenPackManager.packs._setObject(packName, zenPack)
     419              zenPack = dmd.ZenPackManager.packs._getOb(packName)
     420              #hack because ZenPack.install is overridden by a lot of zenpacks
     421              #so we can't change the signature of install to take the
     422              #previousVerison
     423              zenPack.prevZenPackVersion = previousVersion
     424              zenPack.install(dmd)
     425              zenPack.prevZenPackVersion = None
     426
     427
     428          try:
     429              zenPack = dmd.ZenPackManager.packs._getOb(packName)
     430              for p in packables:
     431                  pId = p.getPrimaryId()
     432                  try:
     433                      # make sure packable still exists; could be deleted by a
     434                      # migrate
     435                      getObjByPath(dmd, pId)
     436                      log.debug("adding packable relation for id %s", pId)
     437                      zenPack.packables.addRelation(p)
     438                  except (KeyError, zExceptions.NotFound):
     439                      log.debug('did not find packable %s',pId)
     440          except AttributeError, e:
     441              # If this happens in the child process or during the non-upgrade
     442              # flow, reraise the exception
     443              if not runExternalZenpack:
     444                  raise
     445
     446              # This specific error will occur when the version of the ZenPack
     447              # being installed subclasses Products.ZenModel.ZenPack, but the
     448              # previous version of the ZenPack did not.
     449              if str(e) == "'ZenPack' object has no attribute '__of__'":
     450                  zenPack = ZenPack(packName)
     451              else:
     452                  # This is the signature error of class-loading issues
     453                  # during zenpack upgrade.  The final state should be okay,
     454                  # except that modified packables may be lost.
     455                  message = "There has been an error during the post-" + \
     456                            "installation steps for the zenpack %s.  In " + \
     457                            "most cases, no further action is required.  If " + \
     458                            "issues persist, please reinstall this zenpack."
     459                  message = message % packName
     460                  log.warning( message )
     461                  raise NonCriticalInstallError( message )
     462
     463          if deferFileDeletion:
     464              # We skipped deleting the existing files from filesystem
     465              # because maybe they'd be needed in migrate scripts.
     466              # Delete them now
     467              oldZpDir = zenPath('Products', existing.id)
     468              if os.path.islink(oldZpDir):
     469                  os.remove(oldZpDir)
     470              else:
     471                  shutil.rmtree(oldZpDir)
     472
     473      cleanupSkins(dmd)
     474      transaction.commit()
     475      return zenPack
     476
     477
     478 -def DiscoverEggs(dmd, zenPackId):
     479      """
     480      Find installed eggs that provide a zenoss.zenpacks entry point.
     481      Return a list of distributions whose ZenPacks need to be installed
     482      or upgraded.  The list is sorted into the order in which this needs to
     483      happen.
     484      """
     485      # Create a set of all available zenoss.zenpack entries that aren't
     486      # already installed in zenoss or need to be upgraded in zenoss.
     487      entries = set()
     488      parse_version = pkg_resources.parse_version
     489      for entry in pkg_resources.iter_entry_points(ZENPACK_ENTRY_POINT):
     490          packName = entry.name
     491          packVers = entry.dist.version
     492          existing = dmd.ZenPackManager.packs._getOb(packName, None)
     493          if existing and existing.isEggPack():
     494              # We use >= to allow migrate to be run on currently installed
     495              # zenpacks whose version has been changed or for whom new
     496              # migrates have been added.
     497              if parse_version(packVers) >= parse_version(existing.version):
     498                  entries.add(entry)
     499          else:
     500              entries.add(entry)
     501
     502      # Starting with the entry representing zenPackId create a list of
     503      # all entrypoints
     504
     505      # orderedEntries lists entries in the opposite order of that in which
     506      # they need to be installed.  This is simply for convenience of using
     507      # .append() in code below.
     508      orderedEntries = []
     509      entriesByName = dict([(e.name, e) for e in entries])
     510
     511      def AddEntryAndProcessDeps(e):
     512          orderedEntries.append(e)
     513          for name in [r.project_name for r in e.dist.requires()]:
     514              if name in [e.name for e in orderedEntries]:
     515                  # This entry depends on something that we've already processed.
     516                  # This might be a circular dependency, might not be.
     517                  # We are just going to bail however.  This should be
     518                  # very unusual and the user can install deps first to work
     519                  # around.
     520                  raise ZenPackException('Unable to resolve ZenPack dependencies.'
     521                      ' Try installing dependencies first.')
     522              if name in entriesByName:
     523                  # The requirement is an entry that has not yet been processed
     524                  # here.  Add it to the list of entries to install/upgrade.
     525                  AddEntryAndProcessDeps(entriesByName[name])
     526              else:
     527                  # The requirement is not in the entries generated above.
     528                  # This either means that the dep is already installed (this
     529                  # is likely) or that easy_install missed something and the dep
     530                  # is not installed/available (this should be unlikely.)
     531                  pass
     532
     533      if zenPackId not in entriesByName:
     534          if zenPackId in dmd.ZenPackManager.packs.objectIds():
     535              return []
     536          else:
     537              raise ZenPackException('Unable to discover ZenPack named %s' %
     538                                      zenPackId)
     539      AddEntryAndProcessDeps(entriesByName[zenPackId])
     540      orderedEntries.reverse()
     541      return [e.dist for e in orderedEntries]
     542
     543
     544 -def AddDistToWorkingSet(distPath):
     545      """
     546      Given the path to a dist (an egg) add it to the current working set.
     547      This is basically a pkg_resources-friendly way of adding it to
     548      sys.path.
     549      Return a list of all distributions on distPath that appear to
     550      be ZenPacks.
     551      """
     552      zpDists = []
     553      for d in pkg_resources.find_distributions(distPath):
     554          pkg_resources.working_set.add(d)
     555          pkg_resources.require(d.project_name)
     556          if d.project_name.startswith('ZenPacks.'):
     557              zpDists.append(d)
     558      return zpDists
     559
     560
     561 -def ReadZenPackInfo(dist):
     562      """
     563      Return a dictionary containing the egg metadata
     564      """
     565      info = {}
     566      if dist.has_metadata('PKG-INFO'):
     567          lines = dist.get_metadata('PKG-INFO')
     568          for line in pkg_resources.yield_lines(lines):
     569              key, value = line.split(':', 1)
     570              info[key.strip()] = value.strip()
     571      if dist.has_metadata('zenpack_info'):
     572          lines = dist.get_metadata('zenpack_info')
     573          for line in pkg_resources.yield_lines(lines):
     574              key, value = line.split(':', 1)
     575              info[key.strip()] = value.strip()
     576      return info
     577
     578
     579 -def CopyMetaDataToZenPackObject(dist, pack):
     580      """
     581      Copy metadata type stuff from the distribution to the zp object.
     582      """
     583      # Version
     584      pack.version = dist.version
     585
     586      # Egg Info
     587      info = ReadZenPackInfo(dist)
     588      pack.author = info.get('Author', '')
     589      if pack.author == 'UNKNOWN':
     590          pack.author = ''
     591      pack.compatZenossVers = info.get('compatZenossVers', '')
     592      pack.prevZenPackName = info.get('prevZenPackName', '')
     593
     594      # Requires
     595      pack.dependencies = {}
     596      for r in dist.requires():
     597          name = r.project_name
     598          spec = str(r)[len(name):]
     599          pack.dependencies[name] = spec
     600
     601
     602 -def CreateZenPacksDir():
     603      """
     604      Make sure $ZENHOME/ZenPacks exists
     605      """
     606      zpDir = zenPath('ZenPacks')
     607      if not os.path.isdir(zpDir):
     608          os.mkdir(zpDir, 0750)
     609
     610
     611 -def DoEasyInstall(eggPath):
     612      """
     613      Use easy_install to install an egg from the filesystem.
     614      easy_install will install the egg, but does not install it into
     615      Zenoss as ZenPacks.
     616      Returns a list of distributions that were installed that appear
     617      to be ZenPacks.
     618      """
     619      from setuptools.command import easy_install
     620
     621      # Make sure $ZENHOME/ZenPacks exists
     622      CreateZenPacksDir()
     623
     624      # Create temp file for easy_install to write results to
     625      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     626      # eggPaths is a set of paths to eggs that were installed.  We need to
     627      # add them to the current workingset so we can discover their
     628      # entry points.
     629      eggPaths = set()
     630      try:
     631          # Execute the easy_install
     632          args = ['--site-dirs', zenPath('ZenPacks'),
     633              '-d', zenPath('ZenPacks'),
     634              # '-i', ZEN_PACK_INDEX_URL,
     635              '--allow-hosts', 'None',
     636              '--record', tempPath,
     637              '--quiet',
     638              eggPath]
     639          easy_install.main(args)
     640          # Collect the paths for eggs that were installed
     641          f = open(tempPath, 'r')
     642          marker = '.egg/'
     643          markerOffset = len(marker)-1
     644          for l in f.readlines():
     645              i = l.find(marker)
     646              if i > 0:
     647                  eggPaths.add(l[:i+markerOffset])
     648      finally:
     649          os.remove(tempPath)
     650      # Add any installed eggs to the current working set
     651      zpDists = []
     652      for path in eggPaths:
     653          zpDists += AddDistToWorkingSet(path)
     654      return zpDists
     655
     656
     657  ########################################
     658  #   Zenoss.Net
     659  ########################################
     660
     661
     662 -def FetchAndInstallZenPack(dmd, zenPackName, zenPackVersion='', sendEvent=True):
     663      """
     664      Fetch the named zenpack and all its dependencies and install them.
     665      Return a list of the ZenPacks that were installed.
     666      """
     667      zenPacks = []
     668      try:
     669          zpDists = FetchZenPack(zenPackName, zenPackVersion)
     670          for d in zpDists:
     671              zenPacks.append(InstallDistAsZenPack(dmd, d))
     672      except:
     673          if sendEvent:
     674              ZPEvent(dmd, 4, 'Failed to install ZenPack %s' % zenPackName,
     675                  '%s: %s' % sys.exc_info()[:2])
     676          raise
     677      if sendEvent:
     678          zenPackIds = [z.id for z in zenPacks]
     679          if zenPackIds:
     680              ZPEvent(dmd, 2, 'Installed ZenPacks: %s' % ', '.join(zenPackIds))
     681          if zenPackName not in zenPackIds:
     682              ZPEvent(dmd, 4, 'Unable to install ZenPack %s' % zenPackName)
     683      return zenPacks
     684
     685
     686 -def FetchZenPack(zenPackName, zenPackVersion=''):
     687      """
     688      Use easy_install to retrieve the given zenpack and any dependencies.
     689      easy_install will install the eggs, but does not install them into
     690      Zenoss as ZenPacks.
     691      Return a list of distributions just installed that appear to be
     692      ZenPacks.
     693
     694      NB: This should be refactored.  It shares most of its code with
     695      DoEasyInstall()
     696      """
     697      from setuptools.command import easy_install
     698
     699      # Make sure $ZENHOME/ZenPacks exists
     700      CreateZenPacksDir()
     701
     702      # Create temp file for easy_install to write results to
     703      _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall')
     704      # eggPaths is a set of paths to eggs that were installed.  We need to
     705      # add them to the current workingset so we can discover their
     706      # entry points.
     707      eggPaths = set()
     708      try:
     709          # Execute the easy_install
     710          args = ['--site-dirs', zenPath('ZenPacks'),
     711              '-d', zenPath('ZenPacks'),
     712              '-i', ZEN_PACK_INDEX_URL,
     713              '--allow-hosts', 'None',
     714              '--record', tempPath,
     715              '--quiet',
     716              zenPackName]
     717          easy_install.main(args)
     718          # Collect the paths for eggs that were installed
     719          f = open(tempPath, 'r')
     720          marker = '.egg/'
     721          markerOffset = len(marker)-1
     722          for l in f.readlines():
     723              i = l.find(marker)
     724              if i > 0:
     725                  eggPaths.add(l[:i+markerOffset])
     726      finally:
     727          os.remove(tempPath)
     728      # Add any installed eggs to the current working set
     729      zpDists = []
     730      for path in eggPaths:
     731          zpDists += AddDistToWorkingSet(path)
     732      return zpDists
     733
     734
     735 -def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):
     736      """
     737      Upload the specified zenpack to the given project.
     738      Project is a string of the form 'enterprise/myproject' or
     739      'community/otherproject'.
     740      """
     741      zp = dmd.ZenPackManager.packs._getOb(packName, None)
     742      if not zp:
     743          raise ZenPackException('No ZenPack named %s' % packName)
     744
     745      # Export the zenpack
     746      fileName = zp.manage_exportPack()
     747      filePath = zenPath('export', fileName)
     748
     749      # Login to Zenoss.net
     750      from DotNetCommunication import DotNetSession
     751      session = DotNetSession()
     752      userSettings = dmd.ZenUsers.getUserSettings()
     753      session.login(znetUser, znetPass)
     754
     755      # Upload
     756      zpFile = open(zenPath('export', fileName), 'r')
     757      try:
     758          response = session.open('%s/createRelease' % project.strip('/'), {
     759                  'description': description,
     760                  'fileStorage': zpFile,
     761                  })
     762      finally:
     763          zpFile.close()
     764      if response:
     765          result = response.read()
     766          if "'success':true" not in result:
     767              raise ZenPackException('Upload failed')
     768      else:
     769          raise ZenPackException('Failed to connect to Zenoss.net')
     770      return
     771
     772
     773  ########################################
     774  #   ZenPack Removal
     775  ########################################
     776
     777
     778 -def RemoveZenPack(dmd, packName, filesOnly=False, skipDepsCheck=False,
     779                      leaveObjects=False, sendEvent=True,
     780                      forceNoFileDeletion=False, uninstallEgg=True):
     781      """
     782      Remove the given ZenPack from Zenoss.
     783      Whether the ZenPack will be removed from the filesystem or not
     784      depends on the result of the ZenPack's shouldDeleteFilesOnRemoval method.
     785      """
     786      try:
     787          if filesOnly:
     788              skipDepsCheck = True
     789
     790          # Check for dependency implications here?
     791          if not skipDepsCheck:
     792              deps = GetDependents(dmd, packName)
     793              if deps:
     794                  raise ZenPackDependentsException('%s cannot be removed ' % packName +
     795                          'because it is required by %s' % ', '.join(deps))
     796
     797          if not filesOnly:
     798              # Fetch the zenpack, call its remove() and remove from packs
     799              zp = None
     800              try:
     801                  zp = dmd.ZenPackManager.packs._getOb(packName)
     802              except AttributeError, ex:
     803                  raise ZenPackNotFoundException('No ZenPack named %s is installed' %
     804                                                  packName)
     805              # If zencatalog hasn't finished yet, we get ugly messages that don't
     806              # mean anything. Hide them.
     807              logFilter = None
     808              if not getattr(dmd.zport, '_zencatalog_completed', False):
     809                  logFilter = CatalogLoggingFilter()
     810                  logging.getLogger('Zope.ZCatalog').addFilter(logFilter)
     811              try:
     812                  zp.remove(dmd, leaveObjects)
     813                  dmd.ZenPackManager.packs._delObject(packName)
     814                  transaction.commit()
     815              finally:
     816                  # Remove our logging filter so we don't hide anything important
     817                  if logFilter is not None:
     818                      logging.getLogger('Zope.ZCatalog').removeFilter(logFilter)
     819
     820          # Uninstall the egg and possibly delete it
     821          # If we can't find the distribution then apparently the zp egg itself is
     822          # missing.  Continue on with the removal and swallow the
     823          # DistributionNotFound exception
     824          try:
     825              dist = zp.getDistribution()
     826          except pkg_resources.DistributionNotFound:
     827              dist = None
     828          if dist:
     829              # Determine deleteFiles before develop -u gets called.  Once
     830              # it is called the egg has problems figuring out some of it's state.
     831              deleteFiles = zp.shouldDeleteFilesOnRemoval()
     832              if uninstallEgg:
     833                  if zp.isDevelopment():
     834                      zenPackDir = zenPath('ZenPacks')
     835                      cmd = ('%s setup.py develop -u '
     836                              % binPath('python') +
     837                              '--site-dirs=%s ' % zenPackDir +
     838                              '-d %s' % zenPackDir)
     839                      p = subprocess.Popen(cmd,
     840                                          stdout=subprocess.PIPE,
     841                                          stderr=subprocess.PIPE,
     842                                          shell=True,
     843                                          cwd=zp.eggPath())
     844                      out, err = p.communicate()
     845                      code = p.wait()
     846                      if code:
     847                          raise ZenPackException(err)
     848                  else:
     849                      DoEasyUninstall(packName)
     850              # elif cleanupEasyInstallPth:
     851              #     # Do we need to call easy_install -m here?  It causes problems
     852              #     # because it tries to install deps.  Cleanup easy-install.pth
     853              #     # ourselves instead.
     854              #     # We don't want to cleanup easy-install.pth when a newer
     855              #     # version of the egg has already been installed (when doing
     856              #     # an upgrade or installing in new location.)
     857              #     eggLink = './%s' % zp.eggName()
     858              #     CleanupEasyInstallPth(eggLink)
     859              if deleteFiles and not forceNoFileDeletion:
     860                  eggDir = zp.eggPath()
     861                  if os.path.islink(eggDir):
     862                      os.remove(eggDir)
     863                  else:
     864                      shutil.rmtree(eggDir)
     865          cleanupSkins(dmd)
     866          transaction.commit()
     867      except:
     868          if sendEvent:
     869              ZPEvent(dmd, 4, 'Error removing ZenPack %s' % packName,
     870                  '%s: %s' % sys.exc_info()[:2])
     871          raise
     872      if sendEvent:
     873          ZPEvent(dmd, 2, 'Removed ZenPack %s' % packName)
     874
     875
     876 -def DoEasyUninstall(name):
     877      """
     878      Execute the easy_install command to unlink the given egg.
     879      What this is really doing is switching the egg to be in
     880      multiple-version mode, however this is the first step in deleting
     881      an egg as described here:
     882      http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages
     883      """
     884      from setuptools.command import easy_install
     885      args = ['--site-dirs', zenPath('ZenPacks'),
     886          '-d', zenPath('ZenPacks'),
     887          #'--record', tempPath,
     888          '--quiet',
     889          '-m',
     890          name]
     891      easy_install.main(args)
     892
     893
     894 -def CanRemoveZenPacks(dmd, packNames):
     895      """
     896      Returns a tuple of (canRemove, otherDependents)
     897      canRemove is True if the listed zenPacks have no dependents not also
     898      listed in packNames, False otherwise.
     899      otherDependents is a list of zenpack names not in packNames that
     900      depend on one or more of the packs in packNames.
     901      """
     902      unhappy = set()
     903      for name in packNames:
     904          deps = GetDependents(dmd, name)
     905          unhappy.update(set([dep for dep in deps if dep not in packNames]))
     906      return (not unhappy and True or False, list(unhappy))
     907
     908
     909  # def CleanupEasyInstallPth(eggLink):
     910  #     """
     911  #     Remove the entry for the given egg from the
     912  #     $ZENHOME/ZenPacks/easy-install.pth file.  If this entry is left
     913  #     in place in can cause problems during the next install of the same
     914  #     egg.
     915  #     """
     916  #     # Remove the path from easy-install.pth
     917  #     eggTail = os.path.split(eggLink)[1]
     918  #     easyPth = zenPath('ZenPacks', 'easy-install.pth')
     919  #     if os.path.isfile(easyPth):
     920  #         needToWrite = False
     921  #         newLines = []
     922  #         f = open(easyPth, 'r')
     923  #         for line in f:
     924  #             if os.path.split(line.strip())[1] == eggTail:
     925  #                 needToWrite = True
     926  #             else:
     927  #                 newLines.append(line)
     928  #         f.close()
     929  #         if needToWrite:
     930  #             f = open(easyPth, 'w')
     931  #             f.writelines(newLines)
     932  #             f.close()
     933
     934
     935 -def GetDependents(dmd, packName):
     936      """
     937      Return a list of installed ZenPack ids that list packName as a dependency
     938      """
     939      return [zp.id for zp in dmd.ZenPackManager.packs()
     940                  if zp.id != packName and zp.dependencies.has_key(packName)]
     941
     942
     943  ########################################
     944  #   __main__, dispatching, etc
     945  ########################################
     946
     947
     948 -def ZPEvent(dmd, severity, summary, message=None):
     949      """
     950      Send an event to Zenoss.
     951      """
     952      dmd.ZenEventManager.sendEvent(dict(
     953          device=FQDN,
     954          eventClass='/Unknown',
     955          severity=severity,
     956          summary=summary,
     957          message=message))
     958
     959
     960 -class ZenPackCmd(ZenScriptBase):
     961      """
     962      Utilities for creating, installing, removing ZenPacks.
     963
     964      NOTE: Users will probably invoke zenpack from the command line, which
     965      runs zenpack.py rather than this file.  zenpack.py calls functions
     966      in this module when it detects that new-style (egg) ZenPacks are involved.
     967      The plan is that once support for old-style (non-egg) ZenPacks is dropped
     968      zenpack.py can go away and this will take its place.  Until then this
     969      script can be invoked directly via the zenpackcmd script if desired.
     970      Invoking this script directly has the benefit of slightly better
     971      progress/status output to stdout.
     972      """
     973
     974 -    def run(self):
     975          """
     976          Execute the user's request.
     977          """
     978
     979          self.connect()
     980          def PrintInstalled(installed, eggOnly=False):
     981              if installed:
     982                  if eggOnly:
     983                      names = [i['id'] for i in installed]
     984                      what = 'ZenPack egg'
     985                  else:
     986                      names = [i.id for i in installed]
     987                      what = 'ZenPack'
     988                  print('Installed %s%s: %s' % (
     989                          what,
     990                          len(names) != 1 and 's' or '',
     991                          ', '.join(names)))
     992              else:
     993                  print('No ZenPacks installed.')
     994
     995          if not getattr(self.dmd, 'ZenPackManager', None):
     996              raise ZenPackNeedMigrateException('Your Zenoss database appears'
     997                  ' to be out of date. Try running zenmigrate to update.')
     998          if self.options.eggOnly and self.options.eggPath:
     999              zpDists = InstallEgg(self.dmd, self.options.eggPath,
    1000                                              link=self.options.link)
    1001              PrintInstalled([{'id':d.project_name} for d in zpDists],
    1002                              eggOnly=True)
    1003          if self.options.eggPath:
    1004              installed = InstallEggAndZenPack(
    1005                                  self.dmd, self.options.eggPath,
    1006                                  link=self.options.link,
    1007                                  filesOnly=self.options.filesOnly,
    1008                                  previousVersion= self.options.previousVersion)
    1009              PrintInstalled(installed)
    1010          elif self.options.fetch:
    1011              installed = FetchAndInstallZenPack(self.dmd, self.options.fetch)
    1012              PrintInstalled(installed)
    1013          elif self.options.upload:
    1014              return UploadZenPack(self.dmd, self.options.upload,
    1015                                      self.options.znetProject,
    1016                                      self.options.uploadDesc,
    1017                                      self.options.znetUser,
    1018                                      self.options.znetPass)
    1019          elif self.options.removePackName:
    1020              try:
    1021                  RemoveZenPack(self.dmd, self.options.removePackName)
    1022                  print('Removed ZenPack: %s' % self.options.removePackName)
    1023              except ZenPackNotFoundException, e:
    1024                  sys.stderr.write(str(e) + '\n')
    1025          elif self.options.list:
    1026              self.list()
    1027          else:
    1028              self.parser.print_help()
    1029
    1030
    1031 -    def buildOptions(self):
    1032          self.parser.add_option('--install',
    1033                                 dest='eggPath',
    1034                                 default=None,
    1035                                 help="name of the pack to install")
    1036  #        self.parser.add_option('--fetch',
    1037  #                               dest='fetch',
    1038  #                               default=None,
    1039  #                               help='Name of ZenPack to retrieve from '
    1040  #                                    'Zenoss.net and install.')
    1041  #        self.parser.add_option('--fetch-vers',
    1042  #                               dest='fetchVers',
    1043  #                               default=None,
    1044  #                               help='Use with --fetch to specify a version'
    1045  #                                    ' for the ZenPack to download and install.')
    1046  #        self.parser.add_option('--znet-user',
    1047  #                               dest='znetUser',
    1048  #                               default=None,
    1049  #                               help='Use with --fetch or --upload to specify'
    1050  #                                    ' your Zenoss.net username.')
    1051  #        self.parser.add_option('--znet-pass',
    1052  #                               dest='znetPass',
    1053  #                               default=None,
    1054  #                               help='Use with --fetch or --upload to specify'
    1055  #                                    ' your Zenoss.net password.')
    1056  #        self.parser.add_option('--upload',
    1057  #                               dest='upload',
    1058  #                               default=None,
    1059  #                               help='Name of ZenPack to upload to '
    1060  #                                    'Zenoss.net')
    1061  #        self.parser.add_option('--znet-project',
    1062  #                               dest='znetProject',
    1063  #                               default=None,
    1064  #                               help='Use with --upload to specify'
    1065  #                                    ' which Zenoss.net project to create'
    1066  #                                    ' a release on.')
    1067  #        self.parser.add_option('--upload-desc',
    1068  #                               dest='uploadDesc',
    1069  #                               default=None,
    1070  #                               help='Use with --upload to provide'
    1071  #                                    ' a description for the new release.')
    1072          self.parser.add_option('--link',
    1073                                 dest='link',
    1074                                 action='store_true',
    1075                                 default=False,
    1076                                 help='Install the ZenPack in its current '
    1077                                  'location, do not copy to $ZENHOME/ZenPacks. '
    1078                                  'Also mark ZenPack as editable. '
    1079                                  'This only works with source directories '
    1080                                  'containing setup.py files, not '
    1081                                  'egg files.')
    1082          self.parser.add_option('--remove',
    1083                                 dest='removePackName',
    1084                                 default=None,
    1085                                 help="name of the pack to remove")
    1086          self.parser.add_option('--leave-objects',
    1087                                 dest='leaveObjects',
    1088                                 default=False,
    1089                                 action='store_true',
    1090                                 help="When specified with --remove then objects"
    1091                                      ' provided by the ZenPack and those'
    1092                                      ' depending on the ZenPack are not deleted.'
    1093                                      ' This may result in broken objects in your'
    1094                                      ' database unless the ZenPack is'
    1095                                      ' reinstalled.')
    1096          self.parser.add_option('--files-only',
    1097                                 dest='filesOnly',
    1098                                 action="store_true",
    1099                                 default=False,
    1100                                 help='install onto filesystem but not into '
    1101                                          'zenoss')
    1102          self.parser.add_option('--previousversion',
    1103                                 dest='previousVersion',
    1104                                 default=None,
    1105                                 help="Previous version of the zenpack;"
    1106                                      ' used during upgrades')
    1107          ZenScriptBase.buildOptions(self)
    1108
    1109
    1110  if __name__ == '__main__':
    1111      try:
    1112          zp = ZenPackCmd()
    1113          zp.run()
    1114      except ZenPackException, e:
    1115          sys.stderr.write('%s\n' % str(e))
    1116          sys.exit(-1)
    1117