Products.ZenUtils.ZenPackCmd

1 ############################################################################## 2 # 3 # Copyright (C) Zenoss, Inc. 2008, all rights reserved. 4 # 5 # This content is made available according to terms specified in 6 # License.zenoss under the directory where your Zenoss product is installed. 7 # 8 ############################################################################## 9 10 11 __doc__ = "Manage ZenPacks" 12 13 import Globals 14 from ZODB.transact import transact 15 from Products.ZenUtils.ZenScriptBase import ZenScriptBase 16 from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, getObjByPath 17 18 from Products.ZenModel import ZVersion 19 from Products.ZenModel.ZenPack import ZenPackException, \ 20 ZenPackNotFoundException, \ 21 ZenPackNeedMigrateException 22 from Products.ZenModel.ZenPack import ZenPackDependentsException 23 from Products.ZenModel.ZenPack import ZenPack 24 from Products.ZenUtils.PkgResources import pkg_resources 25 from Products.Zuul.utils import CatalogLoggingFilter 26 import Products.ZenModel.ZenPackLoader as ZPL 27 import zenpack as oldzenpack 28 import transaction 29 import os, sys 30 import shutil 31 import string 32 import tempfile 33 import subprocess 34 import socket 35 import logging 36 import zExceptions 37 38 from urlparse import urlparse 39 40 41 log = logging.getLogger('zen.ZenPackCMD') 42 43 #import zenpacksupport 44 45 FQDN = socket.getfqdn() 46 47 ZENPACKS_BASE_URL = 'http://zenpacks.zenoss.com/pypi' 48 49 # All ZenPack eggs have to define exactly one entry point in this group. 50 ZENPACK_ENTRY_POINT = 'zenoss.zenpacks'

51 52 ######################################## 53 # ZenPack Creation 54 ######################################## 55 56 -def CreateZenPack(zpId, prevZenPackName=''):

57 """ 58 Create the zenpack in the filesystem. 59 The zenpack is not installed in Zenoss, it is simply created in 60 the $ZENHOME/ZenPacks directory. Usually this should be followed 61 with a "zenpack install" call. 62 zpId should already be valid, scrubbed value. 63 prevZenPackName is written to PREV_ZENPACK_NAME in setup.py. 64 """ 65 parts = zpId.split('.') 66 67 # Copy template to $ZENHOME/ZenPacks 68 srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate') 69 devDir = zenPath('ZenPacks') 70 if not os.path.exists(devDir): 71 os.mkdir(devDir, 0750) 72 destDir = os.path.join(devDir, zpId) 73 shutil.copytree(srcDir, destDir, symlinks=False) 74 os.system('find %s -name .svn | xargs rm -rf' % destDir) 75 76 # Write setup.py 77 packages = [] 78 for i in range(len(parts)): 79 packages.append('.'.join(parts[:i+1])) 80 mapping = dict( 81 NAME = zpId, 82 VERSION = '1.0.0', 83 AUTHOR = '', 84 LICENSE = '', 85 NAMESPACE_PACKAGES = packages[:-1], 86 PACKAGES = packages, 87 INSTALL_REQUIRES = [], 88 COMPAT_ZENOSS_VERS = '', 89 PREV_ZENPACK_NAME = prevZenPackName, 90 ) 91 WriteSetup(os.path.join(destDir, 'setup.py'), mapping) 92 93 # Create subdirectories 94 base = destDir 95 for part in parts[:-1]: 96 base = os.path.join(base, part) 97 os.mkdir(base) 98 f = open(os.path.join(base, '__init__.py'), 'w') 99 f.write("__import__('pkg_resources').declare_namespace(__name__)\n") 100 f.close() 101 base = os.path.join(base, parts[-1]) 102 shutil.move(os.path.join(destDir, 'CONTENT'), base) 103 104 return destDir

105

106 107 -def WriteSetup(setupPath, values):

108 """ 109 """ 110 f = file(setupPath, 'r') 111 lines = f.readlines() 112 f.close() 113 114 newLines = [] 115 for i, line in enumerate(lines): 116 if line.startswith('STOP_REPLACEMENTS'): 117 newLines += lines[i:] 118 break 119 key = line.split('=')[0].strip() 120 if key in values: 121 value = values[key] 122 if isinstance(value, basestring): 123 fmt = '%s = "%s"\n' 124 else: 125 fmt = '%s = %s\n' 126 newLines.append(fmt % (key, value)) 127 else: 128 newLines.append(line) 129 130 f = file(setupPath, 'w') 131 f.writelines(newLines) 132 f.close()

133

134 135 -def CanCreateZenPack(dmd, zpId):

136 """ 137 Return tuple (bool, string) where first element is true if a new zenpack 138 can be created with the given info and false if not. If first element 139 is True then the second part of the tuple contains the scrubbed ZenPack id. 140 If the first part is False then the second contains an explanatory 141 message. 142 """ 143 # Check if id and package looks reasonable 144 (allowable, idOrMsg) = ScrubZenPackId(zpId) 145 if allowable: 146 zpId = idOrMsg 147 else: 148 return (False, idOrMsg) 149 150 # Is the id already in use? 151 if dmd: 152 if zpId in dmd.ZenPackManager.packs.objectIds(): 153 return (False, 'A ZenPack named %s already exists.' % zpId) 154 155 # Is there another zenpack in the way? 156 # Now that zenpacks are created in $ZENHOME/ZenPacks instead of 157 # $ZENHOME/ZenPackDev this may no longer be necessary because a 158 # zp in the way should be installed and caught by the already in use 159 # check above. 160 if os.path.exists(zenPath('ZenPacks', zpId)): 161 return (False, 'A directory named %s already exists' % zpId + 162 ' in $ZENHOME/ZenPacks. Use a different name' 163 ' or remove that directory.') 164 165 return (True, idOrMsg)

166

167 168 -def ScrubZenPackId(name):

169 """ 170 If the given name conforms to ZenPack naming rules, or can easily be 171 modified to do so, then return (True, scrubbedName) where scrubbedName 172 is either name or a slightly modified name. If the given name does 173 not conform to naming rules and we can't easily modify it to do so 174 then return (False, errorMsg) where errorMsg describes why name 175 is unacceptable. 176 """ 177 parts = name.split('.') 178 179 # Remove leading dots, trailing dots, adjacent dots and strip whitespace 180 # from each part 181 parts = [p.strip() for p in parts] 182 parts = [p for p in parts if p] 183 184 # Add/fix leading 'ZenPacks' 185 if parts[0] != 'ZenPacks': 186 if parts[0].lower() == 'zenpacks': 187 parts[0] = 'ZenPacks' 188 else: 189 parts.insert(0, 'ZenPacks') 190 191 # Must be at least 3 parts 192 if len(parts) < 3: 193 return (False, 'ZenPack names must contain at least three package ' 194 'names separated by periods.') 195 196 # Each part must start with a letter 197 for p in parts: 198 if p[0] not in string.letters: 199 return (False, 'Each package name must start with a letter.') 200 201 # Only letters, numbers and underscores in each part 202 allowable = string.letters + string.digits + '_' 203 for p in parts: 204 for c in p: 205 if c not in allowable: 206 return (False, 'Package names may only contain letters, ' 207 'numbers and underscores.') 208 209 return (True, '.'.join(parts))

210

211 212 ######################################## 213 # ZenPack Installation 214 ######################################## 215 216 -class NonCriticalInstallError(Exception):

217 - def __init__(self, message):

218 Exception.__init__(self, message) 219 self.message = message

220

221 -def InstallEggAndZenPack(dmd, eggPath, link=False, 222 filesOnly=False, sendEvent=True, 223 previousVersion=None, forceRunExternal=False, 224 fromUI=False):

225 """ 226 Installs the given egg, instantiates the ZenPack, installs in 227 dmd.ZenPackManager.packs, and runs the zenpacks's install method. 228 Returns a list of ZenPacks that were installed. 229 """ 230 zenPacks = [] 231 nonCriticalErrorEncountered = False 232 try: 233 zpDists = InstallEgg(dmd, eggPath, link=link) 234 for d in zpDists: 235 try: 236 zp = InstallDistAsZenPack(dmd, 237 d, 238 eggPath, 239 link, 240 filesOnly=filesOnly, 241 previousVersion=previousVersion, 242 forceRunExternal=forceRunExternal, 243 fromUI=fromUI) 244 zenPacks.append(zp) 245 except NonCriticalInstallError, ex: 246 nonCriticalErrorEncountered = True 247 if sendEvent: 248 ZPEvent(dmd, 3, ex.message) 249 except Exception, e: 250 # Get that exception out there in case it gets blown away by ZPEvent 251 log.exception("Error installing ZenPack %s" % eggPath) 252 if sendEvent: 253 ZPEvent(dmd, 4, 'Error installing ZenPack %s' % eggPath, 254 '%s: %s' % sys.exc_info()[:2]) 255 # Don't just raise, because if ZPEvent blew away exception context 256 # it'll be None, which is bad. This manipulates the stack to look like 257 # this is the source of the exception, but we logged it above so no 258 # info is lost. 259 raise e 260 if sendEvent: 261 zenPackIds = [zp.id for zp in zenPacks] 262 if zenPackIds: 263 ZPEvent(dmd, 2, 'Installed ZenPacks %s' % ','.join(zenPackIds)) 264 elif not nonCriticalErrorEncountered: 265 ZPEvent(dmd, 4, 'Unable to install %s' % eggPath) 266 return zenPacks

267

268 269 -def InstallEgg(dmd, eggPath, link=False):

270 """ 271 Install the given egg and add to the current working set. 272 This does not install the egg as a ZenPack. 273 Return a list of distributions that should be installed as ZenPacks. 274 """ 275 eggPath = os.path.abspath(eggPath) 276 zenPackDir = zenPath('ZenPacks') 277 eggInZenPacksDir = eggPath.startswith(zenPackDir + '/') 278 279 # Make sure $ZENHOME/ZenPacks exists 280 CreateZenPacksDir() 281 282 # Install the egg 283 if link: 284 cmd = ('%s setup.py develop ' % binPath('python') + 285 '--site-dirs=%s ' % zenPackDir + 286 '-d %s' % zenPackDir) 287 p = subprocess.Popen(cmd, 288 stdout=subprocess.PIPE, 289 stderr=subprocess.PIPE, 290 shell=True, 291 cwd=eggPath) 292 out, err = p.communicate() 293 p.wait() 294 if p.returncode: 295 DoEasyUninstall(eggPath) 296 raise ZenPackException('Error installing the egg (%s): %s' % 297 (p.returncode, err)) 298 zpDists = AddDistToWorkingSet(eggPath) 299 else: 300 try: 301 zpDists = DoEasyInstall(eggPath) 302 except: 303 DoEasyUninstall(eggPath) 304 raise 305 # cmd = 'easy_install --always-unzip --site-dirs=%s -d %s %s' % ( 306 # zenPackDir, 307 # zenPackDir, 308 # eggPath) 309 # p = subprocess.Popen(cmd, 310 # stdout=subprocess.PIPE, 311 # stderr=subprocess.PIPE, 312 # shell=True) 313 # p.wait() 314 # eggName = os.path.split(eggPath)[1] 315 # eggPath = os.path.join(zenPackDir, eggName) 316 return zpDists

317

318 319 # def GetZenPackNamesFromEggPath(eggPath): 320 # """ 321 # Given a path to a ZenPack egg (installed or not) return the 322 # name of the ZenPack it contains. 323 # """ 324 # zpNames = [] 325 # for d in pkg_resources.find_distributions(eggPath) 326 # if d.project_name.startswith('ZenPacks.'): 327 # zpNames.append(d.project_name) 328 # return zpNames 329 330 331 -def InstallDistAsZenPack(dmd, dist, eggPath, link=False, filesOnly=False, 332 previousVersion=None, forceRunExternal=False, 333 fromUI=False):

334 """ 335 Given an installed dist, install it into Zenoss as a ZenPack. 336 Return the ZenPack instance. 337 """ 338 @transact 339 def transactional_actions(): 340 # Instantiate ZenPack 341 entryMap = pkg_resources.get_entry_map(dist, ZENPACK_ENTRY_POINT) 342 if not entryMap or len(entryMap) > 1: 343 raise ZenPackException('A ZenPack egg must contain exactly one' 344 ' zenoss.zenpacks entry point. This egg appears to contain' 345 ' %s such entry points.' % len(entryMap)) 346 packName, packEntry = entryMap.items()[0] 347 runExternalZenpack = True 348 #if zenpack with same name exists we can't load both modules 349 #installing new egg zenpack will be done in a sub process 350 existing = dmd.ZenPackManager.packs._getOb(packName, None) 351 if existing: 352 log.info("Previous ZenPack exists with same name %s" % packName) 353 if filesOnly or not existing: 354 #running files only or zenpack by same name doesn't already exists 355 # so no need to install the zenpack in an external process 356 runExternalZenpack = False 357 module = packEntry.load() 358 if hasattr(module, 'ZenPack'): 359 zenPack = module.ZenPack(packName) 360 else: 361 zenPack = ZenPack(packName) 362 zenPack.eggPack = True 363 CopyMetaDataToZenPackObject(dist, zenPack) 364 if filesOnly: 365 for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()): 366 loader.load(zenPack, None) 367 if fromUI and not zenPack.installableFromUI: 368 raise ZenPackException("This ZenPack cannot be installed through the UI.") 369 370 371 if not filesOnly: 372 # Look for an installed ZenPack to be upgraded. In this case 373 # upgraded means that it is removed before the new one is installed 374 # but that its objects are not removed and the packables are 375 # copied to the new instance. 376 existing = dmd.ZenPackManager.packs._getOb(packName, None) 377 if not existing and zenPack.prevZenPackName: 378 existing = dmd.ZenPackManager.packs._getOb( 379 zenPack.prevZenPackName, None) 380 381 deferFileDeletion = False 382 packables = [] 383 upgradingFrom = None 384 if existing: 385 upgradingFrom = existing.version 386 for p in existing.packables(): 387 packables.append(p) 388 existing.packables.removeRelation(p) 389 if existing.isEggPack(): 390 forceNoFileDeletion = existing.eggPath() == dist.location 391 RemoveZenPack(dmd, existing.id, 392 skipDepsCheck=True, leaveObjects=True, 393 forceNoFileDeletion=forceNoFileDeletion, 394 uninstallEgg=False) 395 else: 396 # Don't delete files, might still be needed for 397 # migrate scripts to be run below. 398 deferFileDeletion = True 399 oldzenpack.RemoveZenPack(dmd, existing.id, 400 skipDepsCheck=True, leaveObjects=True, 401 deleteFiles=False) 402 if runExternalZenpack or forceRunExternal: 403 log.info("installing zenpack %s; launching process" % packName) 404 cmd = [binPath('zenpack')] 405 if link: 406 cmd += ["--link"] 407 cmd += ["--install", eggPath] 408 if upgradingFrom: 409 cmd += ['--previousversion', upgradingFrom] 410 if fromUI: 411 cmd += ["--fromui"] 412 413 cmdStr = " ".join(cmd) 414 log.debug("launching sub process command: %s" % cmdStr) 415 p = subprocess.Popen(cmdStr, 416 shell=True) 417 out, err = p.communicate() 418 p.wait() 419 if p.returncode: 420 raise ZenPackException('Error installing the egg (%s): %s' % 421 (p.returncode, err)) 422 dmd._p_jar.sync() 423 else: 424 dmd.ZenPackManager.packs._setObject(packName, zenPack) 425 zenPack = dmd.ZenPackManager.packs._getOb(packName) 426 #hack because ZenPack.install is overridden by a lot of zenpacks 427 #so we can't change the signature of install to take the 428 #previousVerison 429 zenPack.prevZenPackVersion = previousVersion 430 zenPack.install(dmd) 431 zenPack.prevZenPackVersion = None 432 433 try: 434 zenPack = dmd.ZenPackManager.packs._getOb(packName) 435 for p in packables: 436 pId = p.getPrimaryId() 437 try: 438 # make sure packable still exists; could be deleted by a 439 # migrate 440 getObjByPath(dmd, pId) 441 log.debug("adding packable relation for id %s", pId) 442 zenPack.packables.addRelation(p) 443 except (KeyError, zExceptions.NotFound): 444 log.debug('did not find packable %s',pId) 445 except AttributeError, e: 446 # If this happens in the child process or during the non-upgrade 447 # flow, reraise the exception 448 if not runExternalZenpack: 449 raise 450 451 # This specific error will occur when the version of the ZenPack 452 # being installed subclasses Products.ZenModel.ZenPack, but the 453 # previous version of the ZenPack did not. 454 if str(e) == "'ZenPack' object has no attribute '__of__'": 455 zenPack = ZenPack(packName) 456 else: 457 # This is the signature error of class-loading issues 458 # during zenpack upgrade. The final state should be okay, 459 # except that modified packables may be lost. 460 message = "There has been an error during the post-" + \ 461 "installation steps for the zenpack %s. In " + \ 462 "most cases, no further action is required. If " + \ 463 "issues persist, please reinstall this zenpack." 464 message = message % packName 465 log.warning( message ) 466 raise NonCriticalInstallError( message ) 467 468 cleanupSkins(dmd) 469 return zenPack, deferFileDeletion, existing

470 471 zenPack, deferFileDeletion, existing = transactional_actions() 472 473 if not filesOnly and deferFileDeletion: 474 # We skipped deleting the existing files from filesystem 475 # because maybe they'd be needed in migrate scripts. 476 # Delete them now 477 oldZpDir = zenPath('Products', existing.id) 478 if os.path.islink(oldZpDir): 479 os.remove(oldZpDir) 480 else: 481 shutil.rmtree(oldZpDir) 482 483 return zenPack 484

485 486 -def DiscoverEggs(dmd, zenPackId):

487 """ 488 Find installed eggs that provide a zenoss.zenpacks entry point. 489 Return a list of distributions whose ZenPacks need to be installed 490 or upgraded. The list is sorted into the order in which this needs to 491 happen. 492 """ 493 # Create a set of all available zenoss.zenpack entries that aren't 494 # already installed in zenoss or need to be upgraded in zenoss. 495 entries = set() 496 parse_version = pkg_resources.parse_version 497 for entry in pkg_resources.iter_entry_points(ZENPACK_ENTRY_POINT): 498 packName = entry.name 499 packVers = entry.dist.version 500 existing = dmd.ZenPackManager.packs._getOb(packName, None) 501 if existing and existing.isEggPack(): 502 # We use >= to allow migrate to be run on currently installed 503 # zenpacks whose version has been changed or for whom new 504 # migrates have been added. 505 if parse_version(packVers) >= parse_version(existing.version): 506 entries.add(entry) 507 else: 508 entries.add(entry) 509 510 # Starting with the entry representing zenPackId create a list of 511 # all entrypoints 512 513 # orderedEntries lists entries in the opposite order of that in which 514 # they need to be installed. This is simply for convenience of using 515 # .append() in code below. 516 orderedEntries = [] 517 entriesByName = dict((e.name, e) for e in entries) 518 519 def AddEntryAndProcessDeps(e): 520 orderedEntries.append(e) 521 for name in [r.project_name for r in e.dist.requires()]: 522 if name in [e.name for e in orderedEntries]: 523 # This entry depends on something that we've already processed. 524 # This might be a circular dependency, might not be. 525 # We are just going to bail however. This should be 526 # very unusual and the user can install deps first to work 527 # around. 528 raise ZenPackException('Unable to resolve ZenPack dependencies.' 529 ' Try installing dependencies first.') 530 if name in entriesByName: 531 # The requirement is an entry that has not yet been processed 532 # here. Add it to the list of entries to install/upgrade. 533 AddEntryAndProcessDeps(entriesByName[name]) 534 else: 535 # The requirement is not in the entries generated above. 536 # This either means that the dep is already installed (this 537 # is likely) or that easy_install missed something and the dep 538 # is not installed/available (this should be unlikely.) 539 pass

540 541 if zenPackId not in entriesByName: 542 if zenPackId in dmd.ZenPackManager.packs.objectIds(): 543 return [] 544 else: 545 raise ZenPackException('Unable to discover ZenPack named %s' % 546 zenPackId) 547 AddEntryAndProcessDeps(entriesByName[zenPackId]) 548 orderedEntries.reverse() 549 return [e.dist for e in orderedEntries] 550

551 552 -def AddDistToWorkingSet(distPath):

553 """ 554 Given the path to a dist (an egg) add it to the current working set. 555 This is basically a pkg_resources-friendly way of adding it to 556 sys.path. 557 Return a list of all distributions on distPath that appear to 558 be ZenPacks. 559 """ 560 zpDists = [] 561 for d in pkg_resources.find_distributions(distPath): 562 pkg_resources.working_set.add(d) 563 pkg_resources.require(d.project_name) 564 if d.project_name.startswith('ZenPacks.'): 565 zpDists.append(d) 566 return zpDists

567

568 569 -def ReadZenPackInfo(dist):

570 """ 571 Return a dictionary containing the egg metadata 572 """ 573 info = {} 574 if dist.has_metadata('PKG-INFO'): 575 lines = dist.get_metadata('PKG-INFO') 576 for line in pkg_resources.yield_lines(lines): 577 key, value = line.split(':', 1) 578 info[key.strip()] = value.strip() 579 if dist.has_metadata('zenpack_info'): 580 lines = dist.get_metadata('zenpack_info') 581 for line in pkg_resources.yield_lines(lines): 582 key, value = line.split(':', 1) 583 info[key.strip()] = value.strip() 584 return info

585

586 587 -def CopyMetaDataToZenPackObject(dist, pack):

588 """ 589 Copy metadata type stuff from the distribution to the zp object. 590 """ 591 # Version 592 pack.version = dist.version 593 594 # Egg Info 595 info = ReadZenPackInfo(dist) 596 pack.author = info.get('Author', '') 597 if pack.author == 'UNKNOWN': 598 pack.author = '' 599 600 pack.license = info.get('License', '') 601 if pack.license == 'UNKNOWN': 602 pack.license = '' 603 604 pack.compatZenossVers = info.get('compatZenossVers', '') 605 pack.prevZenPackName = info.get('prevZenPackName', '') 606 607 # Requires 608 pack.dependencies = {} 609 for r in dist.requires(): 610 name = r.project_name 611 spec = str(r)[len(name):] 612 pack.dependencies[name] = spec

613

614 615 -def CreateZenPacksDir():

616 """ 617 Make sure $ZENHOME/ZenPacks exists 618 """ 619 zpDir = zenPath('ZenPacks') 620 if not os.path.isdir(zpDir): 621 os.mkdir(zpDir, 0750)

622

623 624 -def DoEasyInstall(eggPath):

625 """ 626 Use easy_install to install an egg from the filesystem. 627 easy_install will install the egg, but does not install it into 628 Zenoss as ZenPacks. 629 Returns a list of distributions that were installed that appear 630 to be ZenPacks. 631 """ 632 from setuptools.command import easy_install 633 634 # Make sure $ZENHOME/ZenPacks exists 635 CreateZenPacksDir() 636 637 # Create temp file for easy_install to write results to 638 _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall') 639 # eggPaths is a set of paths to eggs that were installed. We need to 640 # add them to the current workingset so we can discover their 641 # entry points. 642 eggPaths = set() 643 try: 644 # Execute the easy_install 645 args = ['--site-dirs', zenPath('ZenPacks'), 646 '-d', zenPath('ZenPacks'), 647 '--allow-hosts', 'None', 648 '--record', tempPath, 649 '--quiet', 650 eggPath] 651 easy_install.main(args) 652 # Collect the paths for eggs that were installed 653 f = open(tempPath, 'r') 654 marker = '.egg/' 655 markerOffset = len(marker)-1 656 for l in f.readlines(): 657 i = l.find(marker) 658 if i > 0: 659 eggPaths.add(l[:i+markerOffset]) 660 finally: 661 os.remove(tempPath) 662 # Add any installed eggs to the current working set 663 zpDists = [] 664 for path in eggPaths: 665 zpDists += AddDistToWorkingSet(path) 666 return zpDists

667

668 669 ######################################## 670 # Zenoss.Net 671 ######################################## 672 673 674 -def FetchAndInstallZenPack(dmd, zenPackName, sendEvent=True):

675 """ 676 Fetch the named zenpack and all its dependencies and install them. 677 Return a list of the ZenPacks that were installed. 678 """ 679 zenPacks = [] 680 try: 681 zpDists = FetchZenPack(dmd, zenPackName) 682 for d in zpDists: 683 zenPacks.append(InstallDistAsZenPack(dmd, d, d.location)) 684 except Exception, ex: 685 log.exception("Error fetching ZenPack %s" % zenPackName) 686 if sendEvent: 687 ZPEvent(dmd, 4, 'Failed to install ZenPack %s' % zenPackName, 688 '%s: %s' % sys.exc_info()[:2]) 689 690 raise ex 691 if sendEvent: 692 zenPackIds = [z.id for z in zenPacks] 693 if zenPackIds: 694 ZPEvent(dmd, 2, 'Installed ZenPacks: %s' % ', '.join(zenPackIds)) 695 if zenPackName not in zenPackIds: 696 ZPEvent(dmd, 4, 'Unable to install ZenPack %s' % zenPackName) 697 return zenPacks

698

699 700 -def FetchZenPack(dmd, zenPackName):

701 """ 702 Use easy_install to retrieve the given zenpack and any dependencies. 703 easy_install will install the eggs, but does not install them into 704 Zenoss as ZenPacks. 705 Return a list of distributions just installed that appear to be 706 ZenPacks. 707 708 NB: This should be refactored. It shares most of its code with 709 DoEasyInstall() 710 """ 711 from setuptools.command import easy_install 712 713 # Make sure $ZENHOME/ZenPacks exists 714 CreateZenPacksDir() 715 716 # Create the proper package index URL. 717 index_url = '%s/%s/%s/' % ( 718 ZENPACKS_BASE_URL, dmd.uuid, ZVersion.VERSION) 719 720 # Create temp file for easy_install to write results to 721 _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall') 722 # eggPaths is a set of paths to eggs that were installed. We need to 723 # add them to the current workingset so we can discover their 724 # entry points. 725 eggPaths = set() 726 try: 727 # Execute the easy_install 728 args = [ 729 '--site-dirs', zenPath('ZenPacks'), 730 '-d', zenPath('ZenPacks'), 731 '-i', index_url, 732 '--allow-hosts', urlparse(index_url).hostname, 733 '--record', tempPath, 734 '--quiet', 735 zenPackName] 736 easy_install.main(args) 737 # Collect the paths for eggs that were installed 738 f = open(tempPath, 'r') 739 marker = '.egg/' 740 markerOffset = len(marker) - 1 741 for l in f.readlines(): 742 i = l.find(marker) 743 if i > 0: 744 eggPaths.add(l[:i + markerOffset]) 745 finally: 746 os.remove(tempPath) 747 # Add any installed eggs to the current working set 748 zpDists = [] 749 for path in eggPaths: 750 zpDists += AddDistToWorkingSet(path) 751 return zpDists

752

753 754 -def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):

755 """ 756 Upload the specified zenpack to the given project. 757 Project is a string of the form 'enterprise/myproject' or 758 'community/otherproject'. 759 """ 760 zp = dmd.ZenPackManager.packs._getOb(packName, None) 761 if not zp: 762 raise ZenPackException('No ZenPack named %s' % packName) 763 764 # Export the zenpack 765 fileName = zp.manage_exportPack() 766 filePath = zenPath('export', fileName) 767 768 # Login to Zenoss.net 769 from DotNetCommunication import DotNetSession 770 session = DotNetSession() 771 userSettings = dmd.ZenUsers.getUserSettings() 772 session.login(znetUser, znetPass) 773 774 # Upload 775 zpFile = open(zenPath('export', fileName), 'r') 776 try: 777 response = session.open('%s/createRelease' % project.strip('/'), { 778 'description': description, 779 'fileStorage': zpFile, 780 }) 781 finally: 782 zpFile.close() 783 if response: 784 result = response.read() 785 if "'success':true" not in result: 786 raise ZenPackException('Upload failed') 787 else: 788 raise ZenPackException('Failed to connect to Zenoss.net') 789 return

790

791 792 ######################################## 793 # ZenPack Removal 794 ######################################## 795 796 797 -def RemoveZenPack(dmd, packName, filesOnly=False, skipDepsCheck=False, 798 leaveObjects=False, sendEvent=True, 799 forceNoFileDeletion=False, uninstallEgg=True):

800 """ 801 Remove the given ZenPack from Zenoss. 802 Whether the ZenPack will be removed from the filesystem or not 803 depends on the result of the ZenPack's shouldDeleteFilesOnRemoval method. 804 """ 805 try: 806 if filesOnly: 807 skipDepsCheck = True 808 809 # Check for dependency implications here? 810 if not skipDepsCheck: 811 deps = GetDependents(dmd, packName) 812 if deps: 813 raise ZenPackDependentsException('%s cannot be removed ' % packName + 814 'because it is required by %s' % ', '.join(deps)) 815 816 if not filesOnly: 817 # Fetch the zenpack, call its remove() and remove from packs 818 zp = None 819 try: 820 zp = dmd.ZenPackManager.packs._getOb(packName) 821 except AttributeError, ex: 822 raise ZenPackNotFoundException('No ZenPack named %s is installed' % 823 packName) 824 # If zencatalog hasn't finished yet, we get ugly messages that don't 825 # mean anything. Hide them. 826 logFilter = None 827 if not getattr(dmd.zport, '_zencatalog_completed', False): 828 logFilter = CatalogLoggingFilter() 829 logging.getLogger('Zope.ZCatalog').addFilter(logFilter) 830 try: 831 zp.remove(dmd, leaveObjects) 832 dmd.ZenPackManager.packs._delObject(packName) 833 transaction.commit() 834 finally: 835 # Remove our logging filter so we don't hide anything important 836 if logFilter is not None: 837 logging.getLogger('Zope.ZCatalog').removeFilter(logFilter) 838 839 # Uninstall the egg and possibly delete it 840 # If we can't find the distribution then apparently the zp egg itself is 841 # missing. Continue on with the removal and swallow the 842 # DistributionNotFound exception 843 try: 844 dist = zp.getDistribution() 845 except pkg_resources.DistributionNotFound: 846 dist = None 847 if dist: 848 # Determine deleteFiles before develop -u gets called. Once 849 # it is called the egg has problems figuring out some of it's state. 850 deleteFiles = zp.shouldDeleteFilesOnRemoval() 851 if uninstallEgg: 852 if zp.isDevelopment(): 853 zenPackDir = zenPath('ZenPacks') 854 cmd = ('%s setup.py develop -u ' 855 % binPath('python') + 856 '--site-dirs=%s ' % zenPackDir + 857 '-d %s' % zenPackDir) 858 p = subprocess.Popen(cmd, 859 stdout=subprocess.PIPE, 860 stderr=subprocess.PIPE, 861 shell=True, 862 cwd=zp.eggPath()) 863 out, err = p.communicate() 864 code = p.wait() 865 if code: 866 raise ZenPackException(err) 867 else: 868 DoEasyUninstall(packName) 869 # elif cleanupEasyInstallPth: 870 # # Do we need to call easy_install -m here? It causes problems 871 # # because it tries to install deps. Cleanup easy-install.pth 872 # # ourselves instead. 873 # # We don't want to cleanup easy-install.pth when a newer 874 # # version of the egg has already been installed (when doing 875 # # an upgrade or installing in new location.) 876 # eggLink = './%s' % zp.eggName() 877 # CleanupEasyInstallPth(eggLink) 878 if deleteFiles and not forceNoFileDeletion: 879 eggDir = zp.eggPath() 880 if os.path.islink(eggDir): 881 os.remove(eggDir) 882 else: 883 shutil.rmtree(eggDir) 884 cleanupSkins(dmd) 885 transaction.commit() 886 except ZenPackDependentsException, ex: 887 log.error(ex) 888 except Exception, ex: 889 # Get that exception out there in case it gets blown away by ZPEvent 890 log.exception("Error removing ZenPack %s" % packName) 891 if sendEvent: 892 ZPEvent(dmd, 4, 'Error removing ZenPack %s' % packName, 893 '%s: %s' % sys.exc_info()[:2]) 894 895 # Don't just raise, because if ZPEvent blew away exception context 896 # it'll be None, which is bad. This manipulates the stack to look like 897 # this is the source of the exception, but we logged it above so no 898 # info is lost. 899 raise ex 900 if sendEvent: 901 ZPEvent(dmd, 2, 'Removed ZenPack %s' % packName)

902

903 904 -def DoEasyUninstall(name):

905 """ 906 Execute the easy_install command to unlink the given egg. 907 What this is really doing is switching the egg to be in 908 multiple-version mode, however this is the first step in deleting 909 an egg as described here: 910 http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages 911 """ 912 from setuptools.command import easy_install 913 args = ['--site-dirs', zenPath('ZenPacks'), 914 '-d', zenPath('ZenPacks'), 915 #'--record', tempPath, 916 '--quiet', 917 '-m', 918 name] 919 easy_install.main(args)

920

921 922 -def CanRemoveZenPacks(dmd, packNames):

923 """ 924 Returns a tuple of (canRemove, otherDependents) 925 canRemove is True if the listed zenPacks have no dependents not also 926 listed in packNames, False otherwise. 927 otherDependents is a list of zenpack names not in packNames that 928 depend on one or more of the packs in packNames. 929 """ 930 unhappy = set() 931 for name in packNames: 932 deps = GetDependents(dmd, name) 933 unhappy.update(set(dep for dep in deps if dep not in packNames)) 934 return (not unhappy, list(unhappy))

935

936 937 # def CleanupEasyInstallPth(eggLink): 938 # """ 939 # Remove the entry for the given egg from the 940 # $ZENHOME/ZenPacks/easy-install.pth file. If this entry is left 941 # in place in can cause problems during the next install of the same 942 # egg. 943 # """ 944 # # Remove the path from easy-install.pth 945 # eggTail = os.path.split(eggLink)[1] 946 # easyPth = zenPath('ZenPacks', 'easy-install.pth') 947 # if os.path.isfile(easyPth): 948 # needToWrite = False 949 # newLines = [] 950 # f = open(easyPth, 'r') 951 # for line in f: 952 # if os.path.split(line.strip())[1] == eggTail: 953 # needToWrite = True 954 # else: 955 # newLines.append(line) 956 # f.close() 957 # if needToWrite: 958 # f = open(easyPth, 'w') 959 # f.writelines(newLines) 960 # f.close() 961 962 963 -def GetDependents(dmd, packName):

964 """ 965 Return a list of installed ZenPack ids that list packName as a dependency 966 """ 967 return [zp.id for zp in dmd.ZenPackManager.packs() 968 if zp.id != packName and packName in zp.dependencies]

969

970 971 ######################################## 972 # __main__, dispatching, etc 973 ######################################## 974 975 976 -def ZPEvent(dmd, severity, summary, message=None):

977 """ 978 Send an event to Zenoss. 979 """ 980 dmd.ZenEventManager.sendEvent(dict( 981 device=FQDN, 982 eventClass='/Unknown', 983 severity=severity, 984 summary=summary, 985 message=message))

986

987 988 -class ZenPackCmd(ZenScriptBase):

989 """ 990 Utilities for creating, installing, removing ZenPacks. 991 992 NOTE: Users will probably invoke zenpack from the command line, which 993 runs zenpack.py rather than this file. zenpack.py calls functions 994 in this module when it detects that new-style (egg) ZenPacks are involved. 995 The plan is that once support for old-style (non-egg) ZenPacks is dropped 996 zenpack.py can go away and this will take its place. Until then this 997 script can be invoked directly via the zenpackcmd script if desired. 998 Invoking this script directly has the benefit of slightly better 999 progress/status output to stdout. 1000 """ 1001

1002 - def run(self):

1003 """ 1004 Execute the user's request. 1005 """ 1006 1007 self.connect() 1008 def PrintInstalled(installed, eggOnly=False): 1009 if installed: 1010 if eggOnly: 1011 names = [i['id'] for i in installed] 1012 what = 'ZenPack egg' 1013 else: 1014 names = [i.id for i in installed] 1015 what = 'ZenPack' 1016 print('Installed %s%s: %s' % ( 1017 what, 1018 len(names) != 1 and 's' or '', 1019 ', '.join(names))) 1020 else: 1021 print('No ZenPacks installed.')

1022 1023 if not getattr(self.dmd, 'ZenPackManager', None): 1024 raise ZenPackNeedMigrateException('Your Zenoss database appears' 1025 ' to be out of date. Try running zenmigrate to update.') 1026 if self.options.eggOnly and self.options.eggPath: 1027 zpDists = InstallEgg(self.dmd, self.options.eggPath, 1028 link=self.options.link) 1029 PrintInstalled([{'id':d.project_name} for d in zpDists], 1030 eggOnly=True) 1031 if self.options.eggPath: 1032 installed = InstallEggAndZenPack( 1033 self.dmd, self.options.eggPath, 1034 link=self.options.link, 1035 filesOnly=self.options.filesOnly, 1036 previousVersion= self.options.previousVersion) 1037 PrintInstalled(installed) 1038 elif self.options.fetch: 1039 installed = FetchAndInstallZenPack(self.dmd, self.options.fetch) 1040 PrintInstalled(installed) 1041 elif self.options.upload: 1042 return UploadZenPack(self.dmd, self.options.upload, 1043 self.options.znetProject, 1044 self.options.uploadDesc, 1045 self.options.znetUser, 1046 self.options.znetPass) 1047 elif self.options.removePackName: 1048 try: 1049 RemoveZenPack(self.dmd, self.options.removePackName) 1050 print('Removed ZenPack: %s' % self.options.removePackName) 1051 except ZenPackNotFoundException, e: 1052 sys.stderr.write(str(e) + '\n') 1053 elif self.options.list: 1054 self.list() 1055 else: 1056 self.parser.print_help()

1057 1058

1059 - def buildOptions(self):

1060 self.parser.add_option('--install', 1061 dest='eggPath', 1062 default=None, 1063 help="name of the pack to install") 1064 # self.parser.add_option('--fetch', 1065 # dest='fetch', 1066 # default=None, 1067 # help='Name of ZenPack to retrieve from ' 1068 # 'Zenoss.net and install.') 1069 # self.parser.add_option('--fetch-vers', 1070 # dest='fetchVers', 1071 # default=None, 1072 # help='Use with --fetch to specify a version' 1073 # ' for the ZenPack to download and install.') 1074 # self.parser.add_option('--znet-user', 1075 # dest='znetUser', 1076 # default=None, 1077 # help='Use with --fetch or --upload to specify' 1078 # ' your Zenoss.net username.') 1079 # self.parser.add_option('--znet-pass', 1080 # dest='znetPass', 1081 # default=None, 1082 # help='Use with --fetch or --upload to specify' 1083 # ' your Zenoss.net password.') 1084 # self.parser.add_option('--upload', 1085 # dest='upload', 1086 # default=None, 1087 # help='Name of ZenPack to upload to ' 1088 # 'Zenoss.net') 1089 # self.parser.add_option('--znet-project', 1090 # dest='znetProject', 1091 # default=None, 1092 # help='Use with --upload to specify' 1093 # ' which Zenoss.net project to create' 1094 # ' a release on.') 1095 # self.parser.add_option('--upload-desc', 1096 # dest='uploadDesc', 1097 # default=None, 1098 # help='Use with --upload to provide' 1099 # ' a description for the new release.') 1100 self.parser.add_option('--link', 1101 dest='link', 1102 action='store_true', 1103 default=False, 1104 help='Install the ZenPack in its current ' 1105 'location, do not copy to $ZENHOME/ZenPacks. ' 1106 'Also mark ZenPack as editable. ' 1107 'This only works with source directories ' 1108 'containing setup.py files, not ' 1109 'egg files.') 1110 self.parser.add_option('--remove', 1111 dest='removePackName', 1112 default=None, 1113 help="name of the pack to remove") 1114 self.parser.add_option('--leave-objects', 1115 dest='leaveObjects', 1116 default=False, 1117 action='store_true', 1118 help="When specified with --remove then objects" 1119 ' provided by the ZenPack and those' 1120 ' depending on the ZenPack are not deleted.' 1121 ' This may result in broken objects in your' 1122 ' database unless the ZenPack is' 1123 ' reinstalled.') 1124 self.parser.add_option('--files-only', 1125 dest='filesOnly', 1126 action="store_true", 1127 default=False, 1128 help='install onto filesystem but not into ' 1129 'zenoss') 1130 self.parser.add_option('--previousversion', 1131 dest='previousVersion', 1132 default=None, 1133 help="Previous version of the zenpack;" 1134 ' used during upgrades') 1135 self.parser.prog = "zenpack" 1136 ZenScriptBase.buildOptions(self)

1137 1138 1139 if __name__ == '__main__': 1140 try: 1141 zp = ZenPackCmd() 1142 zp.run() 1143 except ZenPackException, e: 1144 sys.stderr.write('%s\n' % str(e)) 1145 sys.exit(-1) 1146

Source Code for Module Products.ZenUtils.ZenPackCmd