Products.ZenUtils.ZenPackCmd

1 ########################################################################### 2 # 3 # This program is part of Zenoss Core, an open source monitoring platform. 4 # Copyright (C) 2008, Zenoss Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify it 7 # under the terms of the GNU General Public License version 2 as published by 8 # the Free Software Foundation. 9 # 10 # For complete information please visit: http://www.zenoss.com/oss/ 11 # 12 ########################################################################### 13 14 __doc__ = "Manage ZenPacks" 15 16 import Globals 17 from Products.ZenUtils.ZenScriptBase import ZenScriptBase 18 from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, getObjByPath 19 20 from Products.ZenModel.ZenPack import ZenPackException, \ 21 ZenPackNotFoundException, \ 22 ZenPackNeedMigrateException 23 from Products.ZenModel.ZenPack import ZenPackDependentsException 24 from Products.ZenModel.ZenPack import ZenPack 25 from Products.ZenUtils.PkgResources import pkg_resources 26 from Products.Zuul.utils import CatalogLoggingFilter 27 import Products.ZenModel.ZenPackLoader as ZPL 28 import zenpack as oldzenpack 29 import transaction 30 import os, sys 31 import shutil 32 import string 33 import tempfile 34 import subprocess 35 import socket 36 import logging 37 import zExceptions 38 39 40 log = logging.getLogger('zen.ZenPackCMD') 41 42 #import zenpacksupport 43 44 FQDN = socket.getfqdn() 45 46 ZEN_PACK_INDEX_URL = '' 47 48 # All ZenPack eggs have to define exactly one entry point in this group. 49 ZENPACK_ENTRY_POINT = 'zenoss.zenpacks' 50 51 ######################################## 52 # ZenPack Creation 53 ######################################## 54

55 -def CreateZenPack(zpId, prevZenPackName=''):

56 """ 57 Create the zenpack in the filesystem. 58 The zenpack is not installed in Zenoss, it is simply created in 59 the $ZENHOME/ZenPacks directory. Usually this should be followed 60 with a "zenpack install" call. 61 zpId should already be valid, scrubbed value. 62 prevZenPackName is written to PREV_ZENPACK_NAME in setup.py. 63 """ 64 parts = zpId.split('.') 65 66 # Copy template to $ZENHOME/ZenPacks 67 srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate') 68 devDir = zenPath('ZenPacks') 69 if not os.path.exists(devDir): 70 os.mkdir(devDir, 0750) 71 destDir = os.path.join(devDir, zpId) 72 shutil.copytree(srcDir, destDir, symlinks=False) 73 os.system('find %s -name .svn | xargs rm -rf' % destDir) 74 75 # Write setup.py 76 packages = [] 77 for i in range(len(parts)): 78 packages.append('.'.join(parts[:i+1])) 79 mapping = dict( 80 NAME = zpId, 81 VERSION = '1.0', 82 AUTHOR = '', 83 LICENSE = '', 84 NAMESPACE_PACKAGES = packages[:-1], 85 PACKAGES = packages, 86 INSTALL_REQUIRES = [], 87 COMPAT_ZENOSS_VERS = '', 88 PREV_ZENPACK_NAME = prevZenPackName, 89 ) 90 WriteSetup(os.path.join(destDir, 'setup.py'), mapping) 91 92 # Create subdirectories 93 base = destDir 94 for part in parts[:-1]: 95 base = os.path.join(base, part) 96 os.mkdir(base) 97 f = open(os.path.join(base, '__init__.py'), 'w') 98 f.write("__import__('pkg_resources').declare_namespace(__name__)\n") 99 f.close() 100 base = os.path.join(base, parts[-1]) 101 shutil.move(os.path.join(destDir, 'CONTENT'), base) 102 103 # Create the skins subdirs 104 skinsDir = os.path.join(base, 'skins', zpId) 105 os.makedirs(skinsDir) 106 107 # Stick a placeholder in the skins dir so that the egg will include 108 # the dir even if empty. 109 f = file(os.path.join(skinsDir, 'placeholder.txt'), 'w') 110 f.close() 111 112 return destDir

113 114

115 -def WriteSetup(setupPath, values):

116 """ 117 """ 118 f = file(setupPath, 'r') 119 lines = f.readlines() 120 f.close() 121 122 newLines = [] 123 for i, line in enumerate(lines): 124 if line.startswith('STOP_REPLACEMENTS'): 125 newLines += lines[i:] 126 break 127 key = line.split('=')[0].strip() 128 if values.has_key(key): 129 value = values[key] 130 if isinstance(value, basestring): 131 fmt = '%s = "%s"\n' 132 else: 133 fmt = '%s = %s\n' 134 newLines.append(fmt % (key, value)) 135 else: 136 newLines.append(line) 137 138 f = file(setupPath, 'w') 139 f.writelines(newLines) 140 f.close()

141 142

143 -def CanCreateZenPack(dmd, zpId):

144 """ 145 Return tuple (bool, string) where first element is true if a new zenpack 146 can be created with the given info and false if not. If first element 147 is True then the second part of the tuple contains the scrubbed ZenPack id. 148 If the first part is False then the second contains an explanatory 149 message. 150 """ 151 # Check if id and package looks reasonable 152 (allowable, idOrMsg) = ScrubZenPackId(zpId) 153 if allowable: 154 zpId = idOrMsg 155 else: 156 return (False, idOrMsg) 157 158 # Is the id already in use? 159 if dmd: 160 if zpId in dmd.ZenPackManager.packs.objectIds(): 161 return (False, 'A ZenPack named %s already exists.' % zpId) 162 163 # Is there another zenpack in the way? 164 # Now that zenpacks are created in $ZENHOME/ZenPacks instead of 165 # $ZENHOME/ZenPackDev this may no longer be necessary because a 166 # zp in the way should be installed and caught by the already in use 167 # check above. 168 if os.path.exists(zenPath('ZenPacks', zpId)): 169 return (False, 'A directory named %s already exists' % zpId + 170 ' in $ZENHOME/ZenPacks. Use a different name' 171 ' or remove that directory.') 172 173 return (True, idOrMsg)

174 175

176 -def ScrubZenPackId(name):

177 """ 178 If the given name conforms to ZenPack naming rules, or can easily be 179 modified to do so, then return (True, scrubbedName) where scrubbedName 180 is either name or a slightly modified name. If the given name does 181 not conform to naming rules and we can't easily modify it to do so 182 then return (False, errorMsg) where errorMsg describes why name 183 is unacceptable. 184 """ 185 parts = name.split('.') 186 187 # Remove leading dots, trailing dots, adjacent dots and strip whitespace 188 # from each part 189 parts = [p.strip() for p in parts] 190 parts = [p for p in parts if p] 191 192 # Add/fix leading 'ZenPacks' 193 if parts[0] != 'ZenPacks': 194 if parts[0].lower() == 'zenpacks': 195 parts[0] = 'ZenPacks' 196 else: 197 parts.insert(0, 'ZenPacks') 198 199 # Must be at least 3 parts 200 if len(parts) < 3: 201 return (False, 'ZenPack names must contain at least three package ' 202 'names separated by periods.') 203 204 # Each part must start with a letter 205 for p in parts: 206 if p[0] not in string.letters: 207 return (False, 'Each package name must start with a letter.') 208 209 # Only letters, numbers and underscores in each part 210 allowable = string.letters + string.digits + '_' 211 for p in parts: 212 for c in p: 213 if c not in allowable: 214 return (False, 'Package names may only contain letters, ' 215 'numbers and underscores.') 216 217 return (True, '.'.join(parts))

218 219 220 ######################################## 221 # ZenPack Installation 222 ######################################## 223

224 -class NonCriticalInstallError(Exception):

225 - def __init__(self, message):

226 Exception.__init__(self, message) 227 self.message = message

228

229 -def InstallEggAndZenPack(dmd, eggPath, link=False, 230 filesOnly=False, sendEvent=True, 231 previousVersion=None):

232 """ 233 Installs the given egg, instantiates the ZenPack, installs in 234 dmd.ZenPackManager.packs, and runs the zenpacks's install method. 235 Returns a list of ZenPacks that were installed. 236 """ 237 zenPacks = [] 238 nonCriticalErrorEncountered = False 239 try: 240 zpDists = InstallEgg(dmd, eggPath, link=link) 241 for d in zpDists: 242 try: 243 zp = InstallDistAsZenPack(dmd, 244 d, 245 eggPath, 246 link, 247 filesOnly=filesOnly, 248 previousVersion=\ 249 previousVersion) 250 zenPacks.append(zp) 251 except NonCriticalInstallError, ex: 252 nonCriticalErrorEncountered = True 253 if sendEvent: 254 ZPEvent(dmd, 3, ex.message) 255 except: 256 if sendEvent: 257 ZPEvent(dmd, 4, 'Error installing ZenPack %s' % eggPath, 258 '%s: %s' % sys.exc_info()[:2]) 259 raise 260 if sendEvent: 261 zenPackIds = [zp.id for zp in zenPacks] 262 if zenPackIds: 263 ZPEvent(dmd, 2, 'Installed ZenPacks %s' % ','.join(zenPackIds)) 264 elif not nonCriticalErrorEncountered: 265 ZPEvent(dmd, 4, 'Unable to install %s' % eggPath) 266 return zenPacks

267 268

269 -def InstallEgg(dmd, eggPath, link=False):

270 """ 271 Install the given egg and add to the current working set. 272 This does not install the egg as a ZenPack. 273 Return a list of distributions that should be installed as ZenPacks. 274 """ 275 eggPath = os.path.abspath(eggPath) 276 zenPackDir = zenPath('ZenPacks') 277 eggInZenPacksDir = eggPath.startswith(zenPackDir + '/') 278 279 # Make sure $ZENHOME/ZenPacks exists 280 CreateZenPacksDir() 281 282 # Install the egg 283 if link: 284 cmd = ('%s setup.py develop ' % binPath('python') + 285 '--site-dirs=%s ' % zenPackDir + 286 '-d %s' % zenPackDir) 287 p = subprocess.Popen(cmd, 288 stdout=subprocess.PIPE, 289 stderr=subprocess.PIPE, 290 shell=True, 291 cwd=eggPath) 292 out, err = p.communicate() 293 p.wait() 294 if p.returncode: 295 DoEasyUninstall(eggPath) 296 raise ZenPackException('Error installing the egg (%s): %s' % 297 (p.returncode, err)) 298 zpDists = AddDistToWorkingSet(eggPath) 299 else: 300 try: 301 zpDists = DoEasyInstall(eggPath) 302 except: 303 DoEasyUninstall(eggPath) 304 raise 305 # cmd = 'easy_install --always-unzip --site-dirs=%s -d %s %s' % ( 306 # zenPackDir, 307 # zenPackDir, 308 # eggPath) 309 # p = subprocess.Popen(cmd, 310 # stdout=subprocess.PIPE, 311 # stderr=subprocess.PIPE, 312 # shell=True) 313 # p.wait() 314 # eggName = os.path.split(eggPath)[1] 315 # eggPath = os.path.join(zenPackDir, eggName) 316 return zpDists

317 318 319 # def GetZenPackNamesFromEggPath(eggPath): 320 # """ 321 # Given a path to a ZenPack egg (installed or not) return the 322 # name of the ZenPack it contains. 323 # """ 324 # zpNames = [] 325 # for d in pkg_resources.find_distributions(eggPath) 326 # if d.project_name.startswith('ZenPacks.'): 327 # zpNames.append(d.project_name) 328 # return zpNames 329 330

331 -def InstallDistAsZenPack(dmd, dist, eggPath, link=False, filesOnly=False, 332 previousVersion=None):

333 """ 334 Given an installed dist, install it into Zenoss as a ZenPack. 335 Return the ZenPack instance. 336 """ 337 # Instantiate ZenPack 338 entryMap = pkg_resources.get_entry_map(dist, ZENPACK_ENTRY_POINT) 339 if not entryMap or len(entryMap) > 1: 340 raise ZenPackException('A ZenPack egg must contain exactly one' 341 ' zenoss.zenpacks entry point. This egg appears to contain' 342 ' %s such entry points.' % len(entryMap)) 343 packName, packEntry = entryMap.items()[0] 344 zenPack = None; 345 runExternalZenpack = True 346 #if zenpack with same name exists we can't load both modules 347 #installing new egg zenpack will be done in a sub process 348 existing = dmd.ZenPackManager.packs._getOb(packName, None) 349 if existing: 350 log.info("Previous ZenPack exists with same name %s" % packName) 351 if filesOnly or not existing: 352 #running files only or zenpack by same name doesn't already exists 353 # so no need to install the zenpack in an external process 354 runExternalZenpack = False 355 module = packEntry.load() 356 if hasattr(module, 'ZenPack'): 357 zenPack = module.ZenPack(packName) 358 else: 359 zenPack = ZenPack(packName) 360 zenPack.eggPack = True 361 CopyMetaDataToZenPackObject(dist, zenPack) 362 if filesOnly: 363 for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()): 364 loader.load(zenPack, None) 365 366 367 if not filesOnly: 368 # Look for an installed ZenPack to be upgraded. In this case 369 # upgraded means that it is removed before the new one is installed 370 # but that its objects are not removed and the packables are 371 # copied to the new instance. 372 existing = dmd.ZenPackManager.packs._getOb(packName, None) 373 if not existing and zenPack.prevZenPackName: 374 existing = dmd.ZenPackManager.packs._getOb( 375 zenPack.prevZenPackName, None) 376 377 deferFileDeletion = False 378 packables = [] 379 upgradingFrom = None 380 if existing: 381 upgradingFrom = existing.version 382 for p in existing.packables(): 383 packables.append(p) 384 existing.packables.removeRelation(p) 385 if existing.isEggPack(): 386 forceNoFileDeletion = existing.eggPath() == dist.location 387 RemoveZenPack(dmd, existing.id, 388 skipDepsCheck=True, leaveObjects=True, 389 forceNoFileDeletion=forceNoFileDeletion, 390 uninstallEgg=False) 391 else: 392 # Don't delete files, might still be needed for 393 # migrate scripts to be run below. 394 deferFileDeletion = True 395 oldzenpack.RemoveZenPack(dmd, existing.id, 396 skipDepsCheck=True, leaveObjects=True, 397 deleteFiles=False) 398 if runExternalZenpack: 399 log.info("installing zenpack %s; launching process" % packName) 400 cmd = [binPath('zenpack')] 401 if link: 402 cmd += ["--link"] 403 cmd += ["--install", eggPath] 404 if upgradingFrom: 405 cmd += ['--previousversion', upgradingFrom] 406 407 cmdStr = " ".join(cmd) 408 log.debug("launching sub process command: %s" % cmdStr) 409 p = subprocess.Popen(cmdStr, 410 shell=True) 411 out, err = p.communicate() 412 p.wait() 413 if p.returncode: 414 raise ZenPackException('Error installing the egg (%s): %s' % 415 (p.returncode, err)) 416 dmd._p_jar.sync() 417 else: 418 dmd.ZenPackManager.packs._setObject(packName, zenPack) 419 zenPack = dmd.ZenPackManager.packs._getOb(packName) 420 #hack because ZenPack.install is overridden by a lot of zenpacks 421 #so we can't change the signature of install to take the 422 #previousVerison 423 zenPack.prevZenPackVersion = previousVersion 424 zenPack.install(dmd) 425 zenPack.prevZenPackVersion = None 426 427 428 try: 429 zenPack = dmd.ZenPackManager.packs._getOb(packName) 430 for p in packables: 431 pId = p.getPrimaryId() 432 try: 433 # make sure packable still exists; could be deleted by a 434 # migrate 435 getObjByPath(dmd, pId) 436 log.debug("adding packable relation for id %s", pId) 437 zenPack.packables.addRelation(p) 438 except (KeyError, zExceptions.NotFound): 439 log.debug('did not find packable %s',pId) 440 except AttributeError, e: 441 # If this happens in the child process or during the non-upgrade 442 # flow, reraise the exception 443 if not runExternalZenpack: 444 raise 445 446 # This specific error will occur when the version of the ZenPack 447 # being installed subclasses Products.ZenModel.ZenPack, but the 448 # previous version of the ZenPack did not. 449 if str(e) == "'ZenPack' object has no attribute '__of__'": 450 zenPack = ZenPack(packName) 451 else: 452 # This is the signature error of class-loading issues 453 # during zenpack upgrade. The final state should be okay, 454 # except that modified packables may be lost. 455 message = "There has been an error during the post-" + \ 456 "installation steps for the zenpack %s. In " + \ 457 "most cases, no further action is required. If " + \ 458 "issues persist, please reinstall this zenpack." 459 message = message % packName 460 log.warning( message ) 461 raise NonCriticalInstallError( message ) 462 463 if deferFileDeletion: 464 # We skipped deleting the existing files from filesystem 465 # because maybe they'd be needed in migrate scripts. 466 # Delete them now 467 oldZpDir = zenPath('Products', existing.id) 468 if os.path.islink(oldZpDir): 469 os.remove(oldZpDir) 470 else: 471 shutil.rmtree(oldZpDir) 472 473 cleanupSkins(dmd) 474 transaction.commit() 475 return zenPack

476 477

478 -def DiscoverEggs(dmd, zenPackId):

479 """ 480 Find installed eggs that provide a zenoss.zenpacks entry point. 481 Return a list of distributions whose ZenPacks need to be installed 482 or upgraded. The list is sorted into the order in which this needs to 483 happen. 484 """ 485 # Create a set of all available zenoss.zenpack entries that aren't 486 # already installed in zenoss or need to be upgraded in zenoss. 487 entries = set() 488 parse_version = pkg_resources.parse_version 489 for entry in pkg_resources.iter_entry_points(ZENPACK_ENTRY_POINT): 490 packName = entry.name 491 packVers = entry.dist.version 492 existing = dmd.ZenPackManager.packs._getOb(packName, None) 493 if existing and existing.isEggPack(): 494 # We use >= to allow migrate to be run on currently installed 495 # zenpacks whose version has been changed or for whom new 496 # migrates have been added. 497 if parse_version(packVers) >= parse_version(existing.version): 498 entries.add(entry) 499 else: 500 entries.add(entry) 501 502 # Starting with the entry representing zenPackId create a list of 503 # all entrypoints 504 505 # orderedEntries lists entries in the opposite order of that in which 506 # they need to be installed. This is simply for convenience of using 507 # .append() in code below. 508 orderedEntries = [] 509 entriesByName = dict([(e.name, e) for e in entries]) 510 511 def AddEntryAndProcessDeps(e): 512 orderedEntries.append(e) 513 for name in [r.project_name for r in e.dist.requires()]: 514 if name in [e.name for e in orderedEntries]: 515 # This entry depends on something that we've already processed. 516 # This might be a circular dependency, might not be. 517 # We are just going to bail however. This should be 518 # very unusual and the user can install deps first to work 519 # around. 520 raise ZenPackException('Unable to resolve ZenPack dependencies.' 521 ' Try installing dependencies first.') 522 if name in entriesByName: 523 # The requirement is an entry that has not yet been processed 524 # here. Add it to the list of entries to install/upgrade. 525 AddEntryAndProcessDeps(entriesByName[name]) 526 else: 527 # The requirement is not in the entries generated above. 528 # This either means that the dep is already installed (this 529 # is likely) or that easy_install missed something and the dep 530 # is not installed/available (this should be unlikely.) 531 pass

532 533 if zenPackId not in entriesByName: 534 if zenPackId in dmd.ZenPackManager.packs.objectIds(): 535 return [] 536 else: 537 raise ZenPackException('Unable to discover ZenPack named %s' % 538 zenPackId) 539 AddEntryAndProcessDeps(entriesByName[zenPackId]) 540 orderedEntries.reverse() 541 return [e.dist for e in orderedEntries] 542 543

544 -def AddDistToWorkingSet(distPath):

545 """ 546 Given the path to a dist (an egg) add it to the current working set. 547 This is basically a pkg_resources-friendly way of adding it to 548 sys.path. 549 Return a list of all distributions on distPath that appear to 550 be ZenPacks. 551 """ 552 zpDists = [] 553 for d in pkg_resources.find_distributions(distPath): 554 pkg_resources.working_set.add(d) 555 pkg_resources.require(d.project_name) 556 if d.project_name.startswith('ZenPacks.'): 557 zpDists.append(d) 558 return zpDists

559 560

561 -def ReadZenPackInfo(dist):

562 """ 563 Return a dictionary containing the egg metadata 564 """ 565 info = {} 566 if dist.has_metadata('PKG-INFO'): 567 lines = dist.get_metadata('PKG-INFO') 568 for line in pkg_resources.yield_lines(lines): 569 key, value = line.split(':', 1) 570 info[key.strip()] = value.strip() 571 if dist.has_metadata('zenpack_info'): 572 lines = dist.get_metadata('zenpack_info') 573 for line in pkg_resources.yield_lines(lines): 574 key, value = line.split(':', 1) 575 info[key.strip()] = value.strip() 576 return info

577 578

579 -def CopyMetaDataToZenPackObject(dist, pack):

580 """ 581 Copy metadata type stuff from the distribution to the zp object. 582 """ 583 # Version 584 pack.version = dist.version 585 586 # Egg Info 587 info = ReadZenPackInfo(dist) 588 pack.author = info.get('Author', '') 589 if pack.author == 'UNKNOWN': 590 pack.author = '' 591 pack.compatZenossVers = info.get('compatZenossVers', '') 592 pack.prevZenPackName = info.get('prevZenPackName', '') 593 594 # Requires 595 pack.dependencies = {} 596 for r in dist.requires(): 597 name = r.project_name 598 spec = str(r)[len(name):] 599 pack.dependencies[name] = spec

600 601

602 -def CreateZenPacksDir():

603 """ 604 Make sure $ZENHOME/ZenPacks exists 605 """ 606 zpDir = zenPath('ZenPacks') 607 if not os.path.isdir(zpDir): 608 os.mkdir(zpDir, 0750)

609 610

611 -def DoEasyInstall(eggPath):

612 """ 613 Use easy_install to install an egg from the filesystem. 614 easy_install will install the egg, but does not install it into 615 Zenoss as ZenPacks. 616 Returns a list of distributions that were installed that appear 617 to be ZenPacks. 618 """ 619 from setuptools.command import easy_install 620 621 # Make sure $ZENHOME/ZenPacks exists 622 CreateZenPacksDir() 623 624 # Create temp file for easy_install to write results to 625 _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall') 626 # eggPaths is a set of paths to eggs that were installed. We need to 627 # add them to the current workingset so we can discover their 628 # entry points. 629 eggPaths = set() 630 try: 631 # Execute the easy_install 632 args = ['--site-dirs', zenPath('ZenPacks'), 633 '-d', zenPath('ZenPacks'), 634 # '-i', ZEN_PACK_INDEX_URL, 635 '--allow-hosts', 'None', 636 '--record', tempPath, 637 '--quiet', 638 eggPath] 639 easy_install.main(args) 640 # Collect the paths for eggs that were installed 641 f = open(tempPath, 'r') 642 marker = '.egg/' 643 markerOffset = len(marker)-1 644 for l in f.readlines(): 645 i = l.find(marker) 646 if i > 0: 647 eggPaths.add(l[:i+markerOffset]) 648 finally: 649 os.remove(tempPath) 650 # Add any installed eggs to the current working set 651 zpDists = [] 652 for path in eggPaths: 653 zpDists += AddDistToWorkingSet(path) 654 return zpDists

655 656 657 ######################################## 658 # Zenoss.Net 659 ######################################## 660 661

662 -def FetchAndInstallZenPack(dmd, zenPackName, zenPackVersion='', sendEvent=True):

663 """ 664 Fetch the named zenpack and all its dependencies and install them. 665 Return a list of the ZenPacks that were installed. 666 """ 667 zenPacks = [] 668 try: 669 zpDists = FetchZenPack(zenPackName, zenPackVersion) 670 for d in zpDists: 671 zenPacks.append(InstallDistAsZenPack(dmd, d)) 672 except: 673 if sendEvent: 674 ZPEvent(dmd, 4, 'Failed to install ZenPack %s' % zenPackName, 675 '%s: %s' % sys.exc_info()[:2]) 676 raise 677 if sendEvent: 678 zenPackIds = [z.id for z in zenPacks] 679 if zenPackIds: 680 ZPEvent(dmd, 2, 'Installed ZenPacks: %s' % ', '.join(zenPackIds)) 681 if zenPackName not in zenPackIds: 682 ZPEvent(dmd, 4, 'Unable to install ZenPack %s' % zenPackName) 683 return zenPacks

684 685

686 -def FetchZenPack(zenPackName, zenPackVersion=''):

687 """ 688 Use easy_install to retrieve the given zenpack and any dependencies. 689 easy_install will install the eggs, but does not install them into 690 Zenoss as ZenPacks. 691 Return a list of distributions just installed that appear to be 692 ZenPacks. 693 694 NB: This should be refactored. It shares most of its code with 695 DoEasyInstall() 696 """ 697 from setuptools.command import easy_install 698 699 # Make sure $ZENHOME/ZenPacks exists 700 CreateZenPacksDir() 701 702 # Create temp file for easy_install to write results to 703 _, tempPath = tempfile.mkstemp(prefix='zenpackcmd-easyinstall') 704 # eggPaths is a set of paths to eggs that were installed. We need to 705 # add them to the current workingset so we can discover their 706 # entry points. 707 eggPaths = set() 708 try: 709 # Execute the easy_install 710 args = ['--site-dirs', zenPath('ZenPacks'), 711 '-d', zenPath('ZenPacks'), 712 '-i', ZEN_PACK_INDEX_URL, 713 '--allow-hosts', 'None', 714 '--record', tempPath, 715 '--quiet', 716 zenPackName] 717 easy_install.main(args) 718 # Collect the paths for eggs that were installed 719 f = open(tempPath, 'r') 720 marker = '.egg/' 721 markerOffset = len(marker)-1 722 for l in f.readlines(): 723 i = l.find(marker) 724 if i > 0: 725 eggPaths.add(l[:i+markerOffset]) 726 finally: 727 os.remove(tempPath) 728 # Add any installed eggs to the current working set 729 zpDists = [] 730 for path in eggPaths: 731 zpDists += AddDistToWorkingSet(path) 732 return zpDists

733 734

735 -def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):

736 """ 737 Upload the specified zenpack to the given project. 738 Project is a string of the form 'enterprise/myproject' or 739 'community/otherproject'. 740 """ 741 zp = dmd.ZenPackManager.packs._getOb(packName, None) 742 if not zp: 743 raise ZenPackException('No ZenPack named %s' % packName) 744 745 # Export the zenpack 746 fileName = zp.manage_exportPack() 747 filePath = zenPath('export', fileName) 748 749 # Login to Zenoss.net 750 from DotNetCommunication import DotNetSession 751 session = DotNetSession() 752 userSettings = dmd.ZenUsers.getUserSettings() 753 session.login(znetUser, znetPass) 754 755 # Upload 756 zpFile = open(zenPath('export', fileName), 'r') 757 try: 758 response = session.open('%s/createRelease' % project.strip('/'), { 759 'description': description, 760 'fileStorage': zpFile, 761 }) 762 finally: 763 zpFile.close() 764 if response: 765 result = response.read() 766 if "'success':true" not in result: 767 raise ZenPackException('Upload failed') 768 else: 769 raise ZenPackException('Failed to connect to Zenoss.net') 770 return

771 772 773 ######################################## 774 # ZenPack Removal 775 ######################################## 776 777

778 -def RemoveZenPack(dmd, packName, filesOnly=False, skipDepsCheck=False, 779 leaveObjects=False, sendEvent=True, 780 forceNoFileDeletion=False, uninstallEgg=True):

781 """ 782 Remove the given ZenPack from Zenoss. 783 Whether the ZenPack will be removed from the filesystem or not 784 depends on the result of the ZenPack's shouldDeleteFilesOnRemoval method. 785 """ 786 try: 787 if filesOnly: 788 skipDepsCheck = True 789 790 # Check for dependency implications here? 791 if not skipDepsCheck: 792 deps = GetDependents(dmd, packName) 793 if deps: 794 raise ZenPackDependentsException('%s cannot be removed ' % packName + 795 'because it is required by %s' % ', '.join(deps)) 796 797 if not filesOnly: 798 # Fetch the zenpack, call its remove() and remove from packs 799 zp = None 800 try: 801 zp = dmd.ZenPackManager.packs._getOb(packName) 802 except AttributeError, ex: 803 raise ZenPackNotFoundException('No ZenPack named %s is installed' % 804 packName) 805 # If zencatalog hasn't finished yet, we get ugly messages that don't 806 # mean anything. Hide them. 807 logFilter = None 808 if not getattr(dmd.zport, '_zencatalog_completed', False): 809 logFilter = CatalogLoggingFilter() 810 logging.getLogger('Zope.ZCatalog').addFilter(logFilter) 811 try: 812 zp.remove(dmd, leaveObjects) 813 dmd.ZenPackManager.packs._delObject(packName) 814 transaction.commit() 815 finally: 816 # Remove our logging filter so we don't hide anything important 817 if logFilter is not None: 818 logging.getLogger('Zope.ZCatalog').removeFilter(logFilter) 819 820 # Uninstall the egg and possibly delete it 821 # If we can't find the distribution then apparently the zp egg itself is 822 # missing. Continue on with the removal and swallow the 823 # DistributionNotFound exception 824 try: 825 dist = zp.getDistribution() 826 except pkg_resources.DistributionNotFound: 827 dist = None 828 if dist: 829 # Determine deleteFiles before develop -u gets called. Once 830 # it is called the egg has problems figuring out some of it's state. 831 deleteFiles = zp.shouldDeleteFilesOnRemoval() 832 if uninstallEgg: 833 if zp.isDevelopment(): 834 zenPackDir = zenPath('ZenPacks') 835 cmd = ('%s setup.py develop -u ' 836 % binPath('python') + 837 '--site-dirs=%s ' % zenPackDir + 838 '-d %s' % zenPackDir) 839 p = subprocess.Popen(cmd, 840 stdout=subprocess.PIPE, 841 stderr=subprocess.PIPE, 842 shell=True, 843 cwd=zp.eggPath()) 844 out, err = p.communicate() 845 code = p.wait() 846 if code: 847 raise ZenPackException(err) 848 else: 849 DoEasyUninstall(packName) 850 # elif cleanupEasyInstallPth: 851 # # Do we need to call easy_install -m here? It causes problems 852 # # because it tries to install deps. Cleanup easy-install.pth 853 # # ourselves instead. 854 # # We don't want to cleanup easy-install.pth when a newer 855 # # version of the egg has already been installed (when doing 856 # # an upgrade or installing in new location.) 857 # eggLink = './%s' % zp.eggName() 858 # CleanupEasyInstallPth(eggLink) 859 if deleteFiles and not forceNoFileDeletion: 860 eggDir = zp.eggPath() 861 if os.path.islink(eggDir): 862 os.remove(eggDir) 863 else: 864 shutil.rmtree(eggDir) 865 cleanupSkins(dmd) 866 transaction.commit() 867 except: 868 if sendEvent: 869 ZPEvent(dmd, 4, 'Error removing ZenPack %s' % packName, 870 '%s: %s' % sys.exc_info()[:2]) 871 raise 872 if sendEvent: 873 ZPEvent(dmd, 2, 'Removed ZenPack %s' % packName)

874 875

876 -def DoEasyUninstall(name):

877 """ 878 Execute the easy_install command to unlink the given egg. 879 What this is really doing is switching the egg to be in 880 multiple-version mode, however this is the first step in deleting 881 an egg as described here: 882 http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages 883 """ 884 from setuptools.command import easy_install 885 args = ['--site-dirs', zenPath('ZenPacks'), 886 '-d', zenPath('ZenPacks'), 887 #'--record', tempPath, 888 '--quiet', 889 '-m', 890 name] 891 easy_install.main(args)

892 893

894 -def CanRemoveZenPacks(dmd, packNames):

895 """ 896 Returns a tuple of (canRemove, otherDependents) 897 canRemove is True if the listed zenPacks have no dependents not also 898 listed in packNames, False otherwise. 899 otherDependents is a list of zenpack names not in packNames that 900 depend on one or more of the packs in packNames. 901 """ 902 unhappy = set() 903 for name in packNames: 904 deps = GetDependents(dmd, name) 905 unhappy.update(set([dep for dep in deps if dep not in packNames])) 906 return (not unhappy and True or False, list(unhappy))

907 908 909 # def CleanupEasyInstallPth(eggLink): 910 # """ 911 # Remove the entry for the given egg from the 912 # $ZENHOME/ZenPacks/easy-install.pth file. If this entry is left 913 # in place in can cause problems during the next install of the same 914 # egg. 915 # """ 916 # # Remove the path from easy-install.pth 917 # eggTail = os.path.split(eggLink)[1] 918 # easyPth = zenPath('ZenPacks', 'easy-install.pth') 919 # if os.path.isfile(easyPth): 920 # needToWrite = False 921 # newLines = [] 922 # f = open(easyPth, 'r') 923 # for line in f: 924 # if os.path.split(line.strip())[1] == eggTail: 925 # needToWrite = True 926 # else: 927 # newLines.append(line) 928 # f.close() 929 # if needToWrite: 930 # f = open(easyPth, 'w') 931 # f.writelines(newLines) 932 # f.close() 933 934

935 -def GetDependents(dmd, packName):

936 """ 937 Return a list of installed ZenPack ids that list packName as a dependency 938 """ 939 return [zp.id for zp in dmd.ZenPackManager.packs() 940 if zp.id != packName and zp.dependencies.has_key(packName)]

941 942 943 ######################################## 944 # __main__, dispatching, etc 945 ######################################## 946 947

948 -def ZPEvent(dmd, severity, summary, message=None):

949 """ 950 Send an event to Zenoss. 951 """ 952 dmd.ZenEventManager.sendEvent(dict( 953 device=FQDN, 954 eventClass='/Unknown', 955 severity=severity, 956 summary=summary, 957 message=message))

958 959

960 -class ZenPackCmd(ZenScriptBase):

961 """ 962 Utilities for creating, installing, removing ZenPacks. 963 964 NOTE: Users will probably invoke zenpack from the command line, which 965 runs zenpack.py rather than this file. zenpack.py calls functions 966 in this module when it detects that new-style (egg) ZenPacks are involved. 967 The plan is that once support for old-style (non-egg) ZenPacks is dropped 968 zenpack.py can go away and this will take its place. Until then this 969 script can be invoked directly via the zenpackcmd script if desired. 970 Invoking this script directly has the benefit of slightly better 971 progress/status output to stdout. 972 """ 973

974 - def run(self):

975 """ 976 Execute the user's request. 977 """ 978 979 self.connect() 980 def PrintInstalled(installed, eggOnly=False): 981 if installed: 982 if eggOnly: 983 names = [i['id'] for i in installed] 984 what = 'ZenPack egg' 985 else: 986 names = [i.id for i in installed] 987 what = 'ZenPack' 988 print('Installed %s%s: %s' % ( 989 what, 990 len(names) != 1 and 's' or '', 991 ', '.join(names))) 992 else: 993 print('No ZenPacks installed.')

994 995 if not getattr(self.dmd, 'ZenPackManager', None): 996 raise ZenPackNeedMigrateException('Your Zenoss database appears' 997 ' to be out of date. Try running zenmigrate to update.') 998 if self.options.eggOnly and self.options.eggPath: 999 zpDists = InstallEgg(self.dmd, self.options.eggPath, 1000 link=self.options.link) 1001 PrintInstalled([{'id':d.project_name} for d in zpDists], 1002 eggOnly=True) 1003 if self.options.eggPath: 1004 installed = InstallEggAndZenPack( 1005 self.dmd, self.options.eggPath, 1006 link=self.options.link, 1007 filesOnly=self.options.filesOnly, 1008 previousVersion= self.options.previousVersion) 1009 PrintInstalled(installed) 1010 elif self.options.fetch: 1011 installed = FetchAndInstallZenPack(self.dmd, self.options.fetch) 1012 PrintInstalled(installed) 1013 elif self.options.upload: 1014 return UploadZenPack(self.dmd, self.options.upload, 1015 self.options.znetProject, 1016 self.options.uploadDesc, 1017 self.options.znetUser, 1018 self.options.znetPass) 1019 elif self.options.removePackName: 1020 try: 1021 RemoveZenPack(self.dmd, self.options.removePackName) 1022 print('Removed ZenPack: %s' % self.options.removePackName) 1023 except ZenPackNotFoundException, e: 1024 sys.stderr.write(str(e) + '\n') 1025 elif self.options.list: 1026 self.list() 1027 else: 1028 self.parser.print_help()

1029 1030

1031 - def buildOptions(self):

1032 self.parser.add_option('--install', 1033 dest='eggPath', 1034 default=None, 1035 help="name of the pack to install") 1036 # self.parser.add_option('--fetch', 1037 # dest='fetch', 1038 # default=None, 1039 # help='Name of ZenPack to retrieve from ' 1040 # 'Zenoss.net and install.') 1041 # self.parser.add_option('--fetch-vers', 1042 # dest='fetchVers', 1043 # default=None, 1044 # help='Use with --fetch to specify a version' 1045 # ' for the ZenPack to download and install.') 1046 # self.parser.add_option('--znet-user', 1047 # dest='znetUser', 1048 # default=None, 1049 # help='Use with --fetch or --upload to specify' 1050 # ' your Zenoss.net username.') 1051 # self.parser.add_option('--znet-pass', 1052 # dest='znetPass', 1053 # default=None, 1054 # help='Use with --fetch or --upload to specify' 1055 # ' your Zenoss.net password.') 1056 # self.parser.add_option('--upload', 1057 # dest='upload', 1058 # default=None, 1059 # help='Name of ZenPack to upload to ' 1060 # 'Zenoss.net') 1061 # self.parser.add_option('--znet-project', 1062 # dest='znetProject', 1063 # default=None, 1064 # help='Use with --upload to specify' 1065 # ' which Zenoss.net project to create' 1066 # ' a release on.') 1067 # self.parser.add_option('--upload-desc', 1068 # dest='uploadDesc', 1069 # default=None, 1070 # help='Use with --upload to provide' 1071 # ' a description for the new release.') 1072 self.parser.add_option('--link', 1073 dest='link', 1074 action='store_true', 1075 default=False, 1076 help='Install the ZenPack in its current ' 1077 'location, do not copy to $ZENHOME/ZenPacks. ' 1078 'Also mark ZenPack as editable. ' 1079 'This only works with source directories ' 1080 'containing setup.py files, not ' 1081 'egg files.') 1082 self.parser.add_option('--remove', 1083 dest='removePackName', 1084 default=None, 1085 help="name of the pack to remove") 1086 self.parser.add_option('--leave-objects', 1087 dest='leaveObjects', 1088 default=False, 1089 action='store_true', 1090 help="When specified with --remove then objects" 1091 ' provided by the ZenPack and those' 1092 ' depending on the ZenPack are not deleted.' 1093 ' This may result in broken objects in your' 1094 ' database unless the ZenPack is' 1095 ' reinstalled.') 1096 self.parser.add_option('--files-only', 1097 dest='filesOnly', 1098 action="store_true", 1099 default=False, 1100 help='install onto filesystem but not into ' 1101 'zenoss') 1102 self.parser.add_option('--previousversion', 1103 dest='previousVersion', 1104 default=None, 1105 help="Previous version of the zenpack;" 1106 ' used during upgrades') 1107 ZenScriptBase.buildOptions(self)

1108 1109 1110 if __name__ == '__main__': 1111 try: 1112 zp = ZenPackCmd() 1113 zp.run() 1114 except ZenPackException, e: 1115 sys.stderr.write('%s\n' % str(e)) 1116 sys.exit(-1) 1117

Source Code for Module Products.ZenUtils.ZenPackCmd