Package Products :: Package ZenUtils :: Module ZenRestore |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenRestore

      1  #! /usr/bin/env python
      2  ##############################################################################
      3  #
      4  # Copyright (C) Zenoss, Inc. 2007, 2009, all rights reserved.
      5  #
      6  # This content is made available according to terms specified in
      7  # License.zenoss under the directory where your Zenoss product is installed.
      8  #
      9  ##############################################################################
     10
     11
     12  __doc__='''zenrestore
     13
     14  Restores a zenoss backup created by zenbackup.
     15  '''
     16
     17  import logging
     18  import sys
     19  import os
     20  import subprocess
     21  import tarfile
     22  import ConfigParser
     23
     24  import Globals
     25  from ZCmdBase import ZCmdBase
     26  from Products.ZenUtils.Utils import zenPath, binPath, requiresDaemonShutdown
     27
     28  from ZenBackupBase import *
     29
     30
     31 -class ZenRestore(ZenBackupBase):
     32
     33 -    def __init__(self):
     34          ZenBackupBase.__init__(self)
     35          self.log = logging.getLogger("zenrestore")
     36          logging.basicConfig()
     37          if self.options.verbose:
     38              self.log.setLevel(10)
     39          else:
     40              self.log.setLevel(40)
     41
     42 -    def buildOptions(self):
     43          """basic options setup sub classes can add more options here"""
     44          ZenBackupBase.buildOptions(self)
     45
     46          self.parser.add_option('--file',
     47                                 dest="file",
     48                                 default=None,
     49                                 help='File from which to restore.')
     50          self.parser.add_option('--dir',
     51                                 dest="dir",
     52                                 default=None,
     53                                 help='Path to an untarred backup file'
     54                                          ' from which to restore.')
     55          self.parser.add_option('--no-zodb',
     56                                 dest="noZODB",
     57                                 default=False,
     58                                 action='store_true',
     59                                 help='Do not restore the ZODB.')
     60          self.parser.add_option('--no-eventsdb',
     61                                 dest="noEventsDb",
     62                                 default=False,
     63                                 action='store_true',
     64                                 help='Do not restore the events database.')
     65          self.parser.add_option('--no-perfdata',
     66                                 dest="noPerfdata",
     67                                 default=False,
     68                                 action='store_true',
     69                                 help='Do not restore performance data.')
     70          self.parser.add_option('--deletePreviousPerfData',
     71                                 dest="deletePreviousPerfData",
     72                                 default=False,
     73                                 action='store_true',
     74                                 help='Delete ALL existing performance data before restoring?')
     75          self.parser.add_option('--zenpacks',
     76                                 dest='zenpacks',
     77                                 default=False,
     78                                 action='store_true',
     79                                 help=('Experimental: Restore any ZenPacks in '
     80                                       'the backup. Some ZenPacks may not work '
     81                                       'properly. Reinstall ZenPacks if possible'))
     82
     83 -    def getSettings(self):
     84          ''' Retrieve some options from settings file
     85          We want to take them in the following priority:
     86              1.  command line
     87              2.  settings file
     88              3.  defaults from build options
     89          '''
     90          try:
     91              f = open(os.path.join(self.tempDir, CONFIG_FILE), 'r')
     92          except Exception:
     93              return
     94          try:
     95              config = ConfigParser.SafeConfigParser()
     96              config.readfp(f)
     97              for name, value in config.items(CONFIG_SECTION):
     98                  # If we have a config, then change the default to match the config.
     99                  if name in self.parser.defaults:
    100                      self.parser.defaults[name] = value
    101                  else:
    102                      #we don't have that option with a default, so create it now
    103                      self.parser.add_option('--'+name,
    104                                             dest=name,
    105                                          default=value)
    106              #now reparse the command line to bring in anything the user actually set or the new defaults we created.
    107              (self.options, self.args) = self.parser.parse_args(args=self.inputArgs)
    108          finally:
    109              f.close()
    110
    111 -    def getSqlFile(self, filename):
    112          """
    113          Find path to sql file in backup, trying gzipped versions
    114          Returns path to real file, or none if nothing found
    115          """
    116          pathFile = os.path.join(self.tempDir, filename)
    117          for path in (pathFile, pathFile + '.gz'):
    118              if os.path.isfile(path):
    119                  return path
    120          return None
    121
    122 -    def restoreMySqlDb(self, host, port, db, user, passwd, sqlFile, socket=None):
    123          """
    124          Create MySQL database if it doesn't exist.
    125          """
    126          mysql_cmd = ['mysql', '-u%s' % user]
    127          mysql_cmd.extend(passwd)
    128          if host and host != 'localhost':
    129              mysql_cmd.extend(['--host', host])
    130              if self.options.compressTransport:
    131                  mysql_cmd.append('--compress')
    132          if port and str(port) != '3306':
    133              mysql_cmd.extend(['--port', str(port)])
    134          if socket:
    135              mysql_cmd.extend(['--socket', socket])
    136
    137          mysql_cmd = subprocess.list2cmdline(mysql_cmd)
    138
    139          cmd = 'echo "create database if not exists %s" | %s' % (db, mysql_cmd)
    140          os.system(cmd)
    141
    142          sql_path = os.path.join(self.tempDir, sqlFile)
    143          if sqlFile.endswith('.gz'):
    144              cmd_fmt = "gzip -dc {sql_path}"
    145          else:
    146              cmd_fmt = "cat {sql_path}"
    147          cmd_fmt += " | {mysql_cmd} {db}"
    148          cmd = cmd_fmt.format(**locals())
    149          os.system(cmd)
    150
    151
    152      @requiresDaemonShutdown('zeneventserver')
    153 -    def restoreZEP(self):
    154          '''
    155          Restore ZEP DB and indexes
    156          '''
    157          zepSql = self.getSqlFile('zep.sql')
    158          if not zepSql:
    159              self.msg('This backup does not contain a ZEP database backup.')
    160              return
    161
    162          # Setup defaults for db info
    163          if self.options.fetchArgs:
    164              self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
    165              self.readZEPSettings()
    166
    167          self.msg('Restoring ZEP database.')
    168          self.restoreMySqlDb(self.options.zepdbhost, self.options.zepdbport,
    169                              self.options.zepdbname, self.options.zepdbuser,
    170                              self.getPassArg('zepdbpass'), zepSql)
    171          self.msg('ZEP database restored.')
    172
    173          # Remove any current indexes on the system
    174          index_dir = zenPath('var', 'zeneventserver', 'index')
    175          if os.path.isdir(index_dir):
    176              import shutil
    177              self.msg('Removing existing ZEP indexes.')
    178              shutil.rmtree(index_dir)
    179
    180          index_tar = os.path.join(self.tempDir, 'zep.tar')
    181          if os.path.isfile(index_tar):
    182              self.msg('Restoring ZEP indexes.')
    183              zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'))
    184              zepTar.extractall(zenPath('var'))
    185              self.msg('ZEP indexes restored.')
    186          else:
    187              self.msg('ZEP indexes not found in backup file - will be recreated from database.')
    188
    189
    190
    191 -    def hasZeoBackup(self):
    192          repozoDir = os.path.join(self.tempDir, 'repozo')
    193          return os.path.isdir(repozoDir)
    194
    195 -    def hasSqlBackup(self):
    196          return bool(self.getSqlFile('zodb.sql'))
    197
    198 -    def hasZODBBackup(self):
    199          return self.hasZeoBackup() or self.hasSqlBackup()
    200
    201 -    def restoreZODB(self):
    202          # Relstorage may have already loaded items into the cache in the
    203          # initial connection to the database. We have to expire everything
    204          # in the cache in order to prevent errors with overlapping
    205          # transactions from the backup.
    206          if self.options.zodb_cacheservers:
    207              self.flush_memcached(self.options.zodb_cacheservers.split())
    208          if self.hasSqlBackup():
    209              self.restoreZODBSQL()
    210          elif self.hasZeoBackup():
    211              self.restoreZODBZEO()
    212
    213 -    def restoreZODBSQL(self):
    214          zodbSql = self.getSqlFile('zodb.sql')
    215          if not zodbSql:
    216              self.msg('This archive does not contain a ZODB backup.')
    217              return
    218          self.msg('Restoring ZODB database.')
    219          self.restoreMySqlDb(self.options.zodb_host, self.options.zodb_port,
    220                              self.options.zodb_db, self.options.zodb_user,
    221                              self.getPassArg('zodb_password'), zodbSql,
    222                              socket=self.options.zodb_socket)
    223          self.msg('Done Restoring ZODB database.')
    224
    225 -    def restoreZODBZEO(self):
    226          repozoDir = os.path.join(self.tempDir, 'repozo')
    227          tempFilePath = os.path.join(self.tempDir, 'Data.fs')
    228          tempZodbConvert = os.path.join(self.tempDir, 'convert.conf')
    229
    230          self.msg('Restoring ZEO backup into MySQL.')
    231
    232          # Create a Data.fs from the repozo backup
    233          cmd = []
    234          cmd.append(binPath('repozo'))
    235          cmd.append('--recover')
    236          cmd.append('--repository')
    237          cmd.append(repozoDir)
    238          cmd.append('--output')
    239          cmd.append(tempFilePath)
    240
    241          rc = subprocess.call(cmd, stdout=PIPE, stderr=PIPE)
    242          if rc:
    243              return -1
    244
    245          # Now we have a Data.fs, restore into MySQL with zodbconvert
    246          zodbconvert_conf = open(tempZodbConvert, 'w')
    247          zodbconvert_conf.write('<filestorage source>\n')
    248          zodbconvert_conf.write('  path %s\n' % tempFilePath)
    249          zodbconvert_conf.write('</filestorage>\n\n')
    250
    251          zodbconvert_conf.write('<relstorage destination>\n')
    252          zodbconvert_conf.write('  <mysql>\n')
    253          zodbconvert_conf.write('    host %s\n' % self.options.zodb_host)
    254          zodbconvert_conf.write('    port %s\n' % self.options.zodb_port)
    255          zodbconvert_conf.write('    db %s\n' % self.options.zodb_db)
    256          zodbconvert_conf.write('    user %s\n' % self.options.zodb_user)
    257          zodbconvert_conf.write('    passwd %s\n' % self.options.zodb_password or '')
    258          if self.options.zodb_socket:
    259              zodbconvert_conf.write('    unix_socket %s\n' % self.options.zodb_socket)
    260          zodbconvert_conf.write('  </mysql>\n')
    261          zodbconvert_conf.write('</relstorage>\n')
    262          zodbconvert_conf.close()
    263
    264          rc = subprocess.call(['zodbconvert', '--clear', tempZodbConvert],
    265                               stdout=PIPE, stderr=PIPE)
    266          if rc:
    267              return -1
    268
    269
    270 -    def restoreEtcFiles(self):
    271          self.msg('Restoring config files.')
    272          cmd = 'cp -p %s %s' % (os.path.join(zenPath('etc'), 'global.conf'), self.tempDir)
    273          if os.system(cmd): return -1
    274          cmd = 'rm -rf %s' % zenPath('etc')
    275          if os.system(cmd): return -1
    276          cmd = 'tar Cxf %s %s' % (
    277              zenPath(),
    278              os.path.join(self.tempDir, 'etc.tar')
    279          )
    280          if os.system(cmd): return -1
    281          if not os.path.exists(os.path.join(zenPath('etc'), 'global.conf')):
    282              self.msg('Restoring default global.conf')
    283              cmd = 'mv %s %s' % (os.path.join(self.tempDir, 'global.conf'), zenPath('etc'))
    284              if os.system(cmd): return -1
    285
    286 -    def restoreZenPacks(self):
    287          self.msg('Restoring ZenPacks.')
    288          cmd = 'rm -rf %s' % zenPath('ZenPacks')
    289          if os.system(cmd): return -1
    290          cmd = 'tar Cxf %s %s' % (
    291                          zenPath(),
    292                          os.path.join(self.tempDir, 'ZenPacks.tar'))
    293          if os.system(cmd): return -1
    294          # restore bin dir when restoring zenpacks
    295          #make sure bin dir is in tar
    296          tempBin = os.path.join(self.tempDir, 'bin.tar')
    297          if os.path.isfile(tempBin):
    298              self.msg('Restoring bin dir.')
    299              #k option prevents overwriting existing bin files
    300              cmd = ['tar', 'Cxfk', zenPath(),
    301                     os.path.join(self.tempDir, 'bin.tar')]
    302              self.runCommand(cmd)
    303
    304 -    def restoreZenPackContents(self):
    305          dmd = ZCmdBase(noopts=True).dmd
    306          self.log.info("Restoring ZenPack contents.")
    307          for pack in dmd.ZenPackManager.packs():
    308              pack.restore(self.tempDir, self.log)
    309          self.log.info("ZenPack contents restored.")
    310
    311 -    def restorePerfData(self):
    312          cmd = 'rm -rf %s' % os.path.join(zenPath(), 'perf')
    313          if os.system(cmd): return -1
    314          self.msg('Restoring performance data.')
    315          cmd = 'tar Cxf %s %s' % (
    316                          zenPath(),
    317                          os.path.join(self.tempDir, 'perf.tar'))
    318          if os.system(cmd): return -1
    319
    320 -    def doRestore(self):
    321          """
    322          Restore from a previous backup
    323          """
    324          if self.options.file and self.options.dir:
    325              sys.stderr.write('You cannot specify both --file and --dir.\n')
    326              sys.exit(-1)
    327          elif not self.options.file and not self.options.dir:
    328              sys.stderr.write('You must specify either --file or --dir.\n')
    329              sys.exit(-1)
    330
    331          # Maybe check to see if zeo is up and tell user to quit zenoss first
    332
    333          rootTempDir = ''
    334          if self.options.file:
    335              if not os.path.isfile(self.options.file):
    336                  sys.stderr.write('The specified backup file does not exist: %s\n' %
    337                        self.options.file)
    338                  sys.exit(-1)
    339              # Create temp dir and untar backup into it
    340              self.msg('Unpacking backup file')
    341              rootTempDir = self.getTempDir()
    342              cmd = 'tar xzfC %s %s' % (self.options.file, rootTempDir)
    343              if os.system(cmd): return -1
    344              self.tempDir = os.path.join(rootTempDir, BACKUP_DIR)
    345          else:
    346              self.msg('Using %s as source of restore' % self.options.dir)
    347              if not os.path.isdir(self.options.dir):
    348                  sys.stderr.write('The specified backup directory does not exist:'
    349                                  ' %s\n' % self.options.dir)
    350                  sys.exit(-1)
    351              self.tempDir = self.options.dir
    352
    353          # Maybe use values from backup file as defaults for self.options.
    354          self.getSettings()
    355
    356          if self.options.zenpacks and not self.hasZODBBackup():
    357              sys.stderr.write('Archive does not contain ZODB backup; cannot'
    358                               'restore ZenPacks')
    359              sys.exit(-1)
    360
    361          # ZODB
    362          if self.hasZODBBackup():
    363              self.restoreZODB()
    364          else:
    365              self.msg('Archive does not contain a ZODB backup')
    366
    367          # Configuration
    368          self.restoreEtcFiles()
    369
    370          # ZenPacks
    371          if self.options.zenpacks:
    372              tempPacks = os.path.join(self.tempDir, 'ZenPacks.tar')
    373              if os.path.isfile(tempPacks):
    374                  self.restoreZenPacks()
    375              else:
    376                  self.msg('Backup contains no ZenPacks.')
    377
    378          # Allow each installed ZenPack to restore state from the backup
    379          self.restoreZenPackContents()
    380
    381          # Performance Data
    382          tempPerf = os.path.join(self.tempDir, 'perf.tar')
    383          if os.path.isfile(tempPerf):
    384              self.restorePerfData()
    385          else:
    386              self.msg('Backup contains no perf data.')
    387
    388          # Events
    389          if self.options.noEventsDb:
    390              self.msg('Skipping the events database.')
    391          else:
    392              self.restoreZEP()
    393
    394          # clean up
    395          if self.options.file:
    396              self.msg('Cleaning up temporary files.')
    397              cmd = 'rm -r %s' % rootTempDir
    398              if os.system(cmd): return -1
    399
    400          self.msg('Restore complete.')
    401          # TODO: Audit from command-line without zenpacks loaded.
    402          # audit('Shell.Backup.Restore', file=self.options.file,
    403          #       dir=self.options.dir, zenpacks=self.options.zenpacks)
    404          return 0
    405
    406 -    def flush_memcached(self, cacheservers):
    407          self.msg('Flushing memcached cache.')
    408          import memcache
    409          mc = memcache.Client(cacheservers, debug=0)
    410          mc.flush_all()
    411          mc.disconnect_all()
    412          self.msg('Completed flushing memcached cache.')
    413
    414  if __name__ == '__main__':
    415      zb = ZenRestore()
    416      if zb.doRestore():
    417          sys.exit(-1)
    418