Package Products :: Package ZenUtils :: Module ZenBackup |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenBackup

      1  #! /usr/bin/env python
      2  ###########################################################################
      3  #
      4  # This program is part of Zenoss Core, an open source monitoring platform.
      5  # Copyright (C) 2007, 2009 Zenoss Inc.
      6  #
      7  # This program is free software; you can redistribute it and/or modify it
      8  # under the terms of the GNU General Public License version 2 or (at your
      9  # option) any later version as published by the Free Software Foundation.
     10  #
     11  # For complete information please visit: http://www.zenoss.com/oss/
     12  #
     13  ###########################################################################
     14
     15
     16  __doc__='''zenbackup
     17
     18  Creates backup of Zope data files, Zenoss conf files and the events database.
     19  '''
     20
     21  import sys
     22  import os
     23  import os.path
     24  from datetime import date
     25  import time
     26  import logging
     27  import ConfigParser
     28  import subprocess
     29  import tarfile
     30
     31  import Globals
     32  from ZCmdBase import ZCmdBase
     33  from Products.ZenUtils.Utils import zenPath, binPath, readable_time
     34  from ZenBackupBase import *
     35
     36
     37  MAX_UNIQUE_NAME_ATTEMPTS = 1000
     38
     39
     40 -class ZenBackup(ZenBackupBase):
     41
     42 -    def __init__(self, argv):
     43          # Store sys.argv so we can restore it for ZCmdBase.
     44          self.argv = argv
     45
     46          ZenBackupBase.__init__(self)
     47          self.log = logging.getLogger("zenbackup")
     48          logging.basicConfig()
     49          self.log.setLevel(self.options.logseverity)
     50
     51 -    def isZeoUp(self):
     52          '''
     53          Returns True if Zeo appears to be running, false otherwise.
     54
     55          @return: whether Zeo is up or not
     56          @rtype: boolean
     57          '''
     58          import ZEO
     59          zeohome = os.path.dirname(ZEO.__file__)
     60          cmd = [ binPath('python'),
     61                  os.path.join(zeohome, 'scripts', 'zeoup.py')]
     62          cmd += '-p 8100 -h localhost'.split()
     63          self.log.debug("Can we access ZODB through Zeo?")
     64
     65          (output, warnings, returncode) = self.runCommand(cmd)
     66          if returncode:
     67              return False
     68          return output.startswith('Elapsed time:')
     69
     70 -    def readZEPSettings(self):
     71          '''
     72          Read in and store the ZEP DB configuration options from
     73          'zeneventserver.conf' to the 'options' object.
     74          '''
     75          zepconf = os.path.join(zenPath('etc'), 'zeneventserver.conf')
     76          zepsettings = {'username': 'zepdbuser', 'hostname': 'zepdbhost',
     77                         'dbname': 'zepdbname', 'password': 'zepdbpass',
     78                         'port': 'zepdbport'}
     79
     80          with open(zepconf) as f_zepconf:
     81              for line in f_zepconf:
     82                  line = line.strip()
     83                  if line.startswith('zep.jdbc.'):
     84                      (key, _ignored, value) = line[9:].partition('=')
     85                      if key in zepsettings:
     86                          setattr(self.options, zepsettings[key], value)
     87
     88 -    def saveSettings(self):
     89          '''
     90          Save the database credentials to a file for use during restore.
     91          '''
     92          config = ConfigParser.SafeConfigParser()
     93          config.add_section(CONFIG_SECTION)
     94
     95          config.set(CONFIG_SECTION, 'host', self.options.host)
     96          config.set(CONFIG_SECTION, 'port', self.options.port)
     97          config.set(CONFIG_SECTION, 'mysqldb', self.options.mysqldb)
     98          config.set(CONFIG_SECTION, 'mysqluser', self.options.mysqluser)
     99          config.set(CONFIG_SECTION, 'mysqlpasswd', self.options.mysqlpasswd)
    100
    101          config.set(CONFIG_SECTION, 'zepdbhost', self.options.zepdbhost)
    102          config.set(CONFIG_SECTION, 'zepdbport', self.options.zepdbport)
    103          config.set(CONFIG_SECTION, 'zepdbname', self.options.zepdbname)
    104          config.set(CONFIG_SECTION, 'zepdbuser', self.options.zepdbuser)
    105          config.set(CONFIG_SECTION, 'zepdbpass', self.options.zepdbpass)
    106
    107          creds_file = os.path.join(self.tempDir, CONFIG_FILE)
    108          self.log.debug("Writing MySQL credentials to %s", creds_file)
    109          f = open(creds_file, 'w')
    110          try:
    111              config.write(f)
    112          finally:
    113              f.close()
    114
    115
    116 -    def getDefaultBackupFile(self):
    117          """
    118          Return a name for the backup file or die trying.
    119
    120          @return: unique name for a backup
    121          @rtype: string
    122          """
    123          def getName(index=0):
    124              """
    125              Try to create an unique backup file name.
    126
    127              @return: tar file name
    128              @rtype: string
    129              """
    130              return 'zenbackup_%s%s.tgz' % (date.today().strftime('%Y%m%d'),
    131                                              (index and '_%s' % index) or '')
    132          backupDir = zenPath('backups')
    133          if not os.path.exists(backupDir):
    134              os.mkdir(backupDir, 0750)
    135          for i in range(MAX_UNIQUE_NAME_ATTEMPTS):
    136              name = os.path.join(backupDir, getName(i))
    137              if not os.path.exists(name):
    138                  break
    139          else:
    140              self.log.critical('Cannot determine an unique file name to use'
    141                      ' in the backup directory (%s).' % backupDir +
    142                      ' Use --outfile to specify location for the backup'
    143                      ' file.\n')
    144              sys.exit(-1)
    145          return name
    146
    147
    148 -    def buildOptions(self):
    149          """
    150          Basic options setup
    151          """
    152          # pychecker can't handle strings made of multiple tokens
    153          __pychecker__ = 'no-noeffect no-constCond'
    154          ZenBackupBase.buildOptions(self)
    155          self.parser.add_option('--dont-fetch-args',
    156                                  dest='fetchArgs',
    157                                  default=True,
    158                                  action='store_false',
    159                                  help='By default MySQL connection information'
    160                                      ' is retrieved from Zenoss if not'
    161                                      ' specified and if Zenoss is available.'
    162                                      ' This disables fetching of these values'
    163                                      ' from Zenoss.')
    164          self.parser.add_option('--file',
    165                                 dest="file",
    166                                 default=None,
    167                                 help='Name of file in which the backup will be stored.'
    168                                       ' Backups will by default be placed'
    169                                       ' in $ZENHOME/backups/')
    170          self.parser.add_option('--no-eventsdb',
    171                                 dest="noEventsDb",
    172                                 default=False,
    173                                 action='store_true',
    174                                 help='Do not include the events database'
    175                                      ' in the backup.')
    176          self.parser.add_option('--no-zodb',
    177                                 dest="noZopeDb",
    178                                 default=False,
    179                                 action='store_true',
    180                                 help='Do not include the ZODB'
    181                                      ' in the backup.')
    182          self.parser.add_option('--no-perfdata',
    183                                 dest="noPerfData",
    184                                 default=False,
    185                                 action='store_true',
    186                                 help='Do not include performance data'
    187                                      ' in the backup.')
    188          self.parser.add_option('--stdout',
    189                                 dest="stdout",
    190                                 default=False,
    191                                 action='store_true',
    192                                 help='Send backup to stdout instead of to a file.')
    193          self.parser.add_option('--no-save-mysql-access',
    194                                  dest='saveSettings',
    195                                  default=True,
    196                                  action='store_false',
    197                                  help='Do not include mysqldb, mysqluser, mysqlpasswd, zepdbname, zepdbuser, and zendbpass'
    198                                      ' in the backup'
    199                                      ' file for use during restore.')
    200
    201          self.parser.remove_option('-v')
    202          self.parser.add_option('-v', '--logseverity',
    203                          dest='logseverity',
    204                          default=20,
    205                          type='int',
    206                          help='Logging severity threshold')
    207
    208
    209 -    def backupMySqlDb(self, host, port, db, user, passwdType, sqlFile):
    210          cmd_p1 = ['mysqldump', '-u%s' % user]
    211          cmd_p2 = ['--single-transaction', db]
    212          if host and host != 'localhost':
    213              cmd_p2.append('-h%s' % host)
    214              if self.options.compressTransport:
    215                  cmd_p2.append('--compress')
    216          if port and port != '3306':
    217              cmd_p2.append('--port=%s' % port)
    218
    219          cmd = cmd_p1 + self.getPassArg(passwdType) + cmd_p2
    220          obfuscated_cmd = cmd_p1 + ['*' * 8] + cmd_p2
    221
    222          self.log.debug(' '.join(obfuscated_cmd))
    223
    224          with open(os.path.join(self.tempDir, sqlFile), 'wb') as zipfile:
    225              mysqldump = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    226              gzip = subprocess.Popen(['gzip', '-c'], stdin=mysqldump.stdout, stdout=zipfile)
    227              mysqldump.wait()
    228              gzip.wait()
    229              if gzip.returncode or mysqldump.returncode:
    230                  self.log.critical("Backup of (%s) terminated abnormally." % sqlFile)
    231                  return -1
    232
    233 -    def _zepRunning(self):
    234          """
    235          Returns True if ZEP is running on the system (by invoking
    236          zeneventserver status).
    237          """
    238          zeneventserver_cmd = zenPath('bin', 'zeneventserver')
    239          with open(os.devnull, 'w') as devnull:
    240              return not subprocess.call([zeneventserver_cmd, 'status'], stdout=devnull, stderr=devnull)
    241
    242 -    def backupZEP(self):
    243          '''
    244          Backup ZEP
    245          '''
    246          partBeginTime = time.time()
    247
    248          # Setup defaults for db info
    249          if self.options.fetchArgs and not self.options.noEventsDb:
    250              self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
    251              self.readZEPSettings()
    252
    253          self.log.info('Backing up the ZEP database.')
    254          if self.options.saveSettings:
    255              self.saveSettings()
    256
    257          self.backupMySqlDb(self.options.zepdbhost, self.options.zepdbport,
    258                             self.options.zepdbname, self.options.zepdbuser,
    259                             'zepdbpass', 'zep.sql.gz')
    260
    261          partEndTime = time.time()
    262          subtotalTime = readable_time(partEndTime - partBeginTime)
    263          self.log.info("Backup of ZEP database completed in %s.", subtotalTime)
    264
    265          zeneventserver_dir = zenPath('var', 'zeneventserver')
    266          if self._zepRunning():
    267              self.log.info('Not backing up ZEP indexes - it is currently running.')
    268          elif os.path.isdir(zeneventserver_dir):
    269              self.log.info('Backing up ZEP indexes.')
    270              zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'), 'w')
    271              zepTar.add(zeneventserver_dir, 'zeneventserver')
    272              zepTar.close()
    273              self.log.info('Backing up ZEP indexes completed.')
    274
    275 -    def backupZenPacks(self):
    276          """
    277          Backup the zenpacks dir
    278          """
    279          #can only copy zenpacks backups if ZEO is backed up
    280          if not self.options.noZopeDb and os.path.isdir(zenPath('ZenPacks')):
    281              # Copy /ZenPacks to backup dir
    282              self.log.info('Backing up ZenPacks.')
    283              etcTar = tarfile.open(os.path.join(self.tempDir, 'ZenPacks.tar'), 'w')
    284              etcTar.dereference = True
    285              etcTar.add(zenPath('ZenPacks'), 'ZenPacks')
    286              etcTar.close()
    287              self.log.info("Backup of ZenPacks completed.")
    288              # add /bin dir if backing up zenpacks
    289              # Copy /bin to backup dir
    290              self.log.info('Backing up bin dir.')
    291              etcTar = tarfile.open(os.path.join(self.tempDir, 'bin.tar'), 'w')
    292              etcTar.dereference = True
    293              etcTar.add(zenPath('bin'), 'bin')
    294              etcTar.close()
    295              self.log.info("Backup of bin completed.")
    296
    297 -    def backupZenPackContents(self):
    298          dmd = ZCmdBase(noopts=True).dmd
    299          self.log.info("Backing up ZenPack contents.")
    300          for pack in dmd.ZenPackManager.packs():
    301              pack.backup(self.tempDir, self.log)
    302          self.log.info("Backup of ZenPack contents complete.")
    303
    304 -    def backupZODB(self):
    305          """
    306          Backup the Zope database.
    307          """
    308          partBeginTime = time.time()
    309
    310          self.log.info('Backing up the ZODB.')
    311          if self.options.saveSettings:
    312              self.saveSettings()
    313
    314          self.backupMySqlDb(self.options.host, self.options.port,
    315                             self.options.mysqldb, self.options.mysqluser,
    316                             'mysqlpasswd', 'zodb.sql.gz')
    317
    318          partEndTime = time.time()
    319          subtotalTime = readable_time(partEndTime - partBeginTime)
    320          self.log.info("Backup of ZODB database completed in %s.", subtotalTime)
    321
    322
    323 -    def backupPerfData(self):
    324          """
    325          Back up the RRD files storing performance data.
    326          """
    327          perfDir = zenPath('perf')
    328          if not os.path.isdir(perfDir):
    329              self.log.warning('%s does not exist, skipping.', perfDir)
    330              return
    331
    332          partBeginTime = time.time()
    333
    334          self.log.info('Backing up performance data (RRDs).')
    335          tarFile = os.path.join(self.tempDir, 'perf.tar')
    336          #will change dir to ZENHOME so that tar dir structure is relative
    337          cmd = ['tar', 'chfC', tarFile, zenPath(), 'perf']
    338          (output, warnings, returncode) = self.runCommand(cmd)
    339          if returncode:
    340              self.log.critical("Backup terminated abnormally.")
    341              return -1
    342
    343          partEndTime = time.time()
    344          subtotalTime = readable_time(partEndTime - partBeginTime)
    345          self.log.info("Backup of performance data completed in %s.",
    346                        subtotalTime )
    347
    348
    349 -    def packageStagingBackups(self):
    350          """
    351          Gather all of the other data into one nice, neat file for easy
    352          tracking.
    353          """
    354          self.log.info('Packaging backup file.')
    355          if self.options.file:
    356              outfile = self.options.file
    357          else:
    358              outfile = self.getDefaultBackupFile()
    359          tempHead, tempTail = os.path.split(self.tempDir)
    360          tarFile = outfile
    361          if self.options.stdout:
    362              tarFile = '-'
    363          cmd = ['tar', 'czfC', tarFile, tempHead, tempTail]
    364          (output, warnings, returncode) = self.runCommand(cmd)
    365          if returncode:
    366              self.log.critical("Backup terminated abnormally.")
    367              return -1
    368          self.log.info('Backup written to %s' % outfile)
    369
    370
    371 -    def cleanupTempDir(self):
    372          """
    373          Remove temporary files in staging directory.
    374          """
    375          self.log.info('Cleaning up staging directory %s' % self.rootTempDir)
    376          cmd = ['rm', '-r', self.rootTempDir]
    377          (output, warnings, returncode) = self.runCommand(cmd)
    378          if returncode:
    379              self.log.critical("Backup terminated abnormally.")
    380              return -1
    381
    382
    383 -    def makeBackup(self):
    384          '''
    385          Create a backup of the data and configuration for a Zenoss install.
    386          '''
    387          backupBeginTime = time.time()
    388
    389          # Create temp backup dir
    390          self.rootTempDir = self.getTempDir()
    391          self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
    392          self.log.debug("Use %s as a staging directory for the backup", self.tempDir)
    393          os.mkdir(self.tempDir, 0750)
    394
    395          if self.options.noEventsDb:
    396              self.log.info('Skipping backup of events database.')
    397          else:
    398              self.backupZEP()
    399
    400          if self.options.noZopeDb:
    401              self.log.info('Skipping backup of ZODB.')
    402          else:
    403              self.backupZODB()
    404
    405          # Copy /etc to backup dir (except for sockets)
    406          self.log.info('Backing up config files.')
    407          etcTar = tarfile.open(os.path.join(self.tempDir, 'etc.tar'), 'w')
    408          etcTar.dereference = True
    409          etcTar.add(zenPath('etc'), 'etc')
    410          etcTar.close()
    411          self.log.info("Backup of config files completed.")
    412
    413          self.backupZenPacks()
    414          self.backupZenPackContents()
    415
    416          if self.options.noPerfData:
    417              self.log.info('Skipping backup of performance data.')
    418          else:
    419              self.backupPerfData()
    420
    421          # tar, gzip and send to outfile
    422          self.packageStagingBackups()
    423
    424          self.cleanupTempDir()
    425
    426          backupEndTime = time.time()
    427          totalBackupTime = readable_time(backupEndTime - backupBeginTime)
    428          self.log.info('Backup completed successfully in %s.', totalBackupTime)
    429          return 0
    430
    431
    432  if __name__ == '__main__':
    433      zb = ZenBackup(sys.argv)
    434      if zb.makeBackup():
    435          sys.exit(-1)
    436