Package Products :: Package ZenUtils :: Module ZenBackup |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ZenBackup

      1  #! /usr/bin/env python
      2  ##############################################################################
      3  #
      4  # Copyright (C) Zenoss, Inc. 2007, 2009, all rights reserved.
      5  #
      6  # This content is made available according to terms specified in
      7  # License.zenoss under the directory where your Zenoss product is installed.
      8  #
      9  ##############################################################################
     10
     11
     12  __doc__='''zenbackup
     13
     14  Creates backup of Zope data files, Zenoss conf files and the events database.
     15  '''
     16
     17  import sys
     18  import os
     19  import os.path
     20  from datetime import date
     21  import time
     22  import logging
     23  import ConfigParser
     24  import subprocess
     25  import tarfile
     26  import re
     27  import gzip
     28  from itertools import imap
     29
     30  import Globals
     31  from ZCmdBase import ZCmdBase
     32  from Products.ZenUtils.Utils import zenPath, binPath, readable_time, unused
     33  from ZenBackupBase import *
     34
     35  unused(Globals)
     36
     37  MAX_UNIQUE_NAME_ATTEMPTS = 1000
     38
     39  DEFINER_PATTERN = re.compile(r'/\*((?!/\*).)*DEFINER.*?\*/')
     40
     41 -def strip_definer(mysqldump_line):
     42      """Strips DEFINER statements from mysqldump lines. See ZEN-326."""
     43      if not mysqldump_line.startswith("/*") or len(mysqldump_line) > 500:
     44          # speed things up, lines with DEFINER in them
     45          #    (1) start with '/*'
     46          #    (2) are shorter than 500 characters.
     47          return mysqldump_line
     48      return DEFINER_PATTERN.sub('', mysqldump_line)
     49
     50 -class ZenBackup(ZenBackupBase):
     51
     52 -    def __init__(self, argv):
     53          # Store sys.argv so we can restore it for ZCmdBase.
     54          self.argv = argv
     55
     56          ZenBackupBase.__init__(self)
     57          self.log = logging.getLogger("zenbackup")
     58          logging.basicConfig()
     59          self.log.setLevel(self.options.logseverity)
     60
     61 -    def isZeoUp(self):
     62          '''
     63          Returns True if Zeo appears to be running, false otherwise.
     64
     65          @return: whether Zeo is up or not
     66          @rtype: boolean
     67          '''
     68          import ZEO
     69          zeohome = os.path.dirname(ZEO.__file__)
     70          cmd = [ binPath('python'),
     71                  os.path.join(zeohome, 'scripts', 'zeoup.py')]
     72          cmd += '-p 8100 -h localhost'.split()
     73          self.log.debug("Can we access ZODB through Zeo?")
     74
     75          (output, warnings, returncode) = self.runCommand(cmd)
     76          if returncode:
     77              return False
     78          return output.startswith('Elapsed time:')
     79
     80 -    def saveSettings(self):
     81          '''
     82          Save the database credentials to a file for use during restore.
     83          '''
     84          config = ConfigParser.SafeConfigParser()
     85          config.add_section(CONFIG_SECTION)
     86
     87          config.set(CONFIG_SECTION, 'zodb_host', self.options.zodb_host)
     88          config.set(CONFIG_SECTION, 'zodb_port', str(self.options.zodb_port))
     89          config.set(CONFIG_SECTION, 'zodb_db', self.options.zodb_db)
     90          config.set(CONFIG_SECTION, 'zodb_user', self.options.zodb_user)
     91          config.set(CONFIG_SECTION, 'zodb_password', self.options.zodb_password)
     92          if self.options.zodb_socket:
     93              config.set(CONFIG_SECTION, 'zodb_socket', self.options.zodb_socket)
     94
     95          config.set(CONFIG_SECTION, 'zepdbhost', self.options.zepdbhost)
     96          config.set(CONFIG_SECTION, 'zepdbport', self.options.zepdbport)
     97          config.set(CONFIG_SECTION, 'zepdbname', self.options.zepdbname)
     98          config.set(CONFIG_SECTION, 'zepdbuser', self.options.zepdbuser)
     99          config.set(CONFIG_SECTION, 'zepdbpass', self.options.zepdbpass)
    100
    101          creds_file = os.path.join(self.tempDir, CONFIG_FILE)
    102          self.log.debug("Writing MySQL credentials to %s", creds_file)
    103          f = open(creds_file, 'w')
    104          try:
    105              config.write(f)
    106          finally:
    107              f.close()
    108
    109
    110 -    def getDefaultBackupFile(self):
    111          """
    112          Return a name for the backup file or die trying.
    113
    114          @return: unique name for a backup
    115          @rtype: string
    116          """
    117          def getName(index=0):
    118              """
    119              Try to create an unique backup file name.
    120
    121              @return: tar file name
    122              @rtype: string
    123              """
    124              return 'zenbackup_%s%s.tgz' % (date.today().strftime('%Y%m%d'),
    125                                              (index and '_%s' % index) or '')
    126          backupDir = zenPath('backups')
    127          if not os.path.exists(backupDir):
    128              os.mkdir(backupDir, 0750)
    129          for i in range(MAX_UNIQUE_NAME_ATTEMPTS):
    130              name = os.path.join(backupDir, getName(i))
    131              if not os.path.exists(name):
    132                  break
    133          else:
    134              self.log.critical('Cannot determine an unique file name to use'
    135                      ' in the backup directory (%s).' % backupDir +
    136                      ' Use --outfile to specify location for the backup'
    137                      ' file.\n')
    138              sys.exit(-1)
    139          return name
    140
    141
    142 -    def buildOptions(self):
    143          """
    144          Basic options setup
    145          """
    146          # pychecker can't handle strings made of multiple tokens
    147          __pychecker__ = 'no-noeffect no-constCond'
    148          ZenBackupBase.buildOptions(self)
    149          self.parser.add_option('--file',
    150                                 dest="file",
    151                                 default=None,
    152                                 help='Name of file in which the backup will be stored.'
    153                                       ' Backups will by default be placed'
    154                                       ' in $ZENHOME/backups/')
    155          self.parser.add_option('--no-eventsdb',
    156                                 dest="noEventsDb",
    157                                 default=False,
    158                                 action='store_true',
    159                                 help='Do not include the events database'
    160                                      ' in the backup.')
    161          self.parser.add_option('--no-zodb',
    162                                 dest="noZopeDb",
    163                                 default=False,
    164                                 action='store_true',
    165                                 help='Do not include the ZODB'
    166                                      ' in the backup.')
    167          self.parser.add_option('--no-perfdata',
    168                                 dest="noPerfData",
    169                                 default=False,
    170                                 action='store_true',
    171                                 help='Do not include performance data'
    172                                      ' in the backup.')
    173          self.parser.add_option('--no-zepindexes',
    174                                 dest='noZepIndexes',
    175                                 default=False,
    176                                 action='store_true',
    177                                 help='Do not include zep indexes in the backup')
    178          self.parser.add_option('--no-zenpacks',
    179                                 dest="noZenPacks",
    180                                 default=False,
    181                                 action='store_true',
    182                                 help='Do not include ZenPack data'
    183                                      ' in the backup.')
    184          self.parser.add_option('--stdout',
    185                                 dest="stdout",
    186                                 default=False,
    187                                 action='store_true',
    188                                 help='Send backup to stdout instead of to a file.')
    189          self.parser.add_option('--no-save-mysql-access',
    190                                  dest='saveSettings',
    191                                  default=True,
    192                                  action='store_false',
    193                                  help='Do not include zodb and zep credentials'
    194                                      ' in the backup file for use during restore.')
    195
    196          self.parser.remove_option('-v')
    197          self.parser.add_option('-v', '--logseverity',
    198                          dest='logseverity',
    199                          default=20,
    200                          type='int',
    201                          help='Logging severity threshold')
    202
    203 -    def backupMySqlDb(self, host, port, db, user, passwdType, sqlFile, socket=None, tables=None):
    204          command = ['mysqldump', '-u%s' %user, '--single-transaction']
    205          credential = self.getPassArg(passwdType)
    206          database = [db]
    207
    208          if host and host != 'localhost':
    209              command.append('-h%s' % host)
    210              if self.options.compressTransport:
    211                  command.append('--compress')
    212          if port and str(port) != '3306':
    213              command.append('--port=%s' % port)
    214          if socket:
    215              command.append('--socket=%s' % socket)
    216
    217          with gzip.open(os.path.join(self.tempDir, sqlFile),'wb') as gf:
    218              # If tables are specified, backup db schema and data from selected tables.
    219              if tables:
    220                  self.log.debug(' '.join(command + ['*' * 8] + ['--no-data'] + database))
    221                  schema = subprocess.Popen(command + credential + ['--no-data'] + database,
    222                      stdout=subprocess.PIPE)
    223                  gf.writelines(imap(strip_definer, schema.stdout))
    224                  schema_rc = schema.wait()
    225
    226                  self.log.debug(' '.join(command + ['*' * 8] + ['--no-create-info'] + database + tables))
    227                  data = subprocess.Popen(command + credential + ['--no-create-info'] + database + tables,
    228                      stdout=subprocess.PIPE)
    229                  gf.writelines(imap(strip_definer, data.stdout))
    230                  data_rc = data.wait()
    231              else:
    232                  self.log.debug(' '.join(command + ['*' * 8] + database))
    233                  schema = subprocess.Popen(command + credential + database,
    234                      stdout=subprocess.PIPE)
    235                  gf.writelines(imap(strip_definer, schema.stdout))
    236                  schema_rc = schema.wait()
    237
    238                  data_rc = 0
    239
    240              if schema_rc or data_rc:
    241                  self.log.critical("Backup of (%s) terminated abnormally." % sqlFile)
    242                  return -1
    243
    244 -    def _zepRunning(self):
    245          """
    246          Returns True if ZEP is running on the system (by invoking
    247          zeneventserver status).
    248          """
    249          zeneventserver_cmd = zenPath('bin', 'zeneventserver')
    250          with open(os.devnull, 'w') as devnull:
    251              return not subprocess.call([zeneventserver_cmd, 'status'], stdout=devnull, stderr=devnull)
    252
    253 -    def backupZEP(self):
    254          '''
    255          Backup ZEP
    256          '''
    257          partBeginTime = time.time()
    258
    259          # Setup defaults for db info
    260          if self.options.fetchArgs:
    261              self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
    262              self.readZEPSettings()
    263
    264          if self.options.saveSettings:
    265              self.saveSettings()
    266
    267          if self.options.noEventsDb:
    268              self.log.info('Doing a partial backup of the events database.')
    269              tables=['config','event_detail_index_config','event_trigger','event_trigger_subscription', 'schema_version']
    270          else:
    271              self.log.info('Backing up the events database.')
    272              tables = None
    273
    274          self.backupMySqlDb(self.options.zepdbhost, self.options.zepdbport,
    275                             self.options.zepdbname, self.options.zepdbuser,
    276                             'zepdbpass', 'zep.sql.gz', tables=tables)
    277
    278          partEndTime = time.time()
    279          subtotalTime = readable_time(partEndTime - partBeginTime)
    280          self.log.info("Backup of events database completed in %s.", subtotalTime)
    281
    282          if not self.options.noEventsDb:
    283              zeneventserver_dir = zenPath('var', 'zeneventserver')
    284              if self.options.noZepIndexes:
    285                  self.log.info('Not backing up event indexes.')
    286              elif self._zepRunning():
    287                  self.log.info('Not backing up event indexes - it is currently running.')
    288              elif os.path.isdir(zeneventserver_dir):
    289                  self.log.info('Backing up event indexes.')
    290                  zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'), 'w')
    291                  zepTar.add(zeneventserver_dir, 'zeneventserver')
    292                  zepTar.close()
    293                  self.log.info('Backing up event indexes completed.')
    294
    295 -    def backupZenPacks(self):
    296          """
    297          Backup the zenpacks dir
    298          """
    299          #can only copy zenpacks backups if ZEO is backed up
    300          if not self.options.noZopeDb and os.path.isdir(zenPath('ZenPacks')):
    301              # Copy /ZenPacks to backup dir
    302              self.log.info('Backing up ZenPacks.')
    303              etcTar = tarfile.open(os.path.join(self.tempDir, 'ZenPacks.tar'), 'w')
    304              etcTar.dereference = True
    305              etcTar.add(zenPath('ZenPacks'), 'ZenPacks')
    306              etcTar.close()
    307              self.log.info("Backup of ZenPacks completed.")
    308              # add /bin dir if backing up zenpacks
    309              # Copy /bin to backup dir
    310              self.log.info('Backing up bin dir.')
    311              etcTar = tarfile.open(os.path.join(self.tempDir, 'bin.tar'), 'w')
    312              etcTar.dereference = True
    313              etcTar.add(zenPath('bin'), 'bin')
    314              etcTar.close()
    315              self.log.info("Backup of bin completed.")
    316
    317 -    def backupZenPackContents(self):
    318          dmd = ZCmdBase(noopts=True).dmd
    319          self.log.info("Backing up ZenPack contents.")
    320          for pack in dmd.ZenPackManager.packs():
    321              pack.backup(self.tempDir, self.log)
    322          self.log.info("Backup of ZenPack contents complete.")
    323
    324 -    def backupZODB(self):
    325          """
    326          Backup the Zope database.
    327          """
    328          partBeginTime = time.time()
    329
    330          self.log.info('Backing up the ZODB.')
    331          if self.options.saveSettings:
    332              self.saveSettings()
    333          self.backupMySqlDb(self.options.zodb_host, self.options.zodb_port,
    334                             self.options.zodb_db, self.options.zodb_user,
    335                             'zodb_password', 'zodb.sql.gz',
    336                             socket=self.options.zodb_socket)
    337
    338          partEndTime = time.time()
    339          subtotalTime = readable_time(partEndTime - partBeginTime)
    340          self.log.info("Backup of ZODB database completed in %s.", subtotalTime)
    341
    342 -    def backupPerfData(self):
    343          """
    344          Back up the RRD files storing performance data.
    345          """
    346          perfDir = zenPath('perf')
    347          if not os.path.isdir(perfDir):
    348              self.log.warning('%s does not exist, skipping.', perfDir)
    349              return
    350
    351          partBeginTime = time.time()
    352
    353          self.log.info('Backing up performance data (RRDs).')
    354          tarFile = os.path.join(self.tempDir, 'perf.tar')
    355          #will change dir to ZENHOME so that tar dir structure is relative
    356          cmd = ['tar', 'chfC', tarFile, zenPath(), 'perf']
    357          (output, warnings, returncode) = self.runCommand(cmd)
    358          if returncode:
    359              self.log.critical("Backup terminated abnormally.")
    360              return -1
    361
    362          partEndTime = time.time()
    363          subtotalTime = readable_time(partEndTime - partBeginTime)
    364          self.log.info("Backup of performance data completed in %s.",
    365                        subtotalTime )
    366
    367
    368 -    def packageStagingBackups(self):
    369          """
    370          Gather all of the other data into one nice, neat file for easy
    371          tracking. Returns the filename created.
    372          """
    373          self.log.info('Packaging backup file.')
    374          if self.options.file:
    375              outfile = self.options.file
    376          else:
    377              outfile = self.getDefaultBackupFile()
    378          tempHead, tempTail = os.path.split(self.tempDir)
    379          tarFile = outfile
    380          if self.options.stdout:
    381              tarFile = '-'
    382          cmd = ['tar', 'czfC', tarFile, tempHead, tempTail]
    383          (output, warnings, returncode) = self.runCommand(cmd)
    384          if returncode:
    385              self.log.critical("Backup terminated abnormally.")
    386              return None
    387          self.log.info('Backup written to %s' % outfile)
    388          return outfile
    389
    390
    391 -    def cleanupTempDir(self):
    392          """
    393          Remove temporary files in staging directory.
    394          """
    395          self.log.info('Cleaning up staging directory %s' % self.rootTempDir)
    396          cmd = ['rm', '-r', self.rootTempDir]
    397          (output, warnings, returncode) = self.runCommand(cmd)
    398          if returncode:
    399              self.log.critical("Backup terminated abnormally.")
    400              return -1
    401
    402
    403 -    def makeBackup(self):
    404          '''
    405          Create a backup of the data and configuration for a Zenoss install.
    406          '''
    407          backupBeginTime = time.time()
    408
    409          # Create temp backup dir
    410          self.rootTempDir = self.getTempDir()
    411          self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
    412          self.log.debug("Use %s as a staging directory for the backup", self.tempDir)
    413          os.mkdir(self.tempDir, 0750)
    414
    415          # Do a full backup of zep if noEventsDb is false, otherwise only back
    416          # up a small subset of tables to capture the event triggers.
    417          if not self.options.noEventsDb or not self.options.noZopeDb:
    418              self.backupZEP()
    419          else:
    420              self.log.info('Skipping backup of the events database.')
    421
    422          if self.options.noZopeDb:
    423              self.log.info('Skipping backup of ZODB.')
    424          else:
    425              self.backupZODB()
    426
    427          # Copy /etc to backup dir (except for sockets)
    428          self.log.info('Backing up config files.')
    429          etcTar = tarfile.open(os.path.join(self.tempDir, 'etc.tar'), 'w')
    430          etcTar.dereference = True
    431          etcTar.add(zenPath('etc'), 'etc')
    432          etcTar.close()
    433          self.log.info("Backup of config files completed.")
    434
    435          if self.options.noZenPacks:
    436              self.log.info('Skipping backup of ZenPack data.')
    437          else:
    438              self.backupZenPacks()
    439              self.backupZenPackContents()
    440              self.log.info("Backup of ZenPacks completed.")
    441
    442          if self.options.noPerfData:
    443              self.log.info('Skipping backup of performance data.')
    444          else:
    445              self.backupPerfData()
    446
    447          # tar, gzip and send to outfile
    448          outfile = self.packageStagingBackups()
    449
    450          self.cleanupTempDir()
    451
    452          backupEndTime = time.time()
    453          totalBackupTime = readable_time(backupEndTime - backupBeginTime)
    454          self.log.info('Backup completed successfully in %s.', totalBackupTime)
    455          # TODO: There's no way to tell if this initiated through the UI.
    456          # audit('Shell.Backup.Create', file=outfile)
    457          return 0
    458
    459
    460  if __name__ == '__main__':
    461      zb = ZenBackup(sys.argv)
    462      if zb.makeBackup():
    463          sys.exit(-1)
    464