Package Products :: Package ZenUtils :: Module ZenBackup
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenUtils.ZenBackup

  1  #! /usr/bin/env python 
  2  ############################################################################## 
  3  #  
  4  # Copyright (C) Zenoss, Inc. 2007, 2009, all rights reserved. 
  5  #  
  6  # This content is made available according to terms specified in 
  7  # License.zenoss under the directory where your Zenoss product is installed. 
  8  #  
  9  ############################################################################## 
 10   
 11   
 12  __doc__='''zenbackup 
 13   
 14  Creates backup of Zope data files, Zenoss conf files and the events database. 
 15  ''' 
 16   
 17  import sys 
 18  import os 
 19  import os.path 
 20  from datetime import date 
 21  import time 
 22  import logging 
 23  import ConfigParser 
 24  import subprocess 
 25  import tarfile 
 26  import re 
 27  import gzip 
 28  from itertools import imap 
 29   
 30  import Globals 
 31  from ZCmdBase import ZCmdBase 
 32  from Products.ZenUtils.Utils import zenPath, binPath, readable_time, unused 
 33  from ZenBackupBase import * 
 34   
 35  unused(Globals) 
 36   
 37  MAX_UNIQUE_NAME_ATTEMPTS = 1000 
 38   
 39  DEFINER_PATTERN = re.compile(r'/\*((?!/\*).)*DEFINER.*?\*/') 
 40   
41 -def strip_definer(mysqldump_line):
42 """Strips DEFINER statements from mysqldump lines. See ZEN-326.""" 43 if not mysqldump_line.startswith("/*") or len(mysqldump_line) > 500: 44 # speed things up, lines with DEFINER in them 45 # (1) start with '/*' 46 # (2) are shorter than 500 characters. 47 return mysqldump_line 48 return DEFINER_PATTERN.sub('', mysqldump_line)
49
50 -class ZenBackup(ZenBackupBase):
51
52 - def __init__(self, argv):
53 # Store sys.argv so we can restore it for ZCmdBase. 54 self.argv = argv 55 56 ZenBackupBase.__init__(self) 57 self.log = logging.getLogger("zenbackup") 58 logging.basicConfig() 59 self.log.setLevel(self.options.logseverity)
60
61 - def isZeoUp(self):
62 ''' 63 Returns True if Zeo appears to be running, false otherwise. 64 65 @return: whether Zeo is up or not 66 @rtype: boolean 67 ''' 68 import ZEO 69 zeohome = os.path.dirname(ZEO.__file__) 70 cmd = [ binPath('python'), 71 os.path.join(zeohome, 'scripts', 'zeoup.py')] 72 cmd += '-p 8100 -h localhost'.split() 73 self.log.debug("Can we access ZODB through Zeo?") 74 75 (output, warnings, returncode) = self.runCommand(cmd) 76 if returncode: 77 return False 78 return output.startswith('Elapsed time:')
79
80 - def saveSettings(self):
81 ''' 82 Save the database credentials to a file for use during restore. 83 ''' 84 config = ConfigParser.SafeConfigParser() 85 config.add_section(CONFIG_SECTION) 86 87 config.set(CONFIG_SECTION, 'zodb_host', self.options.zodb_host) 88 config.set(CONFIG_SECTION, 'zodb_port', str(self.options.zodb_port)) 89 config.set(CONFIG_SECTION, 'zodb_db', self.options.zodb_db) 90 config.set(CONFIG_SECTION, 'zodb_user', self.options.zodb_user) 91 config.set(CONFIG_SECTION, 'zodb_password', self.options.zodb_password) 92 if self.options.zodb_socket: 93 config.set(CONFIG_SECTION, 'zodb_socket', self.options.zodb_socket) 94 95 config.set(CONFIG_SECTION, 'zepdbhost', self.options.zepdbhost) 96 config.set(CONFIG_SECTION, 'zepdbport', self.options.zepdbport) 97 config.set(CONFIG_SECTION, 'zepdbname', self.options.zepdbname) 98 config.set(CONFIG_SECTION, 'zepdbuser', self.options.zepdbuser) 99 config.set(CONFIG_SECTION, 'zepdbpass', self.options.zepdbpass) 100 101 creds_file = os.path.join(self.tempDir, CONFIG_FILE) 102 self.log.debug("Writing MySQL credentials to %s", creds_file) 103 f = open(creds_file, 'w') 104 try: 105 config.write(f) 106 finally: 107 f.close()
108 109
110 - def getDefaultBackupFile(self):
111 """ 112 Return a name for the backup file or die trying. 113 114 @return: unique name for a backup 115 @rtype: string 116 """ 117 def getName(index=0): 118 """ 119 Try to create an unique backup file name. 120 121 @return: tar file name 122 @rtype: string 123 """ 124 return 'zenbackup_%s%s.tgz' % (date.today().strftime('%Y%m%d'), 125 (index and '_%s' % index) or '')
126 backupDir = zenPath('backups') 127 if not os.path.exists(backupDir): 128 os.mkdir(backupDir, 0750) 129 for i in range(MAX_UNIQUE_NAME_ATTEMPTS): 130 name = os.path.join(backupDir, getName(i)) 131 if not os.path.exists(name): 132 break 133 else: 134 self.log.critical('Cannot determine an unique file name to use' 135 ' in the backup directory (%s).' % backupDir + 136 ' Use --outfile to specify location for the backup' 137 ' file.\n') 138 sys.exit(-1) 139 return name
140 141
142 - def buildOptions(self):
143 """ 144 Basic options setup 145 """ 146 # pychecker can't handle strings made of multiple tokens 147 __pychecker__ = 'no-noeffect no-constCond' 148 ZenBackupBase.buildOptions(self) 149 self.parser.add_option('--file', 150 dest="file", 151 default=None, 152 help='Name of file in which the backup will be stored.' 153 ' Backups will by default be placed' 154 ' in $ZENHOME/backups/') 155 self.parser.add_option('--no-eventsdb', 156 dest="noEventsDb", 157 default=False, 158 action='store_true', 159 help='Do not include the events database' 160 ' in the backup.') 161 self.parser.add_option('--no-zodb', 162 dest="noZopeDb", 163 default=False, 164 action='store_true', 165 help='Do not include the ZODB' 166 ' in the backup.') 167 self.parser.add_option('--no-perfdata', 168 dest="noPerfData", 169 default=False, 170 action='store_true', 171 help='Do not include performance data' 172 ' in the backup.') 173 self.parser.add_option('--no-zepindexes', 174 dest='noZepIndexes', 175 default=False, 176 action='store_true', 177 help='Do not include zep indexes in the backup') 178 self.parser.add_option('--no-zenpacks', 179 dest="noZenPacks", 180 default=False, 181 action='store_true', 182 help='Do not include ZenPack data' 183 ' in the backup.') 184 self.parser.add_option('--stdout', 185 dest="stdout", 186 default=False, 187 action='store_true', 188 help='Send backup to stdout instead of to a file.') 189 self.parser.add_option('--no-save-mysql-access', 190 dest='saveSettings', 191 default=True, 192 action='store_false', 193 help='Do not include zodb and zep credentials' 194 ' in the backup file for use during restore.') 195 196 self.parser.remove_option('-v') 197 self.parser.add_option('-v', '--logseverity', 198 dest='logseverity', 199 default=20, 200 type='int', 201 help='Logging severity threshold')
202
203 - def backupMySqlDb(self, host, port, db, user, passwdType, sqlFile, socket=None, tables=None):
204 command = ['mysqldump', '-u%s' %user, '--single-transaction'] 205 credential = self.getPassArg(passwdType) 206 database = [db] 207 208 if host and host != 'localhost': 209 command.append('-h%s' % host) 210 if self.options.compressTransport: 211 command.append('--compress') 212 if port and str(port) != '3306': 213 command.append('--port=%s' % port) 214 if socket: 215 command.append('--socket=%s' % socket) 216 217 with gzip.open(os.path.join(self.tempDir, sqlFile),'wb') as gf: 218 # If tables are specified, backup db schema and data from selected tables. 219 if tables: 220 self.log.debug(' '.join(command + ['*' * 8] + ['--no-data'] + database)) 221 schema = subprocess.Popen(command + credential + ['--no-data'] + database, 222 stdout=subprocess.PIPE) 223 gf.writelines(imap(strip_definer, schema.stdout)) 224 schema_rc = schema.wait() 225 226 self.log.debug(' '.join(command + ['*' * 8] + ['--no-create-info'] + database + tables)) 227 data = subprocess.Popen(command + credential + ['--no-create-info'] + database + tables, 228 stdout=subprocess.PIPE) 229 gf.writelines(imap(strip_definer, data.stdout)) 230 data_rc = data.wait() 231 else: 232 self.log.debug(' '.join(command + ['*' * 8] + database)) 233 schema = subprocess.Popen(command + credential + database, 234 stdout=subprocess.PIPE) 235 gf.writelines(imap(strip_definer, schema.stdout)) 236 schema_rc = schema.wait() 237 238 data_rc = 0 239 240 if schema_rc or data_rc: 241 self.log.critical("Backup of (%s) terminated abnormally." % sqlFile) 242 return -1
243
244 - def _zepRunning(self):
245 """ 246 Returns True if ZEP is running on the system (by invoking 247 zeneventserver status). 248 """ 249 zeneventserver_cmd = zenPath('bin', 'zeneventserver') 250 with open(os.devnull, 'w') as devnull: 251 return not subprocess.call([zeneventserver_cmd, 'status'], stdout=devnull, stderr=devnull)
252
253 - def backupZEP(self):
254 ''' 255 Backup ZEP 256 ''' 257 partBeginTime = time.time() 258 259 # Setup defaults for db info 260 if self.options.fetchArgs: 261 self.log.info('Getting ZEP dbname, user, password, port from configuration files.') 262 self.readZEPSettings() 263 264 if self.options.saveSettings: 265 self.saveSettings() 266 267 if self.options.noEventsDb: 268 self.log.info('Doing a partial backup of the events database.') 269 tables=['config','event_detail_index_config','event_trigger','event_trigger_subscription', 'schema_version'] 270 else: 271 self.log.info('Backing up the events database.') 272 tables = None 273 274 self.backupMySqlDb(self.options.zepdbhost, self.options.zepdbport, 275 self.options.zepdbname, self.options.zepdbuser, 276 'zepdbpass', 'zep.sql.gz', tables=tables) 277 278 partEndTime = time.time() 279 subtotalTime = readable_time(partEndTime - partBeginTime) 280 self.log.info("Backup of events database completed in %s.", subtotalTime) 281 282 if not self.options.noEventsDb: 283 zeneventserver_dir = zenPath('var', 'zeneventserver') 284 if self.options.noZepIndexes: 285 self.log.info('Not backing up event indexes.') 286 elif self._zepRunning(): 287 self.log.info('Not backing up event indexes - it is currently running.') 288 elif os.path.isdir(zeneventserver_dir): 289 self.log.info('Backing up event indexes.') 290 zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'), 'w') 291 zepTar.add(zeneventserver_dir, 'zeneventserver') 292 zepTar.close() 293 self.log.info('Backing up event indexes completed.')
294
295 - def backupZenPacks(self):
296 """ 297 Backup the zenpacks dir 298 """ 299 #can only copy zenpacks backups if ZEO is backed up 300 if not self.options.noZopeDb and os.path.isdir(zenPath('ZenPacks')): 301 # Copy /ZenPacks to backup dir 302 self.log.info('Backing up ZenPacks.') 303 etcTar = tarfile.open(os.path.join(self.tempDir, 'ZenPacks.tar'), 'w') 304 etcTar.dereference = True 305 etcTar.add(zenPath('ZenPacks'), 'ZenPacks') 306 etcTar.close() 307 self.log.info("Backup of ZenPacks completed.") 308 # add /bin dir if backing up zenpacks 309 # Copy /bin to backup dir 310 self.log.info('Backing up bin dir.') 311 etcTar = tarfile.open(os.path.join(self.tempDir, 'bin.tar'), 'w') 312 etcTar.dereference = True 313 etcTar.add(zenPath('bin'), 'bin') 314 etcTar.close() 315 self.log.info("Backup of bin completed.")
316
317 - def backupZenPackContents(self):
318 dmd = ZCmdBase(noopts=True).dmd 319 self.log.info("Backing up ZenPack contents.") 320 for pack in dmd.ZenPackManager.packs(): 321 pack.backup(self.tempDir, self.log) 322 self.log.info("Backup of ZenPack contents complete.")
323
324 - def backupZODB(self):
325 """ 326 Backup the Zope database. 327 """ 328 partBeginTime = time.time() 329 330 self.log.info('Backing up the ZODB.') 331 if self.options.saveSettings: 332 self.saveSettings() 333 self.backupMySqlDb(self.options.zodb_host, self.options.zodb_port, 334 self.options.zodb_db, self.options.zodb_user, 335 'zodb_password', 'zodb.sql.gz', 336 socket=self.options.zodb_socket) 337 338 partEndTime = time.time() 339 subtotalTime = readable_time(partEndTime - partBeginTime) 340 self.log.info("Backup of ZODB database completed in %s.", subtotalTime)
341
342 - def backupPerfData(self):
343 """ 344 Back up the RRD files storing performance data. 345 """ 346 perfDir = zenPath('perf') 347 if not os.path.isdir(perfDir): 348 self.log.warning('%s does not exist, skipping.', perfDir) 349 return 350 351 partBeginTime = time.time() 352 353 self.log.info('Backing up performance data (RRDs).') 354 tarFile = os.path.join(self.tempDir, 'perf.tar') 355 #will change dir to ZENHOME so that tar dir structure is relative 356 cmd = ['tar', 'chfC', tarFile, zenPath(), 'perf'] 357 (output, warnings, returncode) = self.runCommand(cmd) 358 if returncode: 359 self.log.critical("Backup terminated abnormally.") 360 return -1 361 362 partEndTime = time.time() 363 subtotalTime = readable_time(partEndTime - partBeginTime) 364 self.log.info("Backup of performance data completed in %s.", 365 subtotalTime )
366 367
368 - def packageStagingBackups(self):
369 """ 370 Gather all of the other data into one nice, neat file for easy 371 tracking. Returns the filename created. 372 """ 373 self.log.info('Packaging backup file.') 374 if self.options.file: 375 outfile = self.options.file 376 else: 377 outfile = self.getDefaultBackupFile() 378 tempHead, tempTail = os.path.split(self.tempDir) 379 tarFile = outfile 380 if self.options.stdout: 381 tarFile = '-' 382 cmd = ['tar', 'czfC', tarFile, tempHead, tempTail] 383 (output, warnings, returncode) = self.runCommand(cmd) 384 if returncode: 385 self.log.critical("Backup terminated abnormally.") 386 return None 387 self.log.info('Backup written to %s' % outfile) 388 return outfile
389 390
391 - def cleanupTempDir(self):
392 """ 393 Remove temporary files in staging directory. 394 """ 395 self.log.info('Cleaning up staging directory %s' % self.rootTempDir) 396 cmd = ['rm', '-r', self.rootTempDir] 397 (output, warnings, returncode) = self.runCommand(cmd) 398 if returncode: 399 self.log.critical("Backup terminated abnormally.") 400 return -1
401 402
403 - def makeBackup(self):
404 ''' 405 Create a backup of the data and configuration for a Zenoss install. 406 ''' 407 backupBeginTime = time.time() 408 409 # Create temp backup dir 410 self.rootTempDir = self.getTempDir() 411 self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR) 412 self.log.debug("Use %s as a staging directory for the backup", self.tempDir) 413 os.mkdir(self.tempDir, 0750) 414 415 # Do a full backup of zep if noEventsDb is false, otherwise only back 416 # up a small subset of tables to capture the event triggers. 417 if not self.options.noEventsDb or not self.options.noZopeDb: 418 self.backupZEP() 419 else: 420 self.log.info('Skipping backup of the events database.') 421 422 if self.options.noZopeDb: 423 self.log.info('Skipping backup of ZODB.') 424 else: 425 self.backupZODB() 426 427 # Copy /etc to backup dir (except for sockets) 428 self.log.info('Backing up config files.') 429 etcTar = tarfile.open(os.path.join(self.tempDir, 'etc.tar'), 'w') 430 etcTar.dereference = True 431 etcTar.add(zenPath('etc'), 'etc') 432 etcTar.close() 433 self.log.info("Backup of config files completed.") 434 435 if self.options.noZenPacks: 436 self.log.info('Skipping backup of ZenPack data.') 437 else: 438 self.backupZenPacks() 439 self.backupZenPackContents() 440 self.log.info("Backup of ZenPacks completed.") 441 442 if self.options.noPerfData: 443 self.log.info('Skipping backup of performance data.') 444 else: 445 self.backupPerfData() 446 447 # tar, gzip and send to outfile 448 outfile = self.packageStagingBackups() 449 450 self.cleanupTempDir() 451 452 backupEndTime = time.time() 453 totalBackupTime = readable_time(backupEndTime - backupBeginTime) 454 self.log.info('Backup completed successfully in %s.', totalBackupTime) 455 # TODO: There's no way to tell if this initiated through the UI. 456 # audit('Shell.Backup.Create', file=outfile) 457 return 0
458 459 460 if __name__ == '__main__': 461 zb = ZenBackup(sys.argv) 462 if zb.makeBackup(): 463 sys.exit(-1) 464