Products.ZenUtils.ZenRestore

32

33 - def __init__(self):

34 ZenBackupBase.__init__(self) 35 self.log = logging.getLogger("zenrestore") 36 logging.basicConfig() 37 if self.options.verbose: 38 self.log.setLevel(10) 39 else: 40 self.log.setLevel(40)

41

42 - def buildOptions(self):

43 """basic options setup sub classes can add more options here""" 44 ZenBackupBase.buildOptions(self) 45 46 self.parser.add_option('--file', 47 dest="file", 48 default=None, 49 help='File from which to restore.') 50 self.parser.add_option('--dir', 51 dest="dir", 52 default=None, 53 help='Path to an untarred backup file' 54 ' from which to restore.') 55 self.parser.add_option('--no-zodb', 56 dest="noZODB", 57 default=False, 58 action='store_true', 59 help='Do not restore the ZODB.') 60 self.parser.add_option('--no-eventsdb', 61 dest="noEventsDb", 62 default=False, 63 action='store_true', 64 help='Do not restore the events database.') 65 self.parser.add_option('--no-perfdata', 66 dest="noPerfdata", 67 default=False, 68 action='store_true', 69 help='Do not restore performance data.') 70 self.parser.add_option('--deletePreviousPerfData', 71 dest="deletePreviousPerfData", 72 default=False, 73 action='store_true', 74 help='Delete ALL existing performance data before restoring?') 75 self.parser.add_option('--zenpacks', 76 dest='zenpacks', 77 default=False, 78 action='store_true', 79 help=('Experimental: Restore any ZenPacks in ' 80 'the backup. Some ZenPacks may not work ' 81 'properly. Reinstall ZenPacks if possible'))

82

83 - def getSettings(self):

84 ''' Retrieve some options from settings file 85 We want to take them in the following priority: 86 1. command line 87 2. settings file 88 3. defaults from build options 89 ''' 90 try: 91 f = open(os.path.join(self.tempDir, CONFIG_FILE), 'r') 92 except Exception: 93 return 94 try: 95 config = ConfigParser.SafeConfigParser() 96 config.readfp(f) 97 for name, value in config.items(CONFIG_SECTION): 98 # If we have a config, then change the default to match the config. 99 if name in self.parser.defaults: 100 self.parser.defaults[name] = value 101 else: 102 #we don't have that option with a default, so create it now 103 self.parser.add_option('--'+name, 104 dest=name, 105 default=value) 106 #now reparse the command line to bring in anything the user actually set or the new defaults we created. 107 (self.options, self.args) = self.parser.parse_args(args=self.inputArgs) 108 finally: 109 f.close()

110

111 - def getSqlFile(self, filename):

112 """ 113 Find path to sql file in backup, trying gzipped versions 114 Returns path to real file, or none if nothing found 115 """ 116 pathFile = os.path.join(self.tempDir, filename) 117 for path in (pathFile, pathFile + '.gz'): 118 if os.path.isfile(path): 119 return path 120 return None

121

122 - def restoreMySqlDb(self, host, port, db, user, passwd, sqlFile, socket=None):

123 """ 124 Create MySQL database if it doesn't exist. 125 """ 126 mysql_cmd = ['mysql', '-u%s' % user] 127 mysql_cmd.extend(passwd) 128 if host and host != 'localhost': 129 mysql_cmd.extend(['--host', host]) 130 if self.options.compressTransport: 131 mysql_cmd.append('--compress') 132 if port and str(port) != '3306': 133 mysql_cmd.extend(['--port', str(port)]) 134 if socket: 135 mysql_cmd.extend(['--socket', socket]) 136 137 mysql_cmd = subprocess.list2cmdline(mysql_cmd) 138 139 cmd = 'echo "create database if not exists %s" | %s' % (db, mysql_cmd) 140 os.system(cmd) 141 142 sql_path = os.path.join(self.tempDir, sqlFile) 143 if sqlFile.endswith('.gz'): 144 cmd_fmt = "gzip -dc {sql_path}" 145 else: 146 cmd_fmt = "cat {sql_path}" 147 cmd_fmt += " | {mysql_cmd} {db}" 148 cmd = cmd_fmt.format(**locals()) 149 os.system(cmd)

150 151 152 @requiresDaemonShutdown('zeneventserver')

153 - def restoreZEP(self):

154 ''' 155 Restore ZEP DB and indexes 156 ''' 157 zepSql = self.getSqlFile('zep.sql') 158 if not zepSql: 159 self.msg('This backup does not contain a ZEP database backup.') 160 return 161 162 # Setup defaults for db info 163 if self.options.fetchArgs: 164 self.log.info('Getting ZEP dbname, user, password, port from configuration files.') 165 self.readZEPSettings() 166 167 self.msg('Restoring ZEP database.') 168 self.restoreMySqlDb(self.options.zepdbhost, self.options.zepdbport, 169 self.options.zepdbname, self.options.zepdbuser, 170 self.getPassArg('zepdbpass'), zepSql) 171 self.msg('ZEP database restored.') 172 173 # Remove any current indexes on the system 174 index_dir = zenPath('var', 'zeneventserver', 'index') 175 if os.path.isdir(index_dir): 176 import shutil 177 self.msg('Removing existing ZEP indexes.') 178 shutil.rmtree(index_dir) 179 180 index_tar = os.path.join(self.tempDir, 'zep.tar') 181 if os.path.isfile(index_tar): 182 self.msg('Restoring ZEP indexes.') 183 zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar')) 184 zepTar.extractall(zenPath('var')) 185 self.msg('ZEP indexes restored.') 186 else: 187 self.msg('ZEP indexes not found in backup file - will be recreated from database.')

188 189 190

191 - def hasZeoBackup(self):

192 repozoDir = os.path.join(self.tempDir, 'repozo') 193 return os.path.isdir(repozoDir)

194

195 - def hasSqlBackup(self):

196 return bool(self.getSqlFile('zodb.sql'))

197

198 - def hasZODBBackup(self):

199 return self.hasZeoBackup() or self.hasSqlBackup()

200

201 - def restoreZODB(self):

202 # Relstorage may have already loaded items into the cache in the 203 # initial connection to the database. We have to expire everything 204 # in the cache in order to prevent errors with overlapping 205 # transactions from the backup. 206 if self.options.zodb_cacheservers: 207 self.flush_memcached(self.options.zodb_cacheservers.split()) 208 if self.hasSqlBackup(): 209 self.restoreZODBSQL() 210 elif self.hasZeoBackup(): 211 self.restoreZODBZEO()

212

213 - def restoreZODBSQL(self):

214 zodbSql = self.getSqlFile('zodb.sql') 215 if not zodbSql: 216 self.msg('This archive does not contain a ZODB backup.') 217 return 218 self.msg('Restoring ZODB database.') 219 self.restoreMySqlDb(self.options.zodb_host, self.options.zodb_port, 220 self.options.zodb_db, self.options.zodb_user, 221 self.getPassArg('zodb_password'), zodbSql, 222 socket=self.options.zodb_socket) 223 self.msg('Done Restoring ZODB database.')

224

225 - def restoreZODBZEO(self):

226 repozoDir = os.path.join(self.tempDir, 'repozo') 227 tempFilePath = os.path.join(self.tempDir, 'Data.fs') 228 tempZodbConvert = os.path.join(self.tempDir, 'convert.conf') 229 230 self.msg('Restoring ZEO backup into MySQL.') 231 232 # Create a Data.fs from the repozo backup 233 cmd = [] 234 cmd.append(binPath('repozo')) 235 cmd.append('--recover') 236 cmd.append('--repository') 237 cmd.append(repozoDir) 238 cmd.append('--output') 239 cmd.append(tempFilePath) 240 241 rc = subprocess.call(cmd, stdout=PIPE, stderr=PIPE) 242 if rc: 243 return -1 244 245 # Now we have a Data.fs, restore into MySQL with zodbconvert 246 zodbconvert_conf = open(tempZodbConvert, 'w') 247 zodbconvert_conf.write('<filestorage source>\n') 248 zodbconvert_conf.write(' path %s\n' % tempFilePath) 249 zodbconvert_conf.write('</filestorage>\n\n') 250 251 zodbconvert_conf.write('<relstorage destination>\n') 252 zodbconvert_conf.write(' <mysql>\n') 253 zodbconvert_conf.write(' host %s\n' % self.options.zodb_host) 254 zodbconvert_conf.write(' port %s\n' % self.options.zodb_port) 255 zodbconvert_conf.write(' db %s\n' % self.options.zodb_db) 256 zodbconvert_conf.write(' user %s\n' % self.options.zodb_user) 257 zodbconvert_conf.write(' passwd %s\n' % self.options.zodb_password or '') 258 if self.options.zodb_socket: 259 zodbconvert_conf.write(' unix_socket %s\n' % self.options.zodb_socket) 260 zodbconvert_conf.write(' </mysql>\n') 261 zodbconvert_conf.write('</relstorage>\n') 262 zodbconvert_conf.close() 263 264 rc = subprocess.call(['zodbconvert', '--clear', tempZodbConvert], 265 stdout=PIPE, stderr=PIPE) 266 if rc: 267 return -1

268 269

270 - def restoreEtcFiles(self):

271 self.msg('Restoring config files.') 272 cmd = 'cp -p %s %s' % (os.path.join(zenPath('etc'), 'global.conf'), self.tempDir) 273 if os.system(cmd): return -1 274 cmd = 'rm -rf %s' % zenPath('etc') 275 if os.system(cmd): return -1 276 cmd = 'tar Cxf %s %s' % ( 277 zenPath(), 278 os.path.join(self.tempDir, 'etc.tar') 279 ) 280 if os.system(cmd): return -1 281 if not os.path.exists(os.path.join(zenPath('etc'), 'global.conf')): 282 self.msg('Restoring default global.conf') 283 cmd = 'mv %s %s' % (os.path.join(self.tempDir, 'global.conf'), zenPath('etc')) 284 if os.system(cmd): return -1

285

286 - def restoreZenPacks(self):

287 self.msg('Restoring ZenPacks.') 288 cmd = 'rm -rf %s' % zenPath('ZenPacks') 289 if os.system(cmd): return -1 290 cmd = 'tar Cxf %s %s' % ( 291 zenPath(), 292 os.path.join(self.tempDir, 'ZenPacks.tar')) 293 if os.system(cmd): return -1 294 # restore bin dir when restoring zenpacks 295 #make sure bin dir is in tar 296 tempBin = os.path.join(self.tempDir, 'bin.tar') 297 if os.path.isfile(tempBin): 298 self.msg('Restoring bin dir.') 299 #k option prevents overwriting existing bin files 300 cmd = ['tar', 'Cxfk', zenPath(), 301 os.path.join(self.tempDir, 'bin.tar')] 302 self.runCommand(cmd)

303

304 - def restoreZenPackContents(self):

305 dmd = ZCmdBase(noopts=True).dmd 306 self.log.info("Restoring ZenPack contents.") 307 for pack in dmd.ZenPackManager.packs(): 308 pack.restore(self.tempDir, self.log) 309 self.log.info("ZenPack contents restored.")

310

311 - def restorePerfData(self):

312 cmd = 'rm -rf %s' % os.path.join(zenPath(), 'perf') 313 if os.system(cmd): return -1 314 self.msg('Restoring performance data.') 315 cmd = 'tar Cxf %s %s' % ( 316 zenPath(), 317 os.path.join(self.tempDir, 'perf.tar')) 318 if os.system(cmd): return -1

319

320 - def doRestore(self):

321 """ 322 Restore from a previous backup 323 """ 324 if self.options.file and self.options.dir: 325 sys.stderr.write('You cannot specify both --file and --dir.\n') 326 sys.exit(-1) 327 elif not self.options.file and not self.options.dir: 328 sys.stderr.write('You must specify either --file or --dir.\n') 329 sys.exit(-1) 330 331 # Maybe check to see if zeo is up and tell user to quit zenoss first 332 333 rootTempDir = '' 334 if self.options.file: 335 if not os.path.isfile(self.options.file): 336 sys.stderr.write('The specified backup file does not exist: %s\n' % 337 self.options.file) 338 sys.exit(-1) 339 # Create temp dir and untar backup into it 340 self.msg('Unpacking backup file') 341 rootTempDir = self.getTempDir() 342 cmd = 'tar xzfC %s %s' % (self.options.file, rootTempDir) 343 if os.system(cmd): return -1 344 self.tempDir = os.path.join(rootTempDir, BACKUP_DIR) 345 else: 346 self.msg('Using %s as source of restore' % self.options.dir) 347 if not os.path.isdir(self.options.dir): 348 sys.stderr.write('The specified backup directory does not exist:' 349 ' %s\n' % self.options.dir) 350 sys.exit(-1) 351 self.tempDir = self.options.dir 352 353 # Maybe use values from backup file as defaults for self.options. 354 self.getSettings() 355 356 if self.options.zenpacks and not self.hasZODBBackup(): 357 sys.stderr.write('Archive does not contain ZODB backup; cannot' 358 'restore ZenPacks') 359 sys.exit(-1) 360 361 # ZODB 362 if self.hasZODBBackup(): 363 self.restoreZODB() 364 else: 365 self.msg('Archive does not contain a ZODB backup') 366 367 # Configuration 368 self.restoreEtcFiles() 369 370 # ZenPacks 371 if self.options.zenpacks: 372 tempPacks = os.path.join(self.tempDir, 'ZenPacks.tar') 373 if os.path.isfile(tempPacks): 374 self.restoreZenPacks() 375 else: 376 self.msg('Backup contains no ZenPacks.') 377 378 # Allow each installed ZenPack to restore state from the backup 379 self.restoreZenPackContents() 380 381 # Performance Data 382 tempPerf = os.path.join(self.tempDir, 'perf.tar') 383 if os.path.isfile(tempPerf): 384 self.restorePerfData() 385 else: 386 self.msg('Backup contains no perf data.') 387 388 # Events 389 if self.options.noEventsDb: 390 self.msg('Skipping the events database.') 391 else: 392 self.restoreZEP() 393 394 # clean up 395 if self.options.file: 396 self.msg('Cleaning up temporary files.') 397 cmd = 'rm -r %s' % rootTempDir 398 if os.system(cmd): return -1 399 400 self.msg('Restore complete.') 401 # TODO: Audit from command-line without zenpacks loaded. 402 # audit('Shell.Backup.Restore', file=self.options.file, 403 # dir=self.options.dir, zenpacks=self.options.zenpacks) 404 return 0

405

406 - def flush_memcached(self, cacheservers):

407 self.msg('Flushing memcached cache.') 408 import memcache 409 mc = memcache.Client(cacheservers, debug=0) 410 mc.flush_all() 411 mc.disconnect_all() 412 self.msg('Completed flushing memcached cache.')

Source Code for Module Products.ZenUtils.ZenRestore