1
2
3
4
5
6
7
8
9
10
11
12 __doc__='''zenbackup
13
14 Creates backup of Zope data files, Zenoss conf files and the events database.
15 '''
16
17 import sys
18 import os
19 import os.path
20 from datetime import date
21 import time
22 import logging
23 import ConfigParser
24 import subprocess
25 import tarfile
26 import re
27 import gzip
28 from itertools import imap
29
30 import Globals
31 from ZCmdBase import ZCmdBase
32 from Products.ZenUtils.Utils import zenPath, binPath, readable_time, unused
33 from ZenBackupBase import *
34
35 unused(Globals)
36
37 MAX_UNIQUE_NAME_ATTEMPTS = 1000
38
39 DEFINER_PATTERN = re.compile(r'/\*((?!/\*).)*DEFINER.*?\*/')
40
42 """Strips DEFINER statements from mysqldump lines. See ZEN-326."""
43 if not mysqldump_line.startswith("/*") or len(mysqldump_line) > 500:
44
45
46
47 return mysqldump_line
48 return DEFINER_PATTERN.sub('', mysqldump_line)
49
51
53
54 self.argv = argv
55
56 ZenBackupBase.__init__(self)
57 self.log = logging.getLogger("zenbackup")
58 logging.basicConfig()
59 self.log.setLevel(self.options.logseverity)
60
62 '''
63 Returns True if Zeo appears to be running, false otherwise.
64
65 @return: whether Zeo is up or not
66 @rtype: boolean
67 '''
68 import ZEO
69 zeohome = os.path.dirname(ZEO.__file__)
70 cmd = [ binPath('python'),
71 os.path.join(zeohome, 'scripts', 'zeoup.py')]
72 cmd += '-p 8100 -h localhost'.split()
73 self.log.debug("Can we access ZODB through Zeo?")
74
75 (output, warnings, returncode) = self.runCommand(cmd)
76 if returncode:
77 return False
78 return output.startswith('Elapsed time:')
79
81 '''
82 Save the database credentials to a file for use during restore.
83 '''
84 config = ConfigParser.SafeConfigParser()
85 config.add_section(CONFIG_SECTION)
86
87 config.set(CONFIG_SECTION, 'zodb_host', self.options.zodb_host)
88 config.set(CONFIG_SECTION, 'zodb_port', str(self.options.zodb_port))
89 config.set(CONFIG_SECTION, 'zodb_db', self.options.zodb_db)
90 config.set(CONFIG_SECTION, 'zodb_user', self.options.zodb_user)
91 config.set(CONFIG_SECTION, 'zodb_password', self.options.zodb_password)
92 if self.options.zodb_socket:
93 config.set(CONFIG_SECTION, 'zodb_socket', self.options.zodb_socket)
94
95 config.set(CONFIG_SECTION, 'zepdbhost', self.options.zepdbhost)
96 config.set(CONFIG_SECTION, 'zepdbport', self.options.zepdbport)
97 config.set(CONFIG_SECTION, 'zepdbname', self.options.zepdbname)
98 config.set(CONFIG_SECTION, 'zepdbuser', self.options.zepdbuser)
99 config.set(CONFIG_SECTION, 'zepdbpass', self.options.zepdbpass)
100
101 creds_file = os.path.join(self.tempDir, CONFIG_FILE)
102 self.log.debug("Writing MySQL credentials to %s", creds_file)
103 f = open(creds_file, 'w')
104 try:
105 config.write(f)
106 finally:
107 f.close()
108
109
111 """
112 Return a name for the backup file or die trying.
113
114 @return: unique name for a backup
115 @rtype: string
116 """
117 def getName(index=0):
118 """
119 Try to create an unique backup file name.
120
121 @return: tar file name
122 @rtype: string
123 """
124 return 'zenbackup_%s%s.tgz' % (date.today().strftime('%Y%m%d'),
125 (index and '_%s' % index) or '')
126 backupDir = zenPath('backups')
127 if not os.path.exists(backupDir):
128 os.mkdir(backupDir, 0750)
129 for i in range(MAX_UNIQUE_NAME_ATTEMPTS):
130 name = os.path.join(backupDir, getName(i))
131 if not os.path.exists(name):
132 break
133 else:
134 self.log.critical('Cannot determine an unique file name to use'
135 ' in the backup directory (%s).' % backupDir +
136 ' Use --outfile to specify location for the backup'
137 ' file.\n')
138 sys.exit(-1)
139 return name
140
141
143 """
144 Basic options setup
145 """
146
147 __pychecker__ = 'no-noeffect no-constCond'
148 ZenBackupBase.buildOptions(self)
149 self.parser.add_option('--file',
150 dest="file",
151 default=None,
152 help='Name of file in which the backup will be stored.'
153 ' Backups will by default be placed'
154 ' in $ZENHOME/backups/')
155 self.parser.add_option('--no-eventsdb',
156 dest="noEventsDb",
157 default=False,
158 action='store_true',
159 help='Do not include the events database'
160 ' in the backup.')
161 self.parser.add_option('--no-zodb',
162 dest="noZopeDb",
163 default=False,
164 action='store_true',
165 help='Do not include the ZODB'
166 ' in the backup.')
167 self.parser.add_option('--no-perfdata',
168 dest="noPerfData",
169 default=False,
170 action='store_true',
171 help='Do not include performance data'
172 ' in the backup.')
173 self.parser.add_option('--no-zepindexes',
174 dest='noZepIndexes',
175 default=False,
176 action='store_true',
177 help='Do not include zep indexes in the backup')
178 self.parser.add_option('--no-zenpacks',
179 dest="noZenPacks",
180 default=False,
181 action='store_true',
182 help='Do not include ZenPack data'
183 ' in the backup.')
184 self.parser.add_option('--stdout',
185 dest="stdout",
186 default=False,
187 action='store_true',
188 help='Send backup to stdout instead of to a file.')
189 self.parser.add_option('--no-save-mysql-access',
190 dest='saveSettings',
191 default=True,
192 action='store_false',
193 help='Do not include zodb and zep credentials'
194 ' in the backup file for use during restore.')
195
196 self.parser.remove_option('-v')
197 self.parser.add_option('-v', '--logseverity',
198 dest='logseverity',
199 default=20,
200 type='int',
201 help='Logging severity threshold')
202
203 - def backupMySqlDb(self, host, port, db, user, passwdType, sqlFile, socket=None, tables=None):
204 command = ['mysqldump', '-u%s' %user, '--single-transaction']
205 credential = self.getPassArg(passwdType)
206 database = [db]
207
208 if host and host != 'localhost':
209 command.append('-h%s' % host)
210 if self.options.compressTransport:
211 command.append('--compress')
212 if port and str(port) != '3306':
213 command.append('--port=%s' % port)
214 if socket:
215 command.append('--socket=%s' % socket)
216
217 with gzip.open(os.path.join(self.tempDir, sqlFile),'wb') as gf:
218
219 if tables:
220 self.log.debug(' '.join(command + ['*' * 8] + ['--no-data'] + database))
221 schema = subprocess.Popen(command + credential + ['--no-data'] + database,
222 stdout=subprocess.PIPE)
223 gf.writelines(imap(strip_definer, schema.stdout))
224 schema_rc = schema.wait()
225
226 self.log.debug(' '.join(command + ['*' * 8] + ['--no-create-info'] + database + tables))
227 data = subprocess.Popen(command + credential + ['--no-create-info'] + database + tables,
228 stdout=subprocess.PIPE)
229 gf.writelines(imap(strip_definer, data.stdout))
230 data_rc = data.wait()
231 else:
232 self.log.debug(' '.join(command + ['*' * 8] + database))
233 schema = subprocess.Popen(command + credential + database,
234 stdout=subprocess.PIPE)
235 gf.writelines(imap(strip_definer, schema.stdout))
236 schema_rc = schema.wait()
237
238 data_rc = 0
239
240 if schema_rc or data_rc:
241 self.log.critical("Backup of (%s) terminated abnormally." % sqlFile)
242 return -1
243
245 """
246 Returns True if ZEP is running on the system (by invoking
247 zeneventserver status).
248 """
249 zeneventserver_cmd = zenPath('bin', 'zeneventserver')
250 with open(os.devnull, 'w') as devnull:
251 return not subprocess.call([zeneventserver_cmd, 'status'], stdout=devnull, stderr=devnull)
252
254 '''
255 Backup ZEP
256 '''
257 partBeginTime = time.time()
258
259
260 if self.options.fetchArgs:
261 self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
262 self.readZEPSettings()
263
264 if self.options.saveSettings:
265 self.saveSettings()
266
267 if self.options.noEventsDb:
268 self.log.info('Doing a partial backup of the events database.')
269 tables=['config','event_detail_index_config','event_trigger','event_trigger_subscription', 'schema_version']
270 else:
271 self.log.info('Backing up the events database.')
272 tables = None
273
274 self.backupMySqlDb(self.options.zepdbhost, self.options.zepdbport,
275 self.options.zepdbname, self.options.zepdbuser,
276 'zepdbpass', 'zep.sql.gz', tables=tables)
277
278 partEndTime = time.time()
279 subtotalTime = readable_time(partEndTime - partBeginTime)
280 self.log.info("Backup of events database completed in %s.", subtotalTime)
281
282 if not self.options.noEventsDb:
283 zeneventserver_dir = zenPath('var', 'zeneventserver')
284 if self.options.noZepIndexes:
285 self.log.info('Not backing up event indexes.')
286 elif self._zepRunning():
287 self.log.info('Not backing up event indexes - it is currently running.')
288 elif os.path.isdir(zeneventserver_dir):
289 self.log.info('Backing up event indexes.')
290 zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'), 'w')
291 zepTar.add(zeneventserver_dir, 'zeneventserver')
292 zepTar.close()
293 self.log.info('Backing up event indexes completed.')
294
296 """
297 Backup the zenpacks dir
298 """
299
300 if not self.options.noZopeDb and os.path.isdir(zenPath('ZenPacks')):
301
302 self.log.info('Backing up ZenPacks.')
303 etcTar = tarfile.open(os.path.join(self.tempDir, 'ZenPacks.tar'), 'w')
304 etcTar.dereference = True
305 etcTar.add(zenPath('ZenPacks'), 'ZenPacks')
306 etcTar.close()
307 self.log.info("Backup of ZenPacks completed.")
308
309
310 self.log.info('Backing up bin dir.')
311 etcTar = tarfile.open(os.path.join(self.tempDir, 'bin.tar'), 'w')
312 etcTar.dereference = True
313 etcTar.add(zenPath('bin'), 'bin')
314 etcTar.close()
315 self.log.info("Backup of bin completed.")
316
318 dmd = ZCmdBase(noopts=True).dmd
319 self.log.info("Backing up ZenPack contents.")
320 for pack in dmd.ZenPackManager.packs():
321 pack.backup(self.tempDir, self.log)
322 self.log.info("Backup of ZenPack contents complete.")
323
325 """
326 Backup the Zope database.
327 """
328 partBeginTime = time.time()
329
330 self.log.info('Backing up the ZODB.')
331 if self.options.saveSettings:
332 self.saveSettings()
333 self.backupMySqlDb(self.options.zodb_host, self.options.zodb_port,
334 self.options.zodb_db, self.options.zodb_user,
335 'zodb_password', 'zodb.sql.gz',
336 socket=self.options.zodb_socket)
337
338 partEndTime = time.time()
339 subtotalTime = readable_time(partEndTime - partBeginTime)
340 self.log.info("Backup of ZODB database completed in %s.", subtotalTime)
341
343 """
344 Back up the RRD files storing performance data.
345 """
346 perfDir = zenPath('perf')
347 if not os.path.isdir(perfDir):
348 self.log.warning('%s does not exist, skipping.', perfDir)
349 return
350
351 partBeginTime = time.time()
352
353 self.log.info('Backing up performance data (RRDs).')
354 tarFile = os.path.join(self.tempDir, 'perf.tar')
355
356 cmd = ['tar', 'chfC', tarFile, zenPath(), 'perf']
357 (output, warnings, returncode) = self.runCommand(cmd)
358 if returncode:
359 self.log.critical("Backup terminated abnormally.")
360 return -1
361
362 partEndTime = time.time()
363 subtotalTime = readable_time(partEndTime - partBeginTime)
364 self.log.info("Backup of performance data completed in %s.",
365 subtotalTime )
366
367
369 """
370 Gather all of the other data into one nice, neat file for easy
371 tracking. Returns the filename created.
372 """
373 self.log.info('Packaging backup file.')
374 if self.options.file:
375 outfile = self.options.file
376 else:
377 outfile = self.getDefaultBackupFile()
378 tempHead, tempTail = os.path.split(self.tempDir)
379 tarFile = outfile
380 if self.options.stdout:
381 tarFile = '-'
382 cmd = ['tar', 'czfC', tarFile, tempHead, tempTail]
383 (output, warnings, returncode) = self.runCommand(cmd)
384 if returncode:
385 self.log.critical("Backup terminated abnormally.")
386 return None
387 self.log.info('Backup written to %s' % outfile)
388 return outfile
389
390
392 """
393 Remove temporary files in staging directory.
394 """
395 self.log.info('Cleaning up staging directory %s' % self.rootTempDir)
396 cmd = ['rm', '-r', self.rootTempDir]
397 (output, warnings, returncode) = self.runCommand(cmd)
398 if returncode:
399 self.log.critical("Backup terminated abnormally.")
400 return -1
401
402
404 '''
405 Create a backup of the data and configuration for a Zenoss install.
406 '''
407 backupBeginTime = time.time()
408
409
410 self.rootTempDir = self.getTempDir()
411 self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
412 self.log.debug("Use %s as a staging directory for the backup", self.tempDir)
413 os.mkdir(self.tempDir, 0750)
414
415
416
417 if not self.options.noEventsDb or not self.options.noZopeDb:
418 self.backupZEP()
419 else:
420 self.log.info('Skipping backup of the events database.')
421
422 if self.options.noZopeDb:
423 self.log.info('Skipping backup of ZODB.')
424 else:
425 self.backupZODB()
426
427
428 self.log.info('Backing up config files.')
429 etcTar = tarfile.open(os.path.join(self.tempDir, 'etc.tar'), 'w')
430 etcTar.dereference = True
431 etcTar.add(zenPath('etc'), 'etc')
432 etcTar.close()
433 self.log.info("Backup of config files completed.")
434
435 if self.options.noZenPacks:
436 self.log.info('Skipping backup of ZenPack data.')
437 else:
438 self.backupZenPacks()
439 self.backupZenPackContents()
440 self.log.info("Backup of ZenPacks completed.")
441
442 if self.options.noPerfData:
443 self.log.info('Skipping backup of performance data.')
444 else:
445 self.backupPerfData()
446
447
448 outfile = self.packageStagingBackups()
449
450 self.cleanupTempDir()
451
452 backupEndTime = time.time()
453 totalBackupTime = readable_time(backupEndTime - backupBeginTime)
454 self.log.info('Backup completed successfully in %s.', totalBackupTime)
455
456
457 return 0
458
459
460 if __name__ == '__main__':
461 zb = ZenBackup(sys.argv)
462 if zb.makeBackup():
463 sys.exit(-1)
464