Products.ZenEvents.SyslogProcessing

1 ############################################################################## 2 # 3 # Copyright (C) Zenoss, Inc. 2007, all rights reserved. 4 # 5 # This content is made available according to terms specified in 6 # License.zenoss under the directory where your Zenoss product is installed. 7 # 8 ############################################################################## 9 10 11 __doc__ = """SyslogProcessing 12 Class for turning syslog events into Zenoss Events 13 """ 14 15 import re 16 import logging 17 slog = logging.getLogger("zen.Syslog") 18 import socket 19 20 import Globals 21 from Products.ZenEvents.syslog_h import * 22 from Products.ZenUtils.IpUtil import isip 23 24 25 # Regular expressions that parse syslog tags from different sources 26 # A tuple can also be specified, in which case the second item in the 27 # tuple is a boolean which tells whether or not to keep the entry (default) 28 # or to discard the entry and not create an event. 29 parsers = ( 30 # generic mark 31 r"^(?P<summary>-- (?P<eventClassKey>MARK) --)", 32 33 # Cisco UCS 34 # : 2010 Oct 19 15:47:45 CDT: snmpd: SNMP Operation (GET) failed. Reason:2 reqId (257790979) errno (42) error index (1) 35 r'^: \d{4} \w{3}\s+\d{1,2}\s+\d{1,2}:\d\d:\d\d \w{3}: (?P<eventClassKey>[^:]+): (?P<summary>.*)', 36 37 # ntsyslog windows msg 38 r"^(?P<component>.+)\[(?P<ntseverity>\D+)\] (?P<ntevid>\d+) (?P<summary>.*)", 39 40 # cisco msg with card indicator 41 r"%CARD-\S+:(SLOT\d+) %(?P<eventClassKey>\S+): (?P<summary>.*)", 42 43 # cisco standard msg 44 r"%(?P<eventClassKey>(?P<component>\S+)-\d-\S+): *(?P<summary>.*)", 45 46 # Cisco ACS 47 r"^(?P<ipAddress>\S+)\s+(?P<summary>(?P<eventClassKey>CisACS_\d\d_\S+)\s+(?P<eventKey>\S+)\s.*)", 48 49 # netscreen device msg 50 r"device_id=\S+\s+\[\S+\](?P<eventClassKey>\S+\d+):\s+(?P<summary>.*)\s+\((?P<originalTime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\)", 51 52 # NetApp 53 # [deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available]) 54 r"^\[[^:]+: (?P<component>[^:]+)[^\]]+\]: (?P<summary>.*)", 55 56 # unix syslog with pid 57 r"(?P<component>\S+)\[(?P<pid>\d+)\]:\s*(?P<summary>.*)", 58 59 # unix syslog without pid 60 r"(?P<component>\S+): (?P<summary>.*)", 61 62 # adtran devices 63 r"^(?P<deviceModel>[^\[]+)\[(?P<deviceManufacturer>ADTRAN)\]:(?P<component>[^\|]+\|\d+\|\d+)\|(?P<summary>.*)", 64 65 r"^date=.+ (?P<summary>devname=.+ log_id=(?P<eventClassKey>\d+) type=(?P<component>\S+).+)", 66 67 # proprietary message passing system 68 r"^(?P<component>\S+)(\.|\s)[A-Z]{3} \d \S+ \d\d:\d\d:\d\d-\d\d:\d\d:\d\d \d{5} \d{2} \d{5} \S+ \d{4} \d{3,5} (- )*(?P<summary>.*) \d{4} \d{4}", 69 70 # Cisco port state logging info 71 r"^Process (?P<process_id>\d+), Nbr (?P<device>\d+\.\d+\.\d+\.\d+) on (?P<interface>\w+/\d+) from (?P<start_state>\w+) to (?P<end_state>\w+), (?P<summary>.+)", 72 73 # Cisco VPN Concentrator 74 # 54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept. 75 r"^\d+ \d+\/\d+\/\d+ \d+:\d+:\d+\.\d+ SEV=\d+ (?P<eventClassKey>\S+) RPT=\d+ (?P<summary>.*)", 76 77 # Dell Storage Array 78 # 2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB. 79 r'^\d+:\d+:(?P<component>[^:]+):\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)', 80 81 # 1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'. 82 r'^\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)', 83 ) 84 85 # compile regex parsers on load 86 compiledParsers = [] 87 for regex in parsers: 88 keepEntry = True 89 if isinstance(regex, tuple): 90 regex, keepEntry = regex 91 try: 92 compiled = re.compile(regex) 93 compiledParsers.append((compiled, keepEntry)) 94 except: 95 pass 96 97

98 -class SyslogProcessor(object):

99 """ 100 Class to process syslog messages and convert them into events viewable 101 in the Zenoss event console. 102 """ 103

104 - def __init__(self,sendEvent,minpriority,parsehost,monitor,defaultPriority):

105 """ 106 Initializer 107 108 @param sendEvent: message from a remote host 109 @type sendEvent: string 110 @param minpriority: ignore anything under this priority 111 @type minpriority: integer 112 @param parsehost: hostname where this parser is running 113 @type parsehost: string 114 @param monitor: name of the distributed collector monitor 115 @type monitor: string 116 @param defaultPriority: priority to use if it can't be understood from the received packet 117 @type defaultPriority: integer 118 """ 119 self.minpriority = minpriority 120 self.parsehost = parsehost 121 self.sendEvent = sendEvent 122 self.monitor = monitor 123 self.defaultPriority = defaultPriority

124 125

126 - def process(self, msg, ipaddr, host, rtime):

127 """ 128 Process an event from syslog and convert to a Zenoss event 129 130 @param msg: message from a remote host 131 @type msg: string 132 @param ipaddr: IP address of the remote host 133 @type ipaddr: string 134 @param host: remote host's name 135 @type host: string 136 @param rtime: time as reported by the remote host 137 @type rtime: string 138 """ 139 evt = dict(device=host, 140 ipAddress=ipaddr, 141 firstTime=rtime, 142 lastTime=rtime, 143 eventGroup='syslog') 144 slog.debug("host=%s, ip=%s", host, ipaddr) 145 slog.debug(msg) 146 147 evt, msg = self.parsePRI(evt, msg) 148 if evt['priority'] > self.minpriority: return 149 150 evt, msg = self.parseHEADER(evt, msg) 151 evt = self.parseTag(evt, msg) 152 if evt: 153 #rest of msg now in summary of event 154 evt = self.buildEventClassKey(evt) 155 evt['monitor'] = self.monitor 156 self.sendEvent(evt)

157 158

159 - def parsePRI(self, evt, msg):

160 """ 161 Parse RFC-3164 PRI part of syslog message to get facility and priority. 162 163 @param evt: dictionary of event properties 164 @type evt: dictionary 165 @param msg: message from host 166 @type msg: string 167 @return: tuple of dictionary of event properties and the message 168 @type: (dictionary, string) 169 """ 170 pri = self.defaultPriority 171 fac = None 172 if msg[:1] == '<': 173 pos = msg.find('>') 174 fac, pri = LOG_UNPACK(int(msg[1:pos])) 175 msg = msg[pos+1:] 176 elif msg and msg[0] < ' ': 177 fac, pri = LOG_KERN, ord(msg[0]) 178 msg = msg[1:] 179 evt['facility'] = fac 180 evt['priority'] = pri 181 evt['severity'] = self.defaultSeverityMap(pri) 182 slog.debug("fac=%s pri=%s", fac, pri) 183 slog.debug("facility=%s severity=%s", evt['facility'], evt['severity']) 184 return evt, msg

185 186

187 - def defaultSeverityMap(self, pri):

188 """ 189 Default mapping from syslog priority to severity. 190 191 @param pri: syslog priority from host 192 @type pri: integer 193 @return: numeric severity 194 @type: integer 195 """ 196 sev = 1 197 if pri < 3: sev = 5 198 elif pri == 3: sev = 4 199 elif pri == 4: sev = 3 200 elif pri == 5 or pri == 6: sev = 2 201 return sev

202 203 204 timeParse = \ 205 re.compile("^(\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[\d ]{2}(?:\.\d{1,3})?) (.*)").search 206 notHostSearch = re.compile("[\[:]").search

207 - def parseHEADER(self, evt, msg):

208 """ 209 Parse RFC-3164 HEADER part of syslog message. TIMESTAMP format is: 210 MMM HH:MM:SS and host is next token without the characters '[' or ':'. 211 212 @param evt: dictionary of event properties 213 @type evt: dictionary 214 @param msg: message from host 215 @type msg: string 216 @return: tuple of dictionary of event properties and the message 217 @type: (dictionary, string) 218 """ 219 slog.debug(msg) 220 m = re.sub("Kiwi_Syslog_Daemon \d+: \d+: " 221 "\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[^:]+: ", "", msg) 222 m = self.timeParse(msg) 223 if m: 224 slog.debug("parseHEADER timestamp=%s", m.group(1)) 225 evt['originalTime'] = m.group(1) 226 msg = m.group(2).strip() 227 msglist = msg.split() 228 if self.parsehost and not self.notHostSearch(msglist[0]): 229 device = msglist[0] 230 if device.find('@') >= 0: 231 device = device.split('@', 1)[1] 232 slog.debug("parseHEADER hostname=%s", evt['device']) 233 msg = " ".join(msglist[1:]) 234 evt['device'] = device 235 if isip(device): 236 evt['ipAddress'] = device 237 else: 238 if 'ipAddress' in evt: 239 del(evt['ipAddress']) 240 return evt, msg

241 242

243 - def parseTag(self, evt, msg):

244 """ 245 Parse the RFC-3164 tag of the syslog message using the regex defined 246 at the top of this module. 247 248 @param evt: dictionary of event properties 249 @type evt: dictionary 250 @param msg: message from host 251 @type msg: string 252 @return: dictionary of event properties 253 @type: dictionary 254 """ 255 slog.debug(msg) 256 for parser, keepEntry in compiledParsers: 257 slog.debug("tag regex: %s", parser.pattern) 258 m = parser.search(msg) 259 if not m: 260 continue 261 elif not keepEntry: 262 slog.debug("Dropping syslog message due to parser rule.") 263 return None 264 slog.debug("tag match: %s", m.groupdict()) 265 evt.update(m.groupdict()) 266 break 267 else: 268 slog.info("No matching parser: '%s'", msg) 269 evt['summary'] = msg 270 return evt

271 272

273 - def buildEventClassKey(self, evt):

274 """ 275 Build the key used to find an events dictionary record. If eventClass 276 is defined it is used. For NT events "Source_Evid" is used. For other 277 syslog events we use the summary of the event to perform a full text 278 or'ed search. 279 280 @param evt: dictionary of event properties 281 @type evt: dictionary 282 @return: dictionary of event properties 283 @type: dictionary 284 """ 285 if 'eventClassKey' in evt or 'eventClass' in evt: 286 return evt 287 elif 'ntevid' in evt: 288 evt['eventClassKey'] = "%s_%s" % (evt['component'],evt['ntevid']) 289 elif 'component' in evt: 290 evt['eventClassKey'] = evt['component'] 291 if 'eventClassKey' in evt: 292 slog.debug("eventClassKey=%s", evt['eventClassKey']) 293 try: 294 evt['eventClassKey'] = evt['eventClassKey'].decode('latin-1') 295 except: 296 evt['eventClassKey'] = evt['eventClassKey'].decode('utf-8') 297 else: 298 slog.debug("No eventClassKey assigned") 299 return evt

300

Source Code for Module Products.ZenEvents.SyslogProcessing