Package Products :: Package ZenEvents :: Module SyslogProcessing |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenEvents.SyslogProcessing

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2007, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 or (at your
      8  # option) any later version as published by the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13
     14  __doc__ = """SyslogProcessing
     15  Class for turning syslog events into Zenoss Events
     16  """
     17
     18  import re
     19  import logging
     20  slog = logging.getLogger("zen.Syslog")
     21  import socket
     22
     23  import Globals
     24  from Products.ZenEvents.syslog_h import *
     25  from Products.ZenUtils.IpUtil import isip
     26
     27
     28  # Regular expressions that parse syslog tags from different sources
     29  # A tuple can also be specified, in which case the second item in the
     30  # tuple is a boolean which tells whether or not to keep the entry (default)
     31  # or to discard the entry and not create an event.
     32  parsers = (
     33  # generic mark
     34  r"^(?P<summary>-- (?P<eventClassKey>MARK) --)",
     35
     36  # Cisco UCS
     37  # : 2010 Oct 19 15:47:45 CDT: snmpd: SNMP Operation (GET) failed. Reason:2 reqId (257790979) errno (42) error index (1)
     38  r'^: \d{4} \w{3}\s+\d{1,2}\s+\d{1,2}:\d\d:\d\d \w{3}: (?P<eventClassKey>[^:]+): (?P<summary>.*)',
     39
     40  # ntsyslog windows msg
     41  r"^(?P<component>.+)\[(?P<ntseverity>\D+)\] (?P<ntevid>\d+) (?P<summary>.*)",
     42
     43  # cisco msg with card indicator
     44  r"%CARD-\S+:(SLOT\d+) %(?P<eventClassKey>\S+): (?P<summary>.*)",
     45
     46  # cisco standard msg
     47  r"%(?P<eventClassKey>(?P<component>\S+)-\d-\S+): *(?P<summary>.*)",
     48
     49  # Cisco ACS
     50  r"^(?P<ipAddress>\S+)\s+(?P<summary>(?P<eventClassKey>CisACS_\d\d_\S+)\s+(?P<eventKey>\S+)\s.*)",
     51
     52  # netscreen device msg
     53  r"device_id=\S+\s+\[\S+\](?P<eventClassKey>\S+\d+):\s+(?P<summary>.*)\s+\((?P<originalTime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\)",
     54
     55  # NetApp
     56  # [deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])
     57  r"^\[[^:]+: (?P<component>[^:]+)[^\]]+\]: (?P<summary>.*)",
     58
     59  # unix syslog with pid
     60  r"(?P<component>\S+)\[(?P<pid>\d+)\]:\s*(?P<summary>.*)",
     61
     62  # unix syslog without pid
     63  r"(?P<component>\S+): (?P<summary>.*)",
     64
     65  # adtran devices
     66  r"^(?P<deviceModel>[^\[]+)\[(?P<deviceManufacturer>ADTRAN)\]:(?P<component>[^\|]+\|\d+\|\d+)\|(?P<summary>.*)",
     67
     68  r"^date=.+ (?P<summary>devname=.+ log_id=(?P<eventClassKey>\d+) type=(?P<component>\S+).+)",
     69
     70  # proprietary message passing system
     71  r"^(?P<component>\S+)(\.|\s)[A-Z]{3} \d \S+ \d\d:\d\d:\d\d-\d\d:\d\d:\d\d \d{5} \d{2} \d{5} \S+ \d{4} \d{3,5} (- )*(?P<summary>.*) \d{4} \d{4}",
     72
     73  # Cisco port state logging info
     74  r"^Process (?P<process_id>\d+), Nbr (?P<device>\d+\.\d+\.\d+\.\d+) on (?P<interface>\w+/\d+) from (?P<start_state>\w+) to (?P<end_state>\w+), (?P<summary>.+)",
     75
     76  # Cisco VPN Concentrator
     77  # 54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept.
     78  r"^\d+ \d+\/\d+\/\d+ \d+:\d+:\d+\.\d+ SEV=\d+ (?P<eventClassKey>\S+) RPT=\d+ (?P<summary>.*)",
     79
     80  # Dell Storage Array
     81  # 2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.
     82  r'^\d+:\d+:(?P<component>[^:]+):\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)',
     83
     84  # 1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.
     85  r'^\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)',
     86  )
     87
     88  # compile regex parsers on load
     89  compiledParsers = []
     90  for regex in parsers:
     91      keepEntry = True
     92      if isinstance(regex, tuple):
     93          regex, keepEntry = regex
     94      try:
     95          compiled = re.compile(regex)
     96          compiledParsers.append((compiled, keepEntry))
     97      except:
     98          pass
     99
    100
    101 -class SyslogProcessor(object):
    102      """
    103      Class to process syslog messages and convert them into events viewable
    104      in the Zenoss event console.
    105      """
    106
    107 -    def __init__(self,sendEvent,minpriority,parsehost,monitor,defaultPriority):
    108          """
    109          Initializer
    110
    111          @param sendEvent: message from a remote host
    112          @type sendEvent: string
    113          @param minpriority: ignore anything under this priority
    114          @type minpriority: integer
    115          @param parsehost: hostname where this parser is running
    116          @type parsehost: string
    117          @param monitor: name of the distributed collector monitor
    118          @type monitor: string
    119          @param defaultPriority: priority to use if it can't be understood from the received packet
    120          @type defaultPriority: integer
    121          """
    122          self.minpriority = minpriority
    123          self.parsehost = parsehost
    124          self.sendEvent = sendEvent
    125          self.monitor = monitor
    126          self.defaultPriority = defaultPriority
    127
    128
    129 -    def process(self, msg, ipaddr, host, rtime):
    130          """
    131          Process an event from syslog and convert to a Zenoss event
    132
    133          @param msg: message from a remote host
    134          @type msg: string
    135          @param ipaddr: IP address of the remote host
    136          @type ipaddr: string
    137          @param host: remote host's name
    138          @type host: string
    139          @param rtime: time as reported by the remote host
    140          @type rtime: string
    141          """
    142          evt = dict(device=host,
    143                     ipAddress=ipaddr,
    144                     firstTime=rtime,
    145                     lastTime=rtime,
    146                     eventGroup='syslog')
    147          slog.debug("host=%s, ip=%s", host, ipaddr)
    148          slog.debug(msg)
    149
    150          evt, msg = self.parsePRI(evt, msg)
    151          if evt['priority'] > self.minpriority: return
    152
    153          evt, msg = self.parseHEADER(evt, msg)
    154          evt = self.parseTag(evt, msg)
    155          if evt:
    156              #rest of msg now in summary of event
    157              evt = self.buildEventClassKey(evt)
    158              evt['monitor'] = self.monitor
    159              self.sendEvent(evt)
    160
    161
    162 -    def parsePRI(self, evt, msg):
    163          """
    164          Parse RFC-3164 PRI part of syslog message to get facility and priority.
    165
    166          @param evt: dictionary of event properties
    167          @type evt: dictionary
    168          @param msg: message from host
    169          @type msg: string
    170          @return: tuple of dictionary of event properties and the message
    171          @type: (dictionary, string)
    172          """
    173          pri = self.defaultPriority
    174          fac = None
    175          if msg[:1] == '<':
    176              pos = msg.find('>')
    177              fac, pri = LOG_UNPACK(int(msg[1:pos]))
    178              msg = msg[pos+1:]
    179          elif msg and msg[0] < ' ':
    180              fac, pri = LOG_KERN, ord(msg[0])
    181              msg = msg[1:]
    182          evt['facility'] = fac
    183          evt['priority'] = pri
    184          evt['severity'] = self.defaultSeverityMap(pri)
    185          slog.debug("fac=%s pri=%s", fac, pri)
    186          slog.debug("facility=%s severity=%s", evt['facility'], evt['severity'])
    187          return evt, msg
    188
    189
    190 -    def defaultSeverityMap(self, pri):
    191          """
    192          Default mapping from syslog priority to severity.
    193
    194          @param pri: syslog priority from host
    195          @type pri: integer
    196          @return: numeric severity
    197          @type: integer
    198          """
    199          sev = 1
    200          if pri < 3: sev = 5
    201          elif pri == 3: sev = 4
    202          elif pri == 4: sev = 3
    203          elif pri == 5 or pri == 6: sev = 2
    204          return sev
    205
    206
    207      timeParse = \
    208          re.compile("^(\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[\d ]{2}) (.*)").search
    209      notHostSearch = re.compile("[\[:]").search
    210 -    def parseHEADER(self, evt, msg):
    211          """
    212          Parse RFC-3164 HEADER part of syslog message.  TIMESTAMP format is:
    213          MMM HH:MM:SS and host is next token without the characters '[' or ':'.
    214
    215          @param evt: dictionary of event properties
    216          @type evt: dictionary
    217          @param msg: message from host
    218          @type msg: string
    219          @return: tuple of dictionary of event properties and the message
    220          @type: (dictionary, string)
    221          """
    222          slog.debug(msg)
    223          m = re.sub("Kiwi_Syslog_Daemon \d+: \d+: "
    224              "\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[^:]+: ", "", msg)
    225          m = self.timeParse(msg)
    226          if m:
    227              slog.debug("parseHEADER timestamp=%s", m.group(1))
    228              evt['originalTime'] = m.group(1)
    229              msg = m.group(2).strip()
    230          msglist = msg.split()
    231          if self.parsehost and not self.notHostSearch(msglist[0]):
    232              device = msglist[0]
    233              if device.find('@') >= 0:
    234                  device = device.split('@', 1)[1]
    235              slog.debug("parseHEADER hostname=%s", evt['device'])
    236              msg = " ".join(msglist[1:])
    237              evt['device'] = device
    238              if isip(device):
    239                  evt['ipAddress'] = device
    240              else:
    241                  if 'ipAddress' in evt:
    242                      del(evt['ipAddress'])
    243          return evt, msg
    244
    245
    246 -    def parseTag(self, evt, msg):
    247          """
    248          Parse the RFC-3164 tag of the syslog message using the regex defined
    249          at the top of this module.
    250
    251          @param evt: dictionary of event properties
    252          @type evt: dictionary
    253          @param msg: message from host
    254          @type msg: string
    255          @return: dictionary of event properties
    256          @type: dictionary
    257          """
    258          slog.debug(msg)
    259          for parser, keepEntry in compiledParsers:
    260              slog.debug("tag regex: %s", parser.pattern)
    261              m = parser.search(msg)
    262              if not m:
    263                  continue
    264              elif not keepEntry:
    265                  slog.debug("Dropping syslog message due to parser rule.")
    266                  return None
    267              slog.debug("tag match: %s", m.groupdict())
    268              evt.update(m.groupdict())
    269              break
    270          else:
    271              slog.info("No matching parser: '%s'", msg)
    272              evt['summary'] = msg
    273          return evt
    274
    275
    276 -    def buildEventClassKey(self, evt):
    277          """
    278          Build the key used to find an events dictionary record. If eventClass
    279          is defined it is used. For NT events "Source_Evid" is used. For other
    280          syslog events we use the summary of the event to perform a full text
    281          or'ed search.
    282
    283          @param evt: dictionary of event properties
    284          @type evt: dictionary
    285          @return: dictionary of event properties
    286          @type: dictionary
    287          """
    288          if 'eventClassKey' in evt or 'eventClass' in evt:
    289              return evt
    290          elif 'ntevid' in evt:
    291              evt['eventClassKey'] = "%s_%s" % (evt['component'],evt['ntevid'])
    292          elif 'component' in evt:
    293              evt['eventClassKey'] = evt['component']
    294          if 'eventClassKey' in evt:
    295              slog.debug("eventClassKey=%s", evt['eventClassKey'])
    296              try:
    297                  evt['eventClassKey'] = evt['eventClassKey'].decode('latin-1')
    298              except:
    299                  evt['eventClassKey'] = evt['eventClassKey'].decode('utf-8')
    300          else:
    301              slog.debug("No eventClassKey assigned")
    302          return evt
    303