Package Products :: Package ZenEvents :: Module SyslogProcessing |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenEvents.SyslogProcessing

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2007, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 as published by
      8  # the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13
     14  __doc__ = """SyslogProcessing
     15  Class for turning syslog events into Zenoss Events
     16  """
     17
     18  import re
     19  import logging
     20  slog = logging.getLogger("zen.Syslog")
     21  import socket
     22
     23  import Globals
     24  from Products.ZenEvents.syslog_h import *
     25  from Products.ZenUtils.IpUtil import isip
     26
     27
     28  # Regular expressions that parse syslog tags from different sources
     29  # A tuple can also be specified, in which case the second item in the
     30  # tuple is a boolean which tells whether or not to keep the entry (default)
     31  # or to discard the entry and not create an event.
     32  parsers = (
     33  # generic mark
     34  r"^(?P<summary>-- (?P<eventClassKey>MARK) --)",
     35
     36  # ntsyslog windows msg
     37  r"^(?P<component>.+)\[(?P<ntseverity>\D+)\] (?P<ntevid>\d+) (?P<summary>.*)",
     38
     39  # cisco msg with card indicator
     40  r"%CARD-\S+:(SLOT\d+) %(?P<eventClassKey>\S+): (?P<summary>.*)",
     41
     42  # cisco standard msg
     43  r"%(?P<eventClassKey>(?P<component>\S+)-\d-\S+): (?P<summary>.*)",
     44
     45  # Cisco ACS
     46  r"^(?P<ipAddress>\S+)\s+(?P<summary>(?P<eventClassKey>CisACS_\d\d_\S+)\s+(?P<eventKey>\S+)\s.*)",
     47
     48  # netscreen device msg
     49  r"device_id=\S+\s+\[\S+\](?P<eventClassKey>\S+\d+):\s+(?P<summary>.*)\s+\((?P<originalTime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\)",
     50
     51  # NetApp
     52  # [deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])
     53  r"^\[[^:]+: (?P<component>[^:]+)[^\]]+\]: (?P<summary>.*)",
     54
     55  # unix syslog with pid
     56  r"(?P<component>\S+)\[(?P<pid>\d+)\]:\s*(?P<summary>.*)",
     57
     58  # unix syslog without pid
     59  r"(?P<component>\S+): (?P<summary>.*)",
     60
     61  # adtran devices
     62  r"^(?P<deviceModel>[^\[]+)\[(?P<deviceManufacturer>ADTRAN)\]:(?P<component>[^\|]+\|\d+\|\d+)\|(?P<summary>.*)",
     63
     64  r"^date=.+ (?P<summary>devname=.+ log_id=(?P<eventClassKey>\d+) type=(?P<component>\S+).+)",
     65
     66  # proprietary message passing system
     67  r"^(?P<component>\S+)(\.|\s)[A-Z]{3} \d \S+ \d\d:\d\d:\d\d-\d\d:\d\d:\d\d \d{5} \d{2} \d{5} \S+ \d{4} \d{3,5} (- )*(?P<summary>.*) \d{4} \d{4}",
     68
     69  # Cisco port state logging info
     70  r"^Process (?P<process_id>\d+), Nbr (?P<device>\d+\.\d+\.\d+\.\d+) on (?P<interface>\w+/\d+) from (?P<start_state>\w+) to (?P<end_state>\w+), (?P<summary>.+)",
     71
     72  # Cisco VPN Concentrator
     73  # 54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept.
     74  r"^\d+ \d+\/\d+\/\d+ \d+:\d+:\d+\.\d+ SEV=\d+ (?P<eventClassKey>\S+) RPT=\d+ (?P<summary>.*)",
     75
     76  # Dell Storage Array
     77  # 2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.
     78  r'^\d+:\d+:(?P<component>[^:]+):\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)',
     79
     80  # 1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.
     81  r'^\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)',
     82  )
     83
     84  # compile regex parsers on load
     85  compiledParsers = []
     86  for regex in parsers:
     87      keepEntry = True
     88      if isinstance(regex, tuple):
     89          regex, keepEntry = regex
     90      try:
     91          compiled = re.compile(regex)
     92          compiledParsers.append((compiled, keepEntry))
     93      except:
     94          pass
     95
     96
     97 -class SyslogProcessor(object):
     98      """
     99      Class to process syslog messages and convert them into events viewable
    100      in the Zenoss event console.
    101      """
    102
    103 -    def __init__(self,sendEvent,minpriority,parsehost,monitor,defaultPriority):
    104          """
    105          Initializer
    106
    107          @param sendEvent: message from a remote host
    108          @type sendEvent: string
    109          @param minpriority: ignore anything under this priority
    110          @type minpriority: integer
    111          @param parsehost: hostname where this parser is running
    112          @type parsehost: string
    113          @param monitor: name of the distributed collector monitor
    114          @type monitor: string
    115          @param defaultPriority: priority to use if it can't be understood from the received packet
    116          @type defaultPriority: integer
    117          """
    118          self.minpriority = minpriority
    119          self.parsehost = parsehost
    120          self.sendEvent = sendEvent
    121          self.monitor = monitor
    122          self.defaultPriority = defaultPriority
    123
    124
    125 -    def process(self, msg, ipaddr, host, rtime):
    126          """
    127          Process an event from syslog and convert to a Zenoss event
    128
    129          @param msg: message from a remote host
    130          @type msg: string
    131          @param ipaddr: IP address of the remote host
    132          @type ipaddr: string
    133          @param host: remote host's name
    134          @type host: string
    135          @param rtime: time as reported by the remote host
    136          @type rtime: string
    137          """
    138          evt = dict(device=host,
    139                     ipAddress=ipaddr,
    140                     firstTime=rtime,
    141                     lastTime=rtime,
    142                     eventGroup='syslog')
    143          slog.debug("host=%s, ip=%s", host, ipaddr)
    144          slog.debug(msg)
    145
    146          evt, msg = self.parsePRI(evt, msg)
    147          if evt['priority'] > self.minpriority: return
    148
    149          evt, msg = self.parseHEADER(evt, msg)
    150          evt = self.parseTag(evt, msg)
    151          if evt:
    152              #rest of msg now in summary of event
    153              evt = self.buildEventClassKey(evt)
    154              evt['monitor'] = self.monitor
    155              self.sendEvent(evt)
    156
    157
    158 -    def parsePRI(self, evt, msg):
    159          """
    160          Parse RFC-3164 PRI part of syslog message to get facility and priority.
    161
    162          @param evt: dictionary of event properties
    163          @type evt: dictionary
    164          @param msg: message from host
    165          @type msg: string
    166          @return: tuple of dictionary of event properties and the message
    167          @type: (dictionary, string)
    168          """
    169          pri = self.defaultPriority
    170          fac = None
    171          if msg[:1] == '<':
    172              pos = msg.find('>')
    173              fac, pri = LOG_UNPACK(int(msg[1:pos]))
    174              msg = msg[pos+1:]
    175          elif msg and msg[0] < ' ':
    176              fac, pri = LOG_KERN, ord(msg[0])
    177              msg = msg[1:]
    178          evt['facility'] = fac_names.get(fac,"unknown")
    179          evt['priority'] = pri
    180          evt['severity'] = self.defaultSeverityMap(pri)
    181          slog.debug("fac=%s pri=%s", fac, pri)
    182          slog.debug("facility=%s severity=%s", evt['facility'], evt['severity'])
    183          return evt, msg
    184
    185
    186 -    def defaultSeverityMap(self, pri):
    187          """
    188          Default mapping from syslog priority to severity.
    189
    190          @param pri: syslog priority from host
    191          @type pri: integer
    192          @return: numeric severity
    193          @type: integer
    194          """
    195          sev = 1
    196          if pri < 3: sev = 5
    197          elif pri == 3: sev = 4
    198          elif pri == 4: sev = 3
    199          elif pri == 5 or pri == 6: sev = 2
    200          return sev
    201
    202
    203      timeParse = \
    204          re.compile("^(\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[\d ]{2}) (.*)").search
    205      notHostSearch = re.compile("[\[:]").search
    206 -    def parseHEADER(self, evt, msg):
    207          """
    208          Parse RFC-3164 HEADER part of syslog message.  TIMESTAMP format is:
    209          MMM HH:MM:SS and host is next token without the characters '[' or ':'.
    210
    211          @param evt: dictionary of event properties
    212          @type evt: dictionary
    213          @param msg: message from host
    214          @type msg: string
    215          @return: tuple of dictionary of event properties and the message
    216          @type: (dictionary, string)
    217          """
    218          slog.debug(msg)
    219          m = re.sub("Kiwi_Syslog_Daemon \d+: \d+: "
    220              "\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[^:]+: ", "", msg)
    221          m = self.timeParse(msg)
    222          if m:
    223              slog.debug("parseHEADER timestamp=%s", m.group(1))
    224              evt['originalTime'] = m.group(1)
    225              msg = m.group(2).strip()
    226          msglist = msg.split()
    227          if self.parsehost and not self.notHostSearch(msglist[0]):
    228              device = msglist[0]
    229              if device.find('@') >= 0:
    230                  device = device.split('@', 1)[1]
    231              slog.debug("parseHEADER hostname=%s", evt['device'])
    232              msg = " ".join(msglist[1:])
    233              evt['device'] = device
    234              if isip(device):
    235                  evt['ipAddress'] = device
    236              else:
    237                  if 'ipAddress' in evt:
    238                      del(evt['ipAddress'])
    239          return evt, msg
    240
    241
    242 -    def parseTag(self, evt, msg):
    243          """
    244          Parse the RFC-3164 tag of the syslog message using the regex defined
    245          at the top of this module.
    246
    247          @param evt: dictionary of event properties
    248          @type evt: dictionary
    249          @param msg: message from host
    250          @type msg: string
    251          @return: dictionary of event properties
    252          @type: dictionary
    253          """
    254          slog.debug(msg)
    255          for parser, keepEntry in compiledParsers:
    256              slog.debug("tag regex: %s", parser.pattern)
    257              m = parser.search(msg)
    258              if not m:
    259                  continue
    260              elif not keepEntry:
    261                  slog.debug("Dropping syslog message due to parser rule.")
    262                  return None
    263              slog.debug("tag match: %s", m.groupdict())
    264              evt.update(m.groupdict())
    265              break
    266          else:
    267              slog.info("No matching parser: '%s'", msg)
    268              evt['summary'] = msg
    269          return evt
    270
    271
    272 -    def buildEventClassKey(self, evt):
    273          """
    274          Build the key used to find an events dictionary record. If eventClass
    275          is defined it is used. For NT events "Source_Evid" is used. For other
    276          syslog events we use the summary of the event to perform a full text
    277          or'ed search.
    278
    279          @param evt: dictionary of event properties
    280          @type evt: dictionary
    281          @return: dictionary of event properties
    282          @type: dictionary
    283          """
    284          if evt.has_key('eventClassKey') or evt.has_key( 'eventClass'):
    285              return evt
    286          elif evt.has_key( 'ntevid'):
    287              evt['eventClassKey'] = "%s_%s" % (evt['component'],evt['ntevid'])
    288          elif evt.has_key( 'component'):
    289              evt['eventClassKey'] = evt['component']
    290          if evt.has_key( 'eventClassKey'):
    291              slog.debug("eventClassKey=%s", evt['eventClassKey'])
    292              try:
    293                  evt['eventClassKey'] = evt['eventClassKey'].decode('latin-1')
    294              except:
    295                  evt['eventClassKey'] = evt['eventClassKey'].decode('utf-8')
    296          else:
    297              slog.debug("No eventClassKey assigned")
    298          return evt
    299