Package Products :: Package ZenEvents :: Module SyslogProcessing |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenEvents.SyslogProcessing

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2007, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  __doc__ = """SyslogProcessing
     12  Class for turning syslog events into Zenoss Events
     13  """
     14
     15  import re
     16  import logging
     17  slog = logging.getLogger("zen.Syslog")
     18  import socket
     19
     20  import Globals
     21  from Products.ZenEvents.syslog_h import *
     22  from Products.ZenUtils.IpUtil import isip
     23
     24
     25  # Regular expressions that parse syslog tags from different sources
     26  # A tuple can also be specified, in which case the second item in the
     27  # tuple is a boolean which tells whether or not to keep the entry (default)
     28  # or to discard the entry and not create an event.
     29  parsers = (
     30  # generic mark
     31  r"^(?P<summary>-- (?P<eventClassKey>MARK) --)",
     32
     33  # Cisco UCS
     34  # : 2010 Oct 19 15:47:45 CDT: snmpd: SNMP Operation (GET) failed. Reason:2 reqId (257790979) errno (42) error index (1)
     35  r'^: \d{4} \w{3}\s+\d{1,2}\s+\d{1,2}:\d\d:\d\d \w{3}: (?P<eventClassKey>[^:]+): (?P<summary>.*)',
     36
     37  # ntsyslog windows msg
     38  r"^(?P<component>.+)\[(?P<ntseverity>\D+)\] (?P<ntevid>\d+) (?P<summary>.*)",
     39
     40  # cisco msg with card indicator
     41  r"%CARD-\S+:(SLOT\d+) %(?P<eventClassKey>\S+): (?P<summary>.*)",
     42
     43  # cisco standard msg
     44  r"%(?P<eventClassKey>(?P<component>\S+)-\d-\S+): *(?P<summary>.*)",
     45
     46  # Cisco ACS
     47  r"^(?P<ipAddress>\S+)\s+(?P<summary>(?P<eventClassKey>CisACS_\d\d_\S+)\s+(?P<eventKey>\S+)\s.*)",
     48
     49  # netscreen device msg
     50  r"device_id=\S+\s+\[\S+\](?P<eventClassKey>\S+\d+):\s+(?P<summary>.*)\s+\((?P<originalTime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\)",
     51
     52  # NetApp
     53  # [deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])
     54  r"^\[[^:]+: (?P<component>[^:]+)[^\]]+\]: (?P<summary>.*)",
     55
     56  # unix syslog with pid
     57  r"(?P<component>\S+)\[(?P<pid>\d+)\]:\s*(?P<summary>.*)",
     58
     59  # unix syslog without pid
     60  r"(?P<component>\S+): (?P<summary>.*)",
     61
     62  # adtran devices
     63  r"^(?P<deviceModel>[^\[]+)\[(?P<deviceManufacturer>ADTRAN)\]:(?P<component>[^\|]+\|\d+\|\d+)\|(?P<summary>.*)",
     64
     65  r"^date=.+ (?P<summary>devname=.+ log_id=(?P<eventClassKey>\d+) type=(?P<component>\S+).+)",
     66
     67  # proprietary message passing system
     68  r"^(?P<component>\S+)(\.|\s)[A-Z]{3} \d \S+ \d\d:\d\d:\d\d-\d\d:\d\d:\d\d \d{5} \d{2} \d{5} \S+ \d{4} \d{3,5} (- )*(?P<summary>.*) \d{4} \d{4}",
     69
     70  # Cisco port state logging info
     71  r"^Process (?P<process_id>\d+), Nbr (?P<device>\d+\.\d+\.\d+\.\d+) on (?P<interface>\w+/\d+) from (?P<start_state>\w+) to (?P<end_state>\w+), (?P<summary>.+)",
     72
     73  # Cisco VPN Concentrator
     74  # 54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept.
     75  r"^\d+ \d+\/\d+\/\d+ \d+:\d+:\d+\.\d+ SEV=\d+ (?P<eventClassKey>\S+) RPT=\d+ (?P<summary>.*)",
     76
     77  # Dell Storage Array
     78  # 2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.
     79  r'^\d+:\d+:(?P<component>[^:]+):\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)',
     80
     81  # 1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.
     82  r'^\d+-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d+:[^:]+:\d+:\w+:(?P<eventClassKey>[^:]+):(?P<summary>.*)',
     83  )
     84
     85  # compile regex parsers on load
     86  compiledParsers = []
     87  for regex in parsers:
     88      keepEntry = True
     89      if isinstance(regex, tuple):
     90          regex, keepEntry = regex
     91      try:
     92          compiled = re.compile(regex)
     93          compiledParsers.append((compiled, keepEntry))
     94      except:
     95          pass
     96
     97
     98 -class SyslogProcessor(object):
     99      """
    100      Class to process syslog messages and convert them into events viewable
    101      in the Zenoss event console.
    102      """
    103
    104 -    def __init__(self,sendEvent,minpriority,parsehost,monitor,defaultPriority):
    105          """
    106          Initializer
    107
    108          @param sendEvent: message from a remote host
    109          @type sendEvent: string
    110          @param minpriority: ignore anything under this priority
    111          @type minpriority: integer
    112          @param parsehost: hostname where this parser is running
    113          @type parsehost: string
    114          @param monitor: name of the distributed collector monitor
    115          @type monitor: string
    116          @param defaultPriority: priority to use if it can't be understood from the received packet
    117          @type defaultPriority: integer
    118          """
    119          self.minpriority = minpriority
    120          self.parsehost = parsehost
    121          self.sendEvent = sendEvent
    122          self.monitor = monitor
    123          self.defaultPriority = defaultPriority
    124
    125
    126 -    def process(self, msg, ipaddr, host, rtime):
    127          """
    128          Process an event from syslog and convert to a Zenoss event
    129
    130          @param msg: message from a remote host
    131          @type msg: string
    132          @param ipaddr: IP address of the remote host
    133          @type ipaddr: string
    134          @param host: remote host's name
    135          @type host: string
    136          @param rtime: time as reported by the remote host
    137          @type rtime: string
    138          """
    139          evt = dict(device=host,
    140                     ipAddress=ipaddr,
    141                     firstTime=rtime,
    142                     lastTime=rtime,
    143                     eventGroup='syslog')
    144          slog.debug("host=%s, ip=%s", host, ipaddr)
    145          slog.debug(msg)
    146
    147          evt, msg = self.parsePRI(evt, msg)
    148          if evt['priority'] > self.minpriority: return
    149
    150          evt, msg = self.parseHEADER(evt, msg)
    151          evt = self.parseTag(evt, msg)
    152          if evt:
    153              #rest of msg now in summary of event
    154              evt = self.buildEventClassKey(evt)
    155              evt['monitor'] = self.monitor
    156              self.sendEvent(evt)
    157
    158
    159 -    def parsePRI(self, evt, msg):
    160          """
    161          Parse RFC-3164 PRI part of syslog message to get facility and priority.
    162
    163          @param evt: dictionary of event properties
    164          @type evt: dictionary
    165          @param msg: message from host
    166          @type msg: string
    167          @return: tuple of dictionary of event properties and the message
    168          @type: (dictionary, string)
    169          """
    170          pri = self.defaultPriority
    171          fac = None
    172          if msg[:1] == '<':
    173              pos = msg.find('>')
    174              fac, pri = LOG_UNPACK(int(msg[1:pos]))
    175              msg = msg[pos+1:]
    176          elif msg and msg[0] < ' ':
    177              fac, pri = LOG_KERN, ord(msg[0])
    178              msg = msg[1:]
    179          evt['facility'] = fac
    180          evt['priority'] = pri
    181          evt['severity'] = self.defaultSeverityMap(pri)
    182          slog.debug("fac=%s pri=%s", fac, pri)
    183          slog.debug("facility=%s severity=%s", evt['facility'], evt['severity'])
    184          return evt, msg
    185
    186
    187 -    def defaultSeverityMap(self, pri):
    188          """
    189          Default mapping from syslog priority to severity.
    190
    191          @param pri: syslog priority from host
    192          @type pri: integer
    193          @return: numeric severity
    194          @type: integer
    195          """
    196          sev = 1
    197          if pri < 3: sev = 5
    198          elif pri == 3: sev = 4
    199          elif pri == 4: sev = 3
    200          elif pri == 5 or pri == 6: sev = 2
    201          return sev
    202
    203
    204      timeParse = \
    205          re.compile("^(\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[\d ]{2}(?:\.\d{1,3})?) (.*)").search
    206      notHostSearch = re.compile("[\[:]").search
    207 -    def parseHEADER(self, evt, msg):
    208          """
    209          Parse RFC-3164 HEADER part of syslog message.  TIMESTAMP format is:
    210          MMM HH:MM:SS and host is next token without the characters '[' or ':'.
    211
    212          @param evt: dictionary of event properties
    213          @type evt: dictionary
    214          @param msg: message from host
    215          @type msg: string
    216          @return: tuple of dictionary of event properties and the message
    217          @type: (dictionary, string)
    218          """
    219          slog.debug(msg)
    220          m = re.sub("Kiwi_Syslog_Daemon \d+: \d+: "
    221              "\S{3} [\d ]{2} [\d ]{2}:[\d ]{2}:[^:]+: ", "", msg)
    222          m = self.timeParse(msg)
    223          if m:
    224              slog.debug("parseHEADER timestamp=%s", m.group(1))
    225              evt['originalTime'] = m.group(1)
    226              msg = m.group(2).strip()
    227          msglist = msg.split()
    228          if self.parsehost and not self.notHostSearch(msglist[0]):
    229              device = msglist[0]
    230              if device.find('@') >= 0:
    231                  device = device.split('@', 1)[1]
    232              slog.debug("parseHEADER hostname=%s", evt['device'])
    233              msg = " ".join(msglist[1:])
    234              evt['device'] = device
    235              if isip(device):
    236                  evt['ipAddress'] = device
    237              else:
    238                  if 'ipAddress' in evt:
    239                      del(evt['ipAddress'])
    240          return evt, msg
    241
    242
    243 -    def parseTag(self, evt, msg):
    244          """
    245          Parse the RFC-3164 tag of the syslog message using the regex defined
    246          at the top of this module.
    247
    248          @param evt: dictionary of event properties
    249          @type evt: dictionary
    250          @param msg: message from host
    251          @type msg: string
    252          @return: dictionary of event properties
    253          @type: dictionary
    254          """
    255          slog.debug(msg)
    256          for parser, keepEntry in compiledParsers:
    257              slog.debug("tag regex: %s", parser.pattern)
    258              m = parser.search(msg)
    259              if not m:
    260                  continue
    261              elif not keepEntry:
    262                  slog.debug("Dropping syslog message due to parser rule.")
    263                  return None
    264              slog.debug("tag match: %s", m.groupdict())
    265              evt.update(m.groupdict())
    266              break
    267          else:
    268              slog.info("No matching parser: '%s'", msg)
    269              evt['summary'] = msg
    270          return evt
    271
    272
    273 -    def buildEventClassKey(self, evt):
    274          """
    275          Build the key used to find an events dictionary record. If eventClass
    276          is defined it is used. For NT events "Source_Evid" is used. For other
    277          syslog events we use the summary of the event to perform a full text
    278          or'ed search.
    279
    280          @param evt: dictionary of event properties
    281          @type evt: dictionary
    282          @return: dictionary of event properties
    283          @type: dictionary
    284          """
    285          if 'eventClassKey' in evt or 'eventClass' in evt:
    286              return evt
    287          elif 'ntevid' in evt:
    288              evt['eventClassKey'] = "%s_%s" % (evt['component'],evt['ntevid'])
    289          elif 'component' in evt:
    290              evt['eventClassKey'] = evt['component']
    291          if 'eventClassKey' in evt:
    292              slog.debug("eventClassKey=%s", evt['eventClassKey'])
    293              try:
    294                  evt['eventClassKey'] = evt['eventClassKey'].decode('latin-1')
    295              except:
    296                  evt['eventClassKey'] = evt['eventClassKey'].decode('utf-8')
    297          else:
    298              slog.debug("No eventClassKey assigned")
    299          return evt
    300