logstash

This is a small set of notes regarding logstash, which I’ve been using to import some log data into an ElasticSearch cluster for analysis.

This is the required grok pattern for my custom HTTP logs, which have a custom date format and fields for “time to first byte” and overall processing time:

input {
  stdin {}
  # cat the logs to nc localhost 3333
  tcp { 
    port => 3333
  } 
}

filter {
  grok {
    match => [
        # kudos to http://grokdebug.herokuapp.com/ for helping me test this
        "message",
        '%{IP:client} %{NOTSPACE} %{NOTSPACE} \[(?%{MONTHDAY}/%{MONTH}/%{YEAR} %{TIME})\] "%{WORD:verb} %{NOTSPACE:url} HTTP/%{NUMBER:http_version}" %{NUMBER:code} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} %{NUMBER:time_firstbyte} %{NUMBER:time_processing}'
    ]
  }
  date {
    match => [
      "timestamp",
      "d/MMM/YYYY HH:mm:ss"
    ]
  }
  geoip {
      source => "client"
      target => "geoip"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }
    mutate {
      convert => [ "[geoip][coordinates]", "float" ]
      convert => [ "[time_firstbyte]", "float" ]
      convert => [ "[time_processing]", "float" ]
    }
}

output {
  elasticsearch {
    # our cluster is behind NAT, with a single exposed machine, so the embedded
    # elasticsearch can't join it and we need to use transport mode.
    host => "cluster.local"
    protocol => "transport"
  }
  stdout { 
    # colorized debug output -- remove for long files, increases CPU load
    codec => rubydebug 
  }
}