IMPORTANT: No additional bug fixes or documentation updates will be released for this version. For the latest information, see the current release documentation.

« Working with Filebeat Modules Deploying and Scaling Logstash »

› ›

Configuration Examples

edit

IMPORTANT: This documentation is no longer updated. Refer to Elastic's version policy and the latest documentation.

Configuration Examples

edit

The examples in this section show you how to configure Filebeat and build Logstash pipelines that parse:

Of course, the paths that you specify in the Filebeat config depend on the location of the logs you are harvesting. The examples show common default locations.

Apache 2 Logs

edit

Here are some configuration examples for shipping and parsing Apache 2 access and error logs.

Apache 2 Access Logs

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/apache2/access.log*
    - /var/log/apache2/other_vhosts_access.log*
  exclude_files: [".gz$"]
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
        "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
      remove_field => "message"
   }
   mutate {
      add_field => { "read_timestamp" => "%{@timestamp}" }
   }
   date {
      match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
      remove_field => "[apache2][access][time]"
   }
   useragent {
      source => "[apache2][access][agent]"
      target => "[apache2][access][user_agent]"
      remove_field => "[apache2][access][agent]"
   }
   geoip {
      source => "[apache2][access][remote_ip]"
      target => "[apache2][access][geoip]"
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

Apache 2 Error Logs

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/apache2/error.log*
  exclude_files: [".gz$"]
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
        "\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
      pattern_definitions => {
        "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
      }
      remove_field => "message"
   }
   mutate {
      rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
   }
   date {
      match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
      remove_field => "[apache2][error][timestamp]"
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

MySQL Logs

edit

Here are some configuration examples for shipping and parsing MySQL error and slowlog logs.

MySQL Error Logs

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/mysql/error.log*
    - /var/log/mysqld.log*
  exclude_files: [".gz$"]
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["%{LOCALDATETIME:[mysql][error][timestamp]} (\[%{DATA:[mysql][error][level]}\] )?%{GREEDYDATA:[mysql][error][message]}",
        "%{TIMESTAMP_ISO8601:[mysql][error][timestamp]} %{NUMBER:[mysql][error][thread_id]} \[%{DATA:[mysql][error][level]}\] %{GREEDYDATA:[mysql][error][message1]}",
        "%{GREEDYDATA:[mysql][error][message2]}"] }
      pattern_definitions => {
        "LOCALDATETIME" => "[0-9]+ %{TIME}"
      }
      remove_field => "message"
   }
   mutate {
      rename => { "[mysql][error][message1]" => "[mysql][error][message]" }
   }
   mutate {
      rename => { "[mysql][error][message2]" => "[mysql][error][message]" }
   }
   date {
      match => [ "[mysql][error][timestamp]", "ISO8601", "YYMMdd H:m:s" ]
      remove_field => "[apache2][access][time]"
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

MySQL Slowlog

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/mysql/mysql-slow.log*
    - /var/lib/mysql/hostname-slow.log
  exclude_files: [".gz$"]
  multiline:
    pattern: "^# User@Host: "
    negate: true
    match: after
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["^# User@Host: %{USER:[mysql][slowlog][user]}(\[[^\]]+\])? @ %{HOSTNAME:[mysql][slowlog][host]} \[(IP:[mysql][slowlog][ip])?\](\s*Id:\s* %{NUMBER:[mysql][slowlog][id]})?\n# Query_time: %{NUMBER:[mysql][slowlog][query_time][sec]}\s* Lock_time: %{NUMBER:[mysql][slowlog][lock_time][sec]}\s* Rows_sent: %{NUMBER:[mysql][slowlog][rows_sent]}\s* Rows_examined: %{NUMBER:[mysql][slowlog][rows_examined]}\n(SET timestamp=%{NUMBER:[mysql][slowlog][timestamp]};\n)?%{GREEDYMULTILINE:[mysql][slowlog][query]}"] }
      pattern_definitions => {
        "GREEDYMULTILINE" => "(.|\n)*"
      }
      remove_field => "message"
   }
   date {
      match => [ "[mysql][slowlog][timestamp]", "UNIX" ]
   }
   mutate {
      gsub => ["[mysql][slowlog][query]", "\n# Time: [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9](\\.[0-9]+)?$", ""]
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

Nginx Logs

edit

Here are some configuration examples for shipping and parsing Nginx access and error logs.

Nginx Access Logs

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/nginx/access.log*
  exclude_files: [".gz$"]
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
      remove_field => "message"
   }
   mutate {
      rename => { "@timestamp" => "read_timestamp" }
   }
   date {
      match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
      remove_field => "[nginx][access][time]"
   }
   useragent {
      source => "[nginx][access][agent]"
      target => "[nginx][access][user_agent]"
      remove_field => "[nginx][access][agent]"
   }
   geoip {
      source => "[nginx][access][remote_ip]"
      target => "[nginx][access][geoip]"
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

Nginx Error Logs

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/nginx/error.log*
  exclude_files: [".gz$"]
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
      remove_field => "message"
   }
   mutate {
      rename => { "@timestamp" => "read_timestamp" }
   }
   date {
      match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
      remove_field => "[nginx][error][time]"
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

System Logs

edit

Here are some configuration examples for shipping and parsing system logs.

System Authorization Logs

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/auth.log*
    - /var/log/secure*
  exclude_files: [".gz$"]
  multiline:
    pattern: "^\\s"
    match: after
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
               "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
               "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
               "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
               "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}",
               "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
               "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}"] }
      pattern_definitions => {
        "GREEDYMULTILINE"=> "(.|\n)*"
      }
      remove_field => "message"
   }
   date {
      match => [ "[system][auth][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
   }
   geoip {
      source => "[system][auth][ssh][ip]"
      target => "[system][auth][ssh][geoip]"
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

Syslog

edit

Example Filebeat config:

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/messages*
    - /var/log/syslog*
  exclude_files: [".gz$"]
  multiline:
    pattern: "^\\s"
    match: after
output.logstash:
  hosts: ["localhost:5044"]

Example Logstash pipeline config:

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
    # The IP address to listen for filebeat connections.
    host => "0.0.0.0"
  }
}
filter {
   grok {
      match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
      pattern_definitions => { "GREEDYMULTILINE" => "(.|\n)*" }
      remove_field => "message"
   }
   date {
      match => [ "[system][syslog][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
   }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

« Working with Filebeat Modules Deploying and Scaling Logstash »