CentOS 7 - Fluentd

Clients - install td-agent

Taken from: https://docs.fluentd.org/installation

td-agent v3:

# curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent3.sh | sh
# systemctl enable td-agent

If we want to log also system logs (/var/log/messages, /var/log/secure, ... accessible under root):

# sed -i -e 's/User=td-agent/User=root/g' -e 's/Group=td-agent/Group=root/g' /etc/systemd/system/multi-user.target.wants/td-agent.service 
# systemctl daemon-reload

Create configuration:

# cp /etc/td-agent/td-agent.conf /etc/td-agent/td-agent.conf.orig

# vim /etc/td-agent/td-agent.conf

#
<label @FLUENT_LOG>
  <match fluent.*>
    @type stdout
  </match>
</label>
#
<source>
  @type tail
  @id input_tail_messages
  <parse>
    @type syslog
  </parse>
  path /var/log/messages
  pos_file /var/log/td-agent/system.pos
  read_from_head false
  tag system.**
  enable_watch_timer true
  enable_stat_watcher false
</source>

<source>
  @type tail
  @id input_tail_secure
  <parse>
    @type syslog
  </parse>
  path /var/log/secure
  pos_file /var/log/td-agent/secure.pos
  read_from_head false
  tag secure.**
  enable_watch_timer true
  enable_stat_watcher false
</source>

<filter *.**>
  @type record_transformer
  enable_ruby
  <record>
    hostname "#{Socket.gethostname}"
  </record>
</filter>

<match system.**>
  @include elasticsearch.conf
  logstash_format true
  logstash_prefix system
    <buffer>
    @type file
    path /var/log/td-agent/buffer/system
    @include out_buffer_params.conf
    </buffer>
</match>

<match secure.**>
  @include elasticsearch.conf
  logstash_format true
  logstash_prefix secure
  <buffer>
    @type file
    path /var/log/td-agent/buffer/secure
    @include out_buffer_params.conf
  </buffer>
</match>

@include nginx.conf

# vim /etc/td-agent/nginx.conf

<source>
  @type tail
  @id input_tail_nginx_error
  <parse>
    @type regexp
    expression ^(?<mytime>.*)\s(?<type>\[\w+\])\s(?<pid>\d+#\d+):\s(?<log>[^ ].*)
  </parse>
  path /var/log/nginx/example.com-error.log
  pos_file /var/log/td-agent/nginx-error.pos
  read_from_head true
  tag nginx.error.**
  enable_watch_timer true
  enable_stat_watcher false
</source>

<source>
  @type tail
  @id input_tail_nginx_access
  <parse>
    @type regexp
     expression ^(?<remot_host>.*)\s-\s(?<remot_user>.*)\s\[(?<time>.*)\]\s"(?<method>\w{3,7})\s(?<request_uri>.*)"\s(?<status>\d{3})\s(?<bytes_sent>.*)\s"(?<http_referer>.*)"\s"(?<http_user_agent>.*)"\s(?<request_time>\d+.\d+)\s(?<upstream_response_time>.*)
    time_format %d/%b/%Y:%H:%M:%S %z
  </parse>
  path /var/log/nginx/example.com-access.log
  pos_file /var/log/td-agent/nginx-access.pos
  read_from_head true
  tag nginx.access.**
  enable_watch_timer true
  enable_stat_watcher false
</source>

<filter *.**>
  @type record_transformer
  enable_ruby
  <record>
    hostname "#{Socket.gethostname}"
  </record>
</filter>

<match nginx.error.**>
  @include elasticsearch.conf
  logstash_format true
  logstash_prefix nginx-error
      <buffer>
      @type file
      path /var/log/td-agent/buffer/nginx-error
      @include out_buffer_params.conf
    </buffer>
</match>

<match nginx.access.**>
  @include elasticsearch.conf
  logstash_format true
  logstash_prefix nginx-access
  <buffer>
    @type file
    path /var/log/td-agent/buffer/nginx-access
    @include out_buffer_params.conf
  </buffer>
</match>

# vim /etc/td-agent/out_buffer_params.conf

flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_forever
retry_max_interval 30
chunk_limit_size 2M
queue_limit_length 100
overflow_action block

# vim /etc/td-agent/elasticsearch.conf

@type elasticsearch
host 192.168.1.1
port 9200
user fluentd
password passwordhere

# systemctl start td-agent && systemctl status td-agent

Configuration

add system messages and secure logs:

nginx access log format:

Other

Fluentd regular expression tester: https://fluentular.herokuapp.com/