ELK+syslog+nginx访问日志收集+分词处理-创新互联

一、nginx访问日志配置:

1、日志格式配置：

log_format json '{"@timestamp":"$time_iso8601",'

          '"host":"$server_addr",'

          '"clientip":"$remote_addr",'

          '"size":$body_bytes_sent,'

          '"xff":"$http_x_forwarded_for",'

          '"upstreamhost":"$upstream_addr",'

          '"status":"$status",'

          '"responsetime":$request_time,'

          '"upstreamtime":"$upstream_response_time",'

          '"http_host":"$host",'

          '"url":"$uri"}';

2、访问日志配置：

access_log  syslog:server=xx.xx.xx.xx:5140 json;

二、logstash配置

input {

   syslog{

      port => "5140"

     }

}

filter {

   json {

     source =>"message"

     remove_field => ["message"]

     }

}

output {

     elasticsearch { hosts => ['xx.xx.xx.xx:9200','xx.xx.xx.xx:9200']

             index => 'nginx_rfd-%{+YYYY.MM.dd.HH}'

             template => "/usr/local/logstash/nginx.json"

             template_name => "nginx_*"

             template_overwrite => true

            }

}

注意index名称要包含到template_name里

三、自定义mappings

cat /usr/local/logstash/nginx.json

{

  "template": "nginx_*",

  "order":1,

  "settings": { "index.refresh_interval" : "60s" },

  "mappings": {

    "_default_": {

      "_all" : { "enabled" : false },

      "properties": {

        "@timestamp" : { "type" : "date" },

        "@version" : { "type" : "integer", "index" : "not_analyzed" },

        "url": {

          "type": "string",

          "index": "not_analyzed"

        },

        "host": {

          "type": "ip",

          "index": "not_analyzed"

        },

        "clientip": {

          "type": "ip",

          "index": "not_analyzed"

        },

        "size": {

          "type": "integer"

        },

        "xff": {

          "type": "string",

          "index": "not_analyzed"

        },

        "upstreamhost": {

          "type": "string",

          "index": "not_analyzed"

        },

        "http_host": {

          "type": "string",

          "index": "not_analyzed"

        },

        "status": {

          "type": "integer"

        },

        "responseTime": {

          "type": "string",

          "index": "not_analyzed"

        },

        "upstreamtime": {

          "type": "string",

          "index": "not_analyzed"

        }

      }

    }

  }

}

四、生成统计图形

五、参考文档

https://elasticsearch.cn/article/154

http://blog.csdn.net/choelea/article/details/53320140

http://www.cnblogs.com/hanyifeng/p/5860731.html

另外有需要云服务器可以了解下创新互联cdcxhl.cn，海内外云服务器15元起步，三天无理由+7*72小时售后在线，公司持有idc许可证，提供“云服务器、裸金属服务器、高防服务器、香港服务器、美国服务器、虚拟主机、免备案服务器”等云主机租用服务以及企业上云的综合解决方案，具有“安全稳定、简单易用、服务可用性高、性价比高”等特点与优势，专为企业上云打造定制，能够满足用户丰富、多元化的应用场景需求。