grafana可视化ingress-nginx日志日志分析,logstash配置

作者: root007 分类: EFK,kubernetes 发布时间: 2019-08-17 02:38

先看效果:


kubernetes日志方案:Filebeat->kafka->Logstash->Elasticsearch->kibana

1、解决logstash解析嵌套json格式数据,原始数据:

{
         "@timestamp" => 2019-08-16T15:12:57.793Z,
      "request_proto" => "HTTP/1.1",
           "duration" => "0.071",
      "http_referrer" => "-",
     "request_length" => "1577",
               "path" => "/app/v2/index/interns",
             "stream" => "stdout",
       "request_time" => 0,
              "vhost" => "xxx.com",
               "tags" => [
        [0] "_dateparsefailure"
    ],
              "geoip" => {
          "country_name" => "China",
         "country_code2" => "CN",
         "country_code3" => "CN",
              "location" => {
            "lon" => 116.9972,
            "lat" => 36.6683
        },
             "city_name" => "Jinan",
        "continent_code" => "AS",
           "region_code" => "37",
                    "ip" => "111.17.50.101",
              "latitude" => 36.6683,
              "timezone" => "Asia/Shanghai",
             "longitude" => 116.9972,
           "region_name" => "Shandong"
    },
             "status" => 200,
        "remote_addr" => "111.17.50.101",
             "method" => "GET",
      "x-forward-for" => "111.17.50.101, 183.222.96.206, 111.17.50.101",
         "kubernetes" => {
        "container_name" => "nginx-ingress-controller",
                  "host" => "rancher-k8s-n8",
             "docker_id" => "7e2609564ea8530acadbf7ef7222c77d0ccde31969f4497ca892d64ffcf6e418",
                "labels" => {
             "pod-template-generation" => "1",
                                 "app" => "ingress-nginx",
            "controller-revision-hash" => "6d4dd55f56"
        },
              "pod_name" => "nginx-ingress-controller-cbf5p",
        "namespace_name" => "ingress-nginx",
                "pod_id" => "eb9a8f6d-ba58-11e9-a2bd-00163e094bb7",
           "annotations" => {
              "prometheus.io/port" => "10254",
            "prometheus.io/scrape" => "true"
        }
    },
                "log" => "{\"time\": \"2019-08-16T15:12:57+00:00\",\"remote_addr\": \"111.17.50.101\",\"x-forward-for\": \"111.17.50.101, 183.x.96.206, 111.17.x.101\",\"request_id\": \"49c8f06316247c4c1bb85ded6132781e\",\"remote_user\": \"-\",\"bytes_sent\": \"1091\",\"request_time\": \"0.071\",\"status\": \"200\",\"vhost\": \"xxxx.com\",\"request_proto\": \"HTTP/1.1\",\"path\": \"/app/v2/index/interns\",\"request_query\": \"stype=new&city=%E5%85%A8%E5%9B%BD&page=4\",\"request_length\": \"1577\",\"duration\": \"0.071\",\"method\": \"GET\",\"http_referrer\": \"-\",\"http_user_agent\": \app/3.2.11\"}\n",
      "request_query" => "stype=new&city=%E5%85%A8%E5%9B%BD&page=4",
        "remote_user" => "-",
    "http_user_agent" => "xxxxx/3.2.11",
               "time" => "2019-08-16T15:12:57+00:00",
         "bytes_sent" => 1091,
           "@version" => "1",
         "request_id" => "49c8f06316247c4c1bb85ded6132781e"
}

2、需要把kubernetes解析出来:

filter
{
	mutate
	{
      add_field => { "@kubernetes" => "%{kubernetes}" } #先新建一个新的字段,并将kubernetes赋值给它
    }
	json
	{
		source => "@kubernetes"	#再进行解析
		remove_field => [ "@kubernetes","xxx" ]	#删除不必要的字段,也可以不用这语句
	}
}

3、 kubernetes字段解析出来:

{
              "geoip" => {
              "location" => {
            "lat" => 36.6683,
            "lon" => 116.9972
        },
        "continent_code" => "AS",
          "country_name" => "China",
              "timezone" => "Asia/Shanghai",
         "country_code3" => "CN",
           "region_code" => "37",
                    "ip" => "140.255.58.241",
             "longitude" => 116.9972,
             "city_name" => "Jinan",
         "country_code2" => "CN",
           "region_name" => "Shandong",
              "latitude" => 36.6683
    },
               "time" => "2019-08-16T16:50:46+00:00",
          "docker_id" => "e2c77ea171de712412294139fbcba66c978714d9cf70d91a8fec418e90c01c66",
         "kubernetes" => {
        "container_name" => "nginx-ingress-controller",
                  "host" => "rancher-k8s-n7",
             "docker_id" => "e2c77ea171de712412294139fbcba66c978714d9cf70d91a8fec418e90c01c66",
        "namespace_name" => "ingress-nginx",
                "pod_id" => "e1c483ef-ba56-11e9-a2bd-00163e094bb7",
                "labels" => {
            "controller-revision-hash" => "6d4dd55f56",
                                 "app" => "ingress-nginx",
             "pod-template-generation" => "1"
        },
           "annotations" => {
            "prometheus.io/scrape" => "true",
              "prometheus.io/port" => "10254"
        },
              "pod_name" => "nginx-ingress-controller-b275w"
    },
        "annotations" => {
        "prometheus.io/scrape" => "true",
          "prometheus.io/port" => "10254"
    },
     "request_length" => "1529",
             "labels" => {
        "controller-revision-hash" => "6d4dd55f56",
                             "app" => "ingress-nginx",
         "pod-template-generation" => "1"
    },
             "status" => 200,
              "vhost" => "xxxxx",
       "request_time" => 0,
           "pod_name" => "nginx-ingress-controller-b275w",
               "host" => "rancher-k8s-n7",
      "request_proto" => "HTTP/1.1",
             "stream" => "stdout",
         "@timestamp" => 2019-08-16T16:50:46.003Z,
               "path" => "/app/mine/baseinfo",
     "namespace_name" => "ingress-nginx",
      "request_query" => "-",
               "tags" => [
        [0] "_dateparsefailure"
    ],
         "bytes_sent" => 742,
    "http_user_agent" => "sxsandroidapp/3.2.12",
        "remote_user" => "-",
      "x-forward-for" => "140.255.58.241, 58.58.81.201, 140.255.58.241",
      "http_referrer" => "-",
             "pod_id" => "e1c483ef-ba56-11e9-a2bd-00163e094bb7",
           "duration" => "0.045",
     "container_name" => "nginx-ingress-controller",
             "method" => "GET",
         "request_id" => "89239640a51a58739b4915277b5a5eae",
                "log" => "{\"time\": \"2019-08-16T16:50:46+00:00\",\"remote_addr\": \"140.255.58.241\",\"x-forward-for\": \"140.xxxxx1, 58.58xxxx1, 140.25xxx1\",\"request_id\": \"89239640a51a58739b4915277b5a5eae\",\"remote_user\": \"-\",\"bytes_sent\": \"742\",\"request_time\": \"0.045\",\"status\": \"200\",\"vhost\": \api.xxxxxx.com\",\"request_proto\": \"HTTP/1.1\",\"path\": \"/app/mine/baseinfo\",\"request_query\": \"-\",\"request_length\": \"1529\",\"duration\": \"0.045\",\"method\": \"GET\",\"http_referrer\": \"-\",\"http_user_agent\": \"dapp/3.2.12\"}\n",
        "remote_addr" => "140.255.58.241",
           "@version" => "1"
}

3、类型转换(整型)和GeoIP 地址查询归类

mutate {
      convert => ["status","integer"]
      convert => ["bytes_sent","integer"]
      convert => ["upstreatime","float"]
      convert => ["request_time","integer"]
     }
geoip {
    source => "remote_addr"   客服端ip字段在线使用geoip,不能联网的得使用离线ip数据库
    }
}

如果在针对ningress-nginx使用if判断配置

   if [kubernetes][labels] == "ningress-nginx" {
mutate {
      convert => ["status","integer"]
      convert => ["bytes_sent","integer"]
      convert => ["upstreatime","float"]
      convert => ["request_time","integer"]
     }
geoip {
    source => "remote_addr"
    }
  }
}

ingress-nginx json格式处理:

测试日志

output {
   stdout {
     codec => rubydebug
   }
 }

完整logstash配置文件

input{
  kafka{
    bootstrap_servers => "x.xx.x.xx:9091,xx.xx.xx.xx:xx,xx.xx.xx.xx:9093"
    topics => "ali-k8s-logs"
    consumer_threads => 20
    decorate_events => true
    codec => json
    auto_offset_reset => "latest"
}

}

filter {
    date {
      match => ["time", "yyyy-MM-dd HH:mm:ss,SSS", "UNIX"]
      target => "@timestamp"
      locale => "cn"
    }

mutate {
   add_field => { "@kubernetes" => "%{kubernetes}" }
    }
json {  
        source => "@kubernetes"
        remove_field => ["@kubernetes"]
}
mutate {
      convert => ["status","integer"]
      convert => ["bytes_sent","integer"]
      convert => ["upstreatime","float"]
      convert => ["request_time","integer"]
     }
geoip {
    source => "remote_addr"
    }
}

output {

  elasticsearch {
    hosts => ["xx.xxx.xx.xx:19200"]
    index => "logstash-ali-k8s-%{pod_name}-%{+YYYY.MM.dd}"
  }
}

热力图Geohash values报错处理

发表回复

您的电子邮箱地址不会被公开。 必填项已用 * 标注