grafana可视化ingress-nginx日志日志分析，logstash配置
先看效果：
kubernetes日志方案:Filebeat->kafka->Logstash->Elasticsearch->kibana
1、解决logstash解析嵌套json格式数据，原始数据：
{
"@timestamp" => 2019-08-16T15:12:57.793Z,
"request_proto" => "HTTP/1.1",
"duration" => "0.071",
"http_referrer" => "-",
"request_length" => "1577",
"path" => "/app/v2/index/interns",
"stream" => "stdout",
"request_time" => 0,
"vhost" => "xxx.com",
"tags" => [
[0] "_dateparsefailure"
],
"geoip" => {
"country_name" => "China",
"country_code2" => "CN",
"country_code3" => "CN",
"location" => {
"lon" => 116.9972,
"lat" => 36.6683
},
"city_name" => "Jinan",
"continent_code" => "AS",
"region_code" => "37",
"ip" => "111.17.50.101",
"latitude" => 36.6683,
"timezone" => "Asia/Shanghai",
"longitude" => 116.9972,
"region_name" => "Shandong"
},
"status" => 200,
"remote_addr" => "111.17.50.101",
"method" => "GET",
"x-forward-for" => "111.17.50.101, 183.222.96.206, 111.17.50.101",
"kubernetes" => {
"container_name" => "nginx-ingress-controller",
"host" => "rancher-k8s-n8",
"docker_id" => "7e2609564ea8530acadbf7ef7222c77d0ccde31969f4497ca892d64ffcf6e418",
"labels" => {
"pod-template-generation" => "1",
"app" => "ingress-nginx",
"controller-revision-hash" => "6d4dd55f56"
},
"pod_name" => "nginx-ingress-controller-cbf5p",
"namespace_name" => "ingress-nginx",
"pod_id" => "eb9a8f6d-ba58-11e9-a2bd-00163e094bb7",
"annotations" => {
"prometheus.io/port" => "10254",
"prometheus.io/scrape" => "true"
}
},
"log" => "{\"time\": \"2019-08-16T15:12:57+00:00\",\"remote_addr\": \"111.17.50.101\",\"x-forward-for\": \"111.17.50.101, 183.x.96.206, 111.17.x.101\",\"request_id\": \"49c8f06316247c4c1bb85ded6132781e\",\"remote_user\": \"-\",\"bytes_sent\": \"1091\",\"request_time\": \"0.071\",\"status\": \"200\",\"vhost\": \"xxxx.com\",\"request_proto\": \"HTTP/1.1\",\"path\": \"/app/v2/index/interns\",\"request_query\": \"stype=new&city=%E5%85%A8%E5%9B%BD&page=4\",\"request_length\": \"1577\",\"duration\": \"0.071\",\"method\": \"GET\",\"http_referrer\": \"-\",\"http_user_agent\": \app/3.2.11\"}\n",
"request_query" => "stype=new&city=%E5%85%A8%E5%9B%BD&page=4",
"remote_user" => "-",
"http_user_agent" => "xxxxx/3.2.11",
"time" => "2019-08-16T15:12:57+00:00",
"bytes_sent" => 1091,
"@version" => "1",
"request_id" => "49c8f06316247c4c1bb85ded6132781e"
}2、需要把kubernetes解析出来：
filter
{
mutate
{
add_field => { "@kubernetes" => "%{kubernetes}" } #先新建一个新的字段，并将kubernetes赋值给它
}
json
{
source => "@kubernetes" #再进行解析
remove_field => [ "@kubernetes","xxx" ] #删除不必要的字段，也可以不用这语句
}
}3、 kubernetes字段解析出来：
{
"geoip" => {
"location" => {
"lat" => 36.6683,
"lon" => 116.9972
},
"continent_code" => "AS",
"country_name" => "China",
"timezone" => "Asia/Shanghai",
"country_code3" => "CN",
"region_code" => "37",
"ip" => "140.255.58.241",
"longitude" => 116.9972,
"city_name" => "Jinan",
"country_code2" => "CN",
"region_name" => "Shandong",
"latitude" => 36.6683
},
"time" => "2019-08-16T16:50:46+00:00",
"docker_id" => "e2c77ea171de712412294139fbcba66c978714d9cf70d91a8fec418e90c01c66",
"kubernetes" => {
"container_name" => "nginx-ingress-controller",
"host" => "rancher-k8s-n7",
"docker_id" => "e2c77ea171de712412294139fbcba66c978714d9cf70d91a8fec418e90c01c66",
"namespace_name" => "ingress-nginx",
"pod_id" => "e1c483ef-ba56-11e9-a2bd-00163e094bb7",
"labels" => {
"controller-revision-hash" => "6d4dd55f56",
"app" => "ingress-nginx",
"pod-template-generation" => "1"
},
"annotations" => {
"prometheus.io/scrape" => "true",
"prometheus.io/port" => "10254"
},
"pod_name" => "nginx-ingress-controller-b275w"
},
"annotations" => {
"prometheus.io/scrape" => "true",
"prometheus.io/port" => "10254"
},
"request_length" => "1529",
"labels" => {
"controller-revision-hash" => "6d4dd55f56",
"app" => "ingress-nginx",
"pod-template-generation" => "1"
},
"status" => 200,
"vhost" => "xxxxx",
"request_time" => 0,
"pod_name" => "nginx-ingress-controller-b275w",
"host" => "rancher-k8s-n7",
"request_proto" => "HTTP/1.1",
"stream" => "stdout",
"@timestamp" => 2019-08-16T16:50:46.003Z,
"path" => "/app/mine/baseinfo",
"namespace_name" => "ingress-nginx",
"request_query" => "-",
"tags" => [
[0] "_dateparsefailure"
],
"bytes_sent" => 742,
"http_user_agent" => "sxsandroidapp/3.2.12",
"remote_user" => "-",
"x-forward-for" => "140.255.58.241, 58.58.81.201, 140.255.58.241",
"http_referrer" => "-",
"pod_id" => "e1c483ef-ba56-11e9-a2bd-00163e094bb7",
"duration" => "0.045",
"container_name" => "nginx-ingress-controller",
"method" => "GET",
"request_id" => "89239640a51a58739b4915277b5a5eae",
"log" => "{\"time\": \"2019-08-16T16:50:46+00:00\",\"remote_addr\": \"140.255.58.241\",\"x-forward-for\": \"140.xxxxx1, 58.58xxxx1, 140.25xxx1\",\"request_id\": \"89239640a51a58739b4915277b5a5eae\",\"remote_user\": \"-\",\"bytes_sent\": \"742\",\"request_time\": \"0.045\",\"status\": \"200\",\"vhost\": \api.xxxxxx.com\",\"request_proto\": \"HTTP/1.1\",\"path\": \"/app/mine/baseinfo\",\"request_query\": \"-\",\"request_length\": \"1529\",\"duration\": \"0.045\",\"method\": \"GET\",\"http_referrer\": \"-\",\"http_user_agent\": \"dapp/3.2.12\"}\n",
"remote_addr" => "140.255.58.241",
"@version" => "1"
}3、类型转换(整型)和GeoIP 地址查询归类
mutate {
convert => ["status","integer"]
convert => ["bytes_sent","integer"]
convert => ["upstreatime","float"]
convert => ["request_time","integer"]
}
geoip {
source => "remote_addr" 客服端ip字段在线使用geoip，不能联网的得使用离线ip数据库
}
}
如果在针对ningress-nginx使用if判断配置
if [kubernetes][labels] == "ningress-nginx" {
mutate {
convert => ["status","integer"]
convert => ["bytes_sent","integer"]
convert => ["upstreatime","float"]
convert => ["request_time","integer"]
}
geoip {
source => "remote_addr"
}
}
}ingress-nginx json格式处理：
测试日志
output {
stdout {
codec => rubydebug
}
}完整logstash配置文件
input{
kafka{
bootstrap_servers => "x.xx.x.xx:9091,xx.xx.xx.xx:xx,xx.xx.xx.xx:9093"
topics => "ali-k8s-logs"
consumer_threads => 20
decorate_events => true
codec => json
auto_offset_reset => "latest"
}
}
filter {
date {
match => ["time", "yyyy-MM-dd HH:mm:ss,SSS", "UNIX"]
target => "@timestamp"
locale => "cn"
}
mutate {
add_field => { "@kubernetes" => "%{kubernetes}" }
}
json {
source => "@kubernetes"
remove_field => ["@kubernetes"]
}
mutate {
convert => ["status","integer"]
convert => ["bytes_sent","integer"]
convert => ["upstreatime","float"]
convert => ["request_time","integer"]
}
geoip {
source => "remote_addr"
}
}
output {
elasticsearch {
hosts => ["xx.xxx.xx.xx:19200"]
index => "logstash-ali-k8s-%{pod_name}-%{+YYYY.MM.dd}"
}
}
热力图Geohash values报错处理
