71 lines
1.8 KiB
JSON
71 lines
1.8 KiB
JSON
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-kafka?pretty -d '
|
|
{
|
|
"description": "monitor-kafka",
|
|
"processors": [
|
|
{
|
|
"grok": {
|
|
"field": "message",
|
|
"patterns": [
|
|
"^%{MONITOR_TIME:monitor.time} %{DATA:monitor.kafka.topic} %{NUMBER:monitor.kafka.partition} %{NUMBER:monitor.kafka.current_offset} %{NUMBER:monitor.kafka.end_offset} %{NUMBER:monitor.kafka.lag} %{HOSTNAME:monitor.kafka.client_host} %{DATA:monitor.kafka.consumer_group}$"
|
|
],
|
|
"pattern_definitions" : {
|
|
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
|
},
|
|
"ignore_missing" : true,
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"rename" : {
|
|
"field" : "@timestamp",
|
|
"target_field" : "read_timestamp"
|
|
}
|
|
},
|
|
{
|
|
"date" : {
|
|
"formats" : [
|
|
"YYYY-MM-dd H:m:s"
|
|
],
|
|
"timezone" : "Asia/Shanghai",
|
|
"field" : "monitor.time",
|
|
"target_field" : "@timestamp",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"convert": {
|
|
"field": "monitor.kafka.partition",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
{
|
|
"convert": {
|
|
"field": "monitor.kafka.current_offset",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
{
|
|
"convert": {
|
|
"field": "monitor.kafka.end_offset",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
{
|
|
"convert": {
|
|
"field": "monitor.kafka.lag",
|
|
"type": "integer"
|
|
}
|
|
},
|
|
{
|
|
"remove": {
|
|
"field": "monitor.time"
|
|
}
|
|
},
|
|
{
|
|
"remove": {
|
|
"field": "message"
|
|
}
|
|
}
|
|
]
|
|
}'
|