first commit
This commit is contained in:
48
pipelines/java.json
Normal file
48
pipelines/java.json
Normal file
@@ -0,0 +1,48 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/java -d '
|
||||
{
|
||||
"description" : "java",
|
||||
"processors" : [
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "message",
|
||||
"patterns" : [
|
||||
"^%{JAVA_TIME:java.log.time}-\\[%{DATA:java.process}\\]-%{WORD:java.log.level}\\[%{DATA:java.thead}\\]%{WORD:java.class}\\.%{WORD:java.function}\\((?:%{NUMBER:java.line_num}|\\?)\\) \\| %{CONTENT:java.log.content}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"JAVA_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]",
|
||||
"CONTENT" : "(.*\n?)*"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "java.log.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "java.log.time",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
52
pipelines/monitor-conn.json
Normal file
52
pipelines/monitor-conn.json
Normal file
@@ -0,0 +1,52 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-conn -d '
|
||||
{
|
||||
"description": "monitor-conn",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{HOSTNAME:monitor.conn.server} %{NUMBER:monitor.conn.port} %{NUMBER:monitor.conn.count}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.conn.count",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
70
pipelines/monitor-cpu.json
Normal file
70
pipelines/monitor-cpu.json
Normal file
@@ -0,0 +1,70 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-cpu -d '
|
||||
{
|
||||
"description": "monitor-cpu",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{NUMBER:monitor.cpu.user} %{NUMBER:monitor.cpu.system} %{NUMBER:monitor.cpu.wait} %{NUMBER:monitor.cpu.idle}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.cpu.user",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.cpu.system",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.cpu.wait",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.cpu.idle",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
52
pipelines/monitor-disk.json
Normal file
52
pipelines/monitor-disk.json
Normal file
@@ -0,0 +1,52 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-disk -d '
|
||||
{
|
||||
"description": "monitor-disk",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{DATA:monitor.disk.partition} %{NUMBER:monitor.disk.used}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.disk.used",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
76
pipelines/monitor-io.json
Normal file
76
pipelines/monitor-io.json
Normal file
@@ -0,0 +1,76 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-io -d '
|
||||
{
|
||||
"description": "monitor-io",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{DATA:monitor.io.dev} %{NUMBER:monitor.io.tps} %{NUMBER:monitor.io.rd} %{NUMBER:monitor.io.wr} %{NUMBER:monitor.io.wait} %{NUMBER:monitor.io.util}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.io.tps",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.io.rd",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.io.wr",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.io.wait",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.io.util",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
70
pipelines/monitor-kafka.json
Normal file
70
pipelines/monitor-kafka.json
Normal file
@@ -0,0 +1,70 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-kafka?pretty -d '
|
||||
{
|
||||
"description": "monitor-kafka",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{DATA:monitor.kafka.topic} %{NUMBER:monitor.kafka.partition} %{NUMBER:monitor.kafka.current_offset} %{NUMBER:monitor.kafka.end_offset} %{NUMBER:monitor.kafka.lag} %{HOSTNAME:monitor.kafka.client_host} %{DATA:monitor.kafka.consumer_group}$"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.kafka.partition",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.kafka.current_offset",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.kafka.end_offset",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.kafka.lag",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
70
pipelines/monitor-mem.json
Normal file
70
pipelines/monitor-mem.json
Normal file
@@ -0,0 +1,70 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-mem -d '
|
||||
{
|
||||
"description": "monitor-mem",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{NUMBER:monitor.mem.used} %{NUMBER:monitor.mem.free} %{NUMBER:monitor.mem.buffers} %{NUMBER:monitor.mem.cache}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.mem.used",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.mem.free",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.mem.buffers",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.mem.cache",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
58
pipelines/monitor-net.json
Normal file
58
pipelines/monitor-net.json
Normal file
@@ -0,0 +1,58 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-net -d '
|
||||
{
|
||||
"description": "monitor-net",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{DATA:monitor.net.dev} %{NUMBER:monitor.net.rx} %{NUMBER:monitor.net.tx}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.net.rx",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.net.tx",
|
||||
"type": "float"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
52
pipelines/monitor-ping.json
Normal file
52
pipelines/monitor-ping.json
Normal file
@@ -0,0 +1,52 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-ping -d '
|
||||
{
|
||||
"description": "monitor-ping",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{HOSTNAME:monitor.ping.server} %{DATA:monitor.ping.state} %{NUMBER:monitor.ping.state_code}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.ping.state_code",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
52
pipelines/monitor-proc.json
Normal file
52
pipelines/monitor-proc.json
Normal file
@@ -0,0 +1,52 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/monitor-proc -d '
|
||||
{
|
||||
"description": "monitor-proc",
|
||||
"processors": [
|
||||
{
|
||||
"grok": {
|
||||
"field": "message",
|
||||
"patterns": [
|
||||
"^%{MONITOR_TIME:monitor.time} %{DATA:monitor.proc.type}#%{DATA:monitor.proc.proc}#%{DATA:monitor.proc.state}#%{NUMBER:monitor.proc.state_code}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"MONITOR_TIME" : "^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"YYYY-MM-dd H:m:s"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "monitor.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert": {
|
||||
"field": "monitor.proc.state_code",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "monitor.time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
38
pipelines/mysql-slow.json
Normal file
38
pipelines/mysql-slow.json
Normal file
@@ -0,0 +1,38 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/mysql-slow -d '
|
||||
{
|
||||
"description" : "mysql-slow",
|
||||
"processors" : [
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "message",
|
||||
"patterns" : [
|
||||
"^# Time: %{DATA:mysql.slowlog.exec_time}\n# User@Host: (?:%{USER:mysql.slowlog.user}|)\\[(?:%{USER:mysql.slowlog.user}|)\\] @ (?:%{HOSTNAME:mysql.slowlog.ip}|) \\[(?:%{HOSTNAME:mysql.slowlog.ip}|)\\][^#]*# Query_time: %{NUMBER:mysql.slowlog.query_time.sec} Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec} Rows_sent: %{NUMBER:mysql.slowlog.rows_sent} Rows_examined: %{NUMBER:mysql.slowlog.rows_examined}\n%{SQLS:mysql.slowlog.query}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"SQLS" : "(.*\n?)*"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "mysql.slowlog.exec_time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
81
pipelines/nginx-access-with-geoip.json
Normal file
81
pipelines/nginx-access-with-geoip.json
Normal file
@@ -0,0 +1,81 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/nginx-access -d '
|
||||
{
|
||||
"description" : "nginx-access",
|
||||
"processors" : [
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "message",
|
||||
"patterns" : [
|
||||
"^%{DATA:nginx.access.remote_ip} - \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url}\" \"%{DATA:nginx.access.args}\" \"%{DATA:nginx.access.request_body}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\" \"%{DATA:nginx.access.x_forward_for}\""
|
||||
],
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"geoip" : {
|
||||
"field" : "nginx.access.remote_ip",
|
||||
"target_field" : "nginx.access.geoip",
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"dd/MMM/YYYY:H:m:s Z"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "nginx.access.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "nginx.access.agent",
|
||||
"patterns" : [
|
||||
"%{ANDROID:nginx.access.os}",
|
||||
"%{LINUX:nginx.access.os}",
|
||||
"%{IOS:nginx.access.os}",
|
||||
"%{MACOSX:nginx.access.os}",
|
||||
"%{WINDOWS:nginx.access.os}",
|
||||
"%{DARWIN:nginx.access.os}",
|
||||
"%{SOGOU:nginx.access.os}",
|
||||
"%{BINGBOT:nginx.access.os}",
|
||||
"%{OFFICE:nginx.access.os}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"ANDROID" : "Android *[0-9]*",
|
||||
"LINUX" : "Linux (x86_64|i386|i686)",
|
||||
"IOS" : "OS [0-9]+",
|
||||
"MACOSX" : "Mac OS X [0-9]+",
|
||||
"WINDOWS" : "Windows NT [0-9.]+",
|
||||
"DARWIN" : "Darwin",
|
||||
"SOGOU" : "Sogou web spider",
|
||||
"BINGBOT" : "bingbot",
|
||||
"OFFICE" : "Microsoft Office [^ ]*"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "nginx.access.time",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
73
pipelines/nginx-access.json
Normal file
73
pipelines/nginx-access.json
Normal file
@@ -0,0 +1,73 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/nginx-access -d '
|
||||
{
|
||||
"description" : "nginx-access",
|
||||
"processors" : [
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "message",
|
||||
"patterns" : [
|
||||
"^%{DATA:nginx.access.remote_ip} - \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url}\" \"%{DATA:nginx.access.args}\" \"%{DATA:nginx.access.request_body}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\" \"%{DATA:nginx.access.x_forward_for}\""
|
||||
],
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"dd/MMM/YYYY:H:m:s Z"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "nginx.access.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "nginx.access.agent",
|
||||
"patterns" : [
|
||||
"%{ANDROID:nginx.access.os}",
|
||||
"%{LINUX:nginx.access.os}",
|
||||
"%{IOS:nginx.access.os}",
|
||||
"%{MACOSX:nginx.access.os}",
|
||||
"%{WINDOWS:nginx.access.os}",
|
||||
"%{DARWIN:nginx.access.os}",
|
||||
"%{SOGOU:nginx.access.os}",
|
||||
"%{BINGBOT:nginx.access.os}",
|
||||
"%{OFFICE:nginx.access.os}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"ANDROID" : "Android *[0-9]*",
|
||||
"LINUX" : "Linux (x86_64|i386|i686)",
|
||||
"IOS" : "OS [0-9]+",
|
||||
"MACOSX" : "Mac OS X [0-9]+",
|
||||
"WINDOWS" : "Windows NT [0-9.]+",
|
||||
"DARWIN" : "Darwin",
|
||||
"SOGOU" : "Sogou web spider",
|
||||
"BINGBOT" : "bingbot",
|
||||
"OFFICE" : "Microsoft Office [^ ]*"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "nginx.access.time",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
51
pipelines/secure-login.json
Normal file
51
pipelines/secure-login.json
Normal file
@@ -0,0 +1,51 @@
|
||||
curl -H "Content-Type: application/json" -X PUT http://127.0.0.1:9200/_ingest/pipeline/secure-login -d '
|
||||
{
|
||||
"description" : "secure-login",
|
||||
"processors" : [
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "message",
|
||||
"patterns" : [
|
||||
"^%{LOGIN_TIME:login.time} [^ ]+ %{WORD:login.method}.*: %{RESULT:login.result} %{WORD:login.user} from %{HOSTNAME:login.rhost}",
|
||||
"^%{LOGIN_TIME:login.time} [^ ]+ %{WORD:login.method}.*: %{RESULT:login.result} for %{WORD:login.user} from %{HOSTNAME:login.rhost}",
|
||||
"^%{LOGIN_TIME:login.time} [^ ]+ %{WORD:login.method}.*: %{RESULT:login.result}; .*ruser=(?:%{WORD:login.ruser}|) rhost=(?:%{HOSTNAME:login.rhost}|) user=%{WORD:login.user}"
|
||||
],
|
||||
"pattern_definitions" : {
|
||||
"LOGIN_TIME" : "[^ ]+ ?[^ ]+ [^ ]+",
|
||||
"RESULT" : "Invalid user|Accepted publickey|Accepted password|authentication failure"
|
||||
},
|
||||
"ignore_missing" : true,
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "@timestamp",
|
||||
"target_field" : "read_timestamp"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"formats" : [
|
||||
"MMM d HH:mm:ss",
|
||||
"MMM dd HH:mm:ss"
|
||||
],
|
||||
"timezone" : "Asia/Shanghai",
|
||||
"field" : "login.time",
|
||||
"target_field" : "@timestamp",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "login.time",
|
||||
"ignore_failure" : true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
Reference in New Issue
Block a user