From 908625aebb1653d30c5a4bb22a2c97104fb45998 Mon Sep 17 00:00:00 2001 From: colben Date: Thu, 29 Dec 2022 17:06:43 +0800 Subject: [PATCH] update --- filebeat-jdk.yml | 74 +++++++++++++++++++++++++++++++++++++ filebeat-mysql.yml | 87 ++++++++++++++++++++++++++++++++++++++++++++ filebeat-nginx.yml | 91 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 filebeat-jdk.yml create mode 100644 filebeat-mysql.yml create mode 100644 filebeat-nginx.yml diff --git a/filebeat-jdk.yml b/filebeat-jdk.yml new file mode 100644 index 0000000..2cb7111 --- /dev/null +++ b/filebeat-jdk.yml @@ -0,0 +1,74 @@ +name: xxxx +logging.level: warning +setup.ilm.enabled: false +setup.template.name: "filebeat-xxxx" +setup.template.pattern: "filebeat-xxxx-*" +setup.template.overwrite: true +setup.template.append_fields: +- name: java.line_num + type: long +- name: log.content + type: text +- name: nginx.client + type: ip +- name: nginx.response.body_size + type: long +- name: mysql.querytime + type: long +- name: mysql.sql + type: text + +filebeat.inputs: +- type: log + enabled: true + paths: + - /path/to/xxxx.log + multiline.pattern: "^20" + multiline.negate: true + multiline.match: after + fields: + log.app: java + fields_under_root: true + +processors: +- include_fields: + fields: + - log.app + - log.file.path + - agent.hostname + - agent.name + - message +- if: + equals: + log.app: java + then: + - dissect: + tokenizer: '%{_logtime},%{_logms} [%{java.jar}] %{log.level} [%{java.thread}] %{java.class}.%{java.function}(%{java.line_num}) : %{log.content}' + target_prefix: "" + overwrite_keys: true +- timestamp: + field: _logtime + timezone: Asia/Shanghai + layouts: + - 2006-01-02T15:04:05 +- drop_fields: + when: + has_fields: + - _logtime + fields: + - _logtime + - message + +output.elasticsearch: + enabled: true + hosts: + - http://x.x.x.x:9200 + index: "filebeat-xxxx-%{+yyyy.MM.dd}" + username: "filebeat" + password: "Filebeat_P@sswo2d" + +output.console: + enabled: false + codec.json: + pretty: true + diff --git a/filebeat-mysql.yml b/filebeat-mysql.yml new file mode 100644 index 0000000..b9a66e1 --- /dev/null +++ b/filebeat-mysql.yml @@ -0,0 +1,87 @@ +name: xxxx +logging.level: warning +setup.ilm.enabled: false +setup.template.name: "filebeat-xxxx" +setup.template.pattern: "filebeat-xxxx-*" +setup.template.overwrite: true +setup.template.append_fields: +- name: java.line_num + type: long +- name: log.content + type: text +- name: nginx.client + type: ip +- name: nginx.response.body_size + type: long +- name: mysql.querytime + type: long +- name: mysql.sql + type: text + +filebeat.inputs: +- type: log + enabled: true + paths: + - /path/to/mysql/log/error.log + include_lines: "[[Error]]" + fields: + log.app: mysql-error + fields_under_root: true +- type: log + enabled: true + paths: + - /path/to/mysql/log/slow.log + multiline.pattern: "^# Time" + multiline.negate: true + multiline.match: after + fields: + log.app: mysql-slow + fields_under_root: true + +processors: +- include_fields: + fields: + - log.app + - log.file.path + - agent.hostname + - agent.name + - message +- if: + equals: + log.app: mysql-error + then: + - dissect: + tokenizer: '%{_logtime}+08:00 %{log.content}' + target_prefix: "" + overwrite_keys: true + else: + - dissect: + tokenizer: '# Time: %{_logtime}+08:00%{}# User@Host: %{mysql.user} @ %{mysql.host} Id: %{}# Query_time: %{mysql.querytime} %{mysql.sql}' + target_prefix: "" + overwrite_keys: true +- timestamp: + field: _logtime + timezone: Asia/Shanghai + layouts: + - 2021-07-15T13:36:57.776566 +- drop_fields: + when: + has_fields: + - _logtime + fields: + - _logtime + - message + +output.elasticsearch: + enabled: true + hosts: + - http://x.x.x.x:9200 + index: "filebeat-xxxx-%{+yyyy.MM.dd}" + username: "filebeat" + password: "Filebeat_P@sswo2d" + +output.console: + enabled: false + codec.json: + pretty: true + diff --git a/filebeat-nginx.yml b/filebeat-nginx.yml new file mode 100644 index 0000000..5fed007 --- /dev/null +++ b/filebeat-nginx.yml @@ -0,0 +1,91 @@ +name: xxxx +logging.level: warning +setup.ilm.enabled: false +setup.template.name: "filebeat-xxxx" +setup.template.pattern: "filebeat-xxxx-*" +setup.template.overwrite: true +setup.template.append_fields: +- name: java.line_num + type: long +- name: log.content + type: text +- name: nginx.client + type: ip +- name: nginx.response.body_size + type: long +- name: mysql.querytime + type: long +- name: mysql.sql + type: text + +filebeat.inputs: +- type: log + enabled: true + paths: + - /path/to/nginx/log/error.log + exclude_lines: + - "signal process started$" + fields: + log.app: nginx-error + fields_under_root: true +- type: log + enabled: true + paths: + - /path/to/nginx/log/access.log + fields: + log.app: nginx-access + fields_under_root: true + #pipeline: "nginx-access" + +processors: +- include_fields: + fields: + - log.app + - log.file.path + - agent.hostname + - agent.name + - message +- if: + equals: + log.app: nginx-access + then: + - dissect: + tokenizer: '%{nginx.client} - - [%{_logtime} +0800] "%{nginx.request.method} %{nginx.request.uri} %{nginx.request.protocol}" %{nginx.response.code} %{nginx.response.body_size} "%{nginx.request.referrer}" "%{nginx.request.user_agent}"' + target_prefix: "" + overwrite_keys: true + - timestamp: + field: _logtime + timezone: Asia/Shanghai + layouts: + - 02/Jan/2006:15:04:05 + else: + - dissect: + tokenizer: '%{_logtime} [%{log.level}] %{log.content}' + target_prefix: "" + overwrite_keys: true + - timestamp: + field: _logtime + timezone: Asia/Shanghai + layouts: + - 2006/01/02 15:04:05 +- drop_fields: + when: + has_fields: + - _logtime + fields: + - _logtime + - message + +output.elasticsearch: + enabled: true + hosts: + - http://x.x.x.x:9200 + index: "filebeat-xxxx-%{+yyyy.MM.dd}" + username: "filebeat" + password: "Filebeat_P@sswo2d" + +output.console: + enabled: false + codec.json: + pretty: true +