update

filebeat-jdk.yml

@@ -0,0 +1,74 @@
+name: xxxx
+logging.level: warning
+setup.ilm.enabled: false
+setup.template.name: "filebeat-xxxx"
+setup.template.pattern: "filebeat-xxxx-*"
+setup.template.overwrite: true
+setup.template.append_fields:
+- name: java.line_num
+  type: long
+- name: log.content
+  type: text
+- name: nginx.client
+  type: ip
+- name: nginx.response.body_size
+  type: long
+- name: mysql.querytime
+  type: long
+- name: mysql.sql
+  type: text
+
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+  - /path/to/xxxx.log
+  multiline.pattern: "^20"
+  multiline.negate: true
+  multiline.match: after
+  fields:
+    log.app: java
+  fields_under_root: true
+
+processors:
+- include_fields:
+    fields:
+    - log.app
+    - log.file.path
+    - agent.hostname
+    - agent.name
+    - message
+- if:
+    equals:
+      log.app: java
+  then:
+  - dissect:
+      tokenizer: '%{_logtime},%{_logms} [%{java.jar}] %{log.level} [%{java.thread}] %{java.class}.%{java.function}(%{java.line_num}) : %{log.content}'
+      target_prefix: ""
+      overwrite_keys: true
+- timestamp:
+    field: _logtime
+    timezone: Asia/Shanghai
+    layouts:
+    - 2006-01-02T15:04:05
+- drop_fields:
+    when:
+      has_fields:
+      - _logtime
+    fields:
+    - _logtime
+    - message
+
+output.elasticsearch:
+  enabled: true
+  hosts:
+  - http://x.x.x.x:9200
+  index: "filebeat-xxxx-%{+yyyy.MM.dd}"
+  username: "filebeat"
+  password: "Filebeat_P@sswo2d"
+
+output.console:
+  enabled: false
+  codec.json:
+    pretty: true
+

filebeat-mysql.yml

@@ -0,0 +1,87 @@
+name: xxxx
+logging.level: warning
+setup.ilm.enabled: false
+setup.template.name: "filebeat-xxxx"
+setup.template.pattern: "filebeat-xxxx-*"
+setup.template.overwrite: true
+setup.template.append_fields:
+- name: java.line_num
+  type: long
+- name: log.content
+  type: text
+- name: nginx.client
+  type: ip
+- name: nginx.response.body_size
+  type: long
+- name: mysql.querytime
+  type: long
+- name: mysql.sql
+  type: text
+
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+  - /path/to/mysql/log/error.log
+  include_lines: "[[Error]]"
+  fields:
+    log.app: mysql-error
+  fields_under_root: true
+- type: log
+  enabled: true
+  paths:
+  - /path/to/mysql/log/slow.log
+  multiline.pattern: "^# Time"
+  multiline.negate: true
+  multiline.match: after
+  fields:
+    log.app: mysql-slow
+  fields_under_root: true
+
+processors:
+- include_fields:
+    fields:
+    - log.app
+    - log.file.path
+    - agent.hostname
+    - agent.name
+    - message
+- if:
+    equals:
+      log.app: mysql-error
+  then:
+  - dissect:
+      tokenizer: '%{_logtime}+08:00 %{log.content}'
+      target_prefix: ""
+      overwrite_keys: true
+  else:
+  - dissect:
+      tokenizer: '# Time: %{_logtime}+08:00%{}# User@Host: %{mysql.user} @ %{mysql.host}  Id: %{}# Query_time: %{mysql.querytime}  %{mysql.sql}'
+      target_prefix: ""
+      overwrite_keys: true
+- timestamp:
+    field: _logtime
+    timezone: Asia/Shanghai
+    layouts:
+    - 2021-07-15T13:36:57.776566
+- drop_fields:
+    when:
+      has_fields:
+      - _logtime
+    fields:
+    - _logtime
+    - message
+
+output.elasticsearch:
+  enabled: true
+  hosts:
+  - http://x.x.x.x:9200
+  index: "filebeat-xxxx-%{+yyyy.MM.dd}"
+  username: "filebeat"
+  password: "Filebeat_P@sswo2d"
+
+output.console:
+  enabled: false
+  codec.json:
+    pretty: true
+

filebeat-nginx.yml

@@ -0,0 +1,91 @@
+name: xxxx
+logging.level: warning
+setup.ilm.enabled: false
+setup.template.name: "filebeat-xxxx"
+setup.template.pattern: "filebeat-xxxx-*"
+setup.template.overwrite: true
+setup.template.append_fields:
+- name: java.line_num
+  type: long
+- name: log.content
+  type: text
+- name: nginx.client
+  type: ip
+- name: nginx.response.body_size
+  type: long
+- name: mysql.querytime
+  type: long
+- name: mysql.sql
+  type: text
+
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+  - /path/to/nginx/log/error.log
+  exclude_lines:
+  - "signal process started$"
+  fields:
+    log.app: nginx-error
+  fields_under_root: true
+- type: log
+  enabled: true
+  paths:
+  - /path/to/nginx/log/access.log
+  fields:
+    log.app: nginx-access
+  fields_under_root: true
+  #pipeline: "nginx-access"
+
+processors:
+- include_fields:
+    fields:
+    - log.app
+    - log.file.path
+    - agent.hostname
+    - agent.name
+    - message
+- if:
+    equals:
+      log.app: nginx-access
+  then:
+  - dissect:
+      tokenizer: '%{nginx.client} - - [%{_logtime} +0800] "%{nginx.request.method} %{nginx.request.uri} %{nginx.request.protocol}" %{nginx.response.code} %{nginx.response.body_size} "%{nginx.request.referrer}" "%{nginx.request.user_agent}"'
+      target_prefix: ""
+      overwrite_keys: true
+  - timestamp:
+      field: _logtime
+      timezone: Asia/Shanghai
+      layouts:
+      - 02/Jan/2006:15:04:05
+  else:
+  - dissect:
+      tokenizer: '%{_logtime} [%{log.level}] %{log.content}'
+      target_prefix: ""
+      overwrite_keys: true
+  - timestamp:
+      field: _logtime
+      timezone: Asia/Shanghai
+      layouts:
+      - 2006/01/02 15:04:05
+- drop_fields:
+    when:
+      has_fields:
+      - _logtime
+    fields:
+    - _logtime
+    - message
+
+output.elasticsearch:
+  enabled: true
+  hosts:
+  - http://x.x.x.x:9200
+  index: "filebeat-xxxx-%{+yyyy.MM.dd}"
+  username: "filebeat"
+  password: "Filebeat_P@sswo2d"
+
+output.console:
+  enabled: false
+  codec.json:
+    pretty: true
+