update

2025-11-01 21:18:29 +08:00
parent 35478d43d6
commit a607719b46
119 changed files with 815 additions and 1221 deletions
--- a/kafka/Demo/MultiNodes/README.md
+++ b/kafka/Demo/MultiNodes/README.md
@@ -0,0 +1,31 @@
+# 部署 kafka 集群
+
+- 根据实际环境修改 docker-compose.yml
+    * CONF: 这里启动三个 controller 节点和两个 brocker 节点，可用于线上
+    * CLUSTER_ID: 这里要替换成完全一样的 uuid，生成命令
+        ```
+        docker run --rm harbor.boyachain.cn:20443/general/kafka:4.0 kafka-storage.sh random-uuid
+        ```
+    * LEAD_CONTROLLER: 指定 bootstrap kraft 集群的节点 **controller 地址**，bootstrap kraft 集群的**第一个节点不能设置该变量**，broker 节点无需设置该变量
+
+- 创建目录
+    ```
+    grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
+    ```
+
+- 启动
+    ```
+    docker-compose up -d
+    ```
+
+- 查看集群状态
+    ```
+    docker exec kafka-controller2 kafka-metadata-quorum.sh \
+        --bootstrap-controller 127.10.11.3:9093 \
+        describe --status
+
+    docker exec kafka-broker2 kafka-metadata-quorum.sh \
+        --bootstrap-controller 127.10.11.2:9093 \
+        describe --replication
+    ```
+
--- a/kafka/Demo/MultiNodes/docker-compose.yml
+++ b/kafka/Demo/MultiNodes/docker-compose.yml
@@ -0,0 +1,123 @@
+services:
+  kafka-controller1:
+    image: harbor.boyachain.cn:20443/general/kafka:4.0
+    container_name: kafka-controller1
+    restart: no
+    environment:
+      CONF: controller
+      CLUSTER_ID: xxxx
+      _CONF_node.id: 1
+      _CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
+      _CONF_listeners: CONTROLLER://127.10.11.1:9093
+      _CONF_offsets.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.min.isr: 3 
+      _CONF_transaction.state.log.replication.factor: 3
+      _CONF_transaction.state.log.min.isr: 3
+    network_mode: host
+    volumes:
+    - type: bind
+      source: ./kafka-controller1/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka-controller1/logs
+      target: /opt/kafka/logs
+
+  kafka-controller2:
+    image: harbor.boyachain.cn:20443/general/kafka:4.0
+    container_name: kafka-controller2
+    restart: no
+    environment:
+      CONF: controller
+      CLUSTER_ID: xxxx
+      LEAD_CONTROLLER: 127.10.11.1:9093
+      _CONF_node.id: 2
+      _CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
+      _CONF_listeners: CONTROLLER://127.10.11.2:9093
+      _CONF_offsets.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.min.isr: 3 
+      _CONF_transaction.state.log.replication.factor: 3
+      _CONF_transaction.state.log.min.isr: 3
+    network_mode: host
+    volumes:
+    - type: bind
+      source: ./kafka-controller2/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka-controller2/logs
+      target: /opt/kafka/logs
+
+  kafka-controller3:
+    image: harbor.boyachain.cn:20443/general/kafka:4.0
+    container_name: kafka-controller3
+    restart: no
+    environment:
+      CONF: controller
+      CLUSTER_ID: xxxx
+      LEAD_CONTROLLER: 127.10.11.1:9093
+      _CONF_node.id: 3
+      _CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
+      _CONF_listeners: CONTROLLER://127.10.11.3:9093
+      _CONF_offsets.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.min.isr: 3 
+      _CONF_transaction.state.log.replication.factor: 3
+      _CONF_transaction.state.log.min.isr: 3
+    network_mode: host
+    volumes:
+    - type: bind
+      source: ./kafka-controller3/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka-controller3/logs
+      target: /opt/kafka/logs
+
+  kafka-broker10:
+    image: harbor.boyachain.cn:20443/general/kafka:4.0
+    container_name: kafka-broker1
+    restart: no
+    environment:
+      CONF: broker
+      CLUSTER_ID: xxxx
+      _CONF_node.id: 10
+      _CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
+      _CONF_listeners: PLAINTEXT://127.10.11.10:9092
+      _CONF_offsets.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.min.isr: 3 
+      _CONF_transaction.state.log.replication.factor: 3
+      _CONF_transaction.state.log.min.isr: 3
+    network_mode: host
+    volumes:
+    - type: bind
+      source: ./kafka-broker1/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka-broker1/logs
+      target: /opt/kafka/logs
+
+  kafka-broker11:
+    image: harbor.boyachain.cn:20443/general/kafka:4.0
+    container_name: kafka-broker11
+    restart: no
+    environment:
+      CONF: broker
+      CLUSTER_ID: xxxx
+      _CONF_node.id: 11
+      _CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
+      _CONF_listeners: PLAINTEXT://127.10.11.11:9092
+      _CONF_offsets.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.replication.factor: 3
+      _CONF_share.coordinator.state.topic.min.isr: 3 
+      _CONF_transaction.state.log.replication.factor: 3
+      _CONF_transaction.state.log.min.isr: 3
+    network_mode: host
+    volumes:
+    - type: bind
+      source: ./kafka-broker11/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka-broker11/logs
+      target: /opt/kafka/logs
+