colben/docker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
colben
2025-11-01 21:18:29 +08:00
parent 35478d43d6
commit a607719b46
119 changed files with 815 additions and 1221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
kafka/Demo/MultiNodes/README.md Normal file
View File

@@ -0,0 +1,31 @@
# 部署 kafka 集群
- 根据实际环境修改 docker-compose.yml
* CONF: 这里启动三个 controller 节点和两个 brocker 节点，可用于线上
* CLUSTER_ID: 这里要替换成完全一样的 uuid，生成命令
```
docker run --rm harbor.boyachain.cn:20443/general/kafka:4.0 kafka-storage.sh random-uuid
```
* LEAD_CONTROLLER: 指定 bootstrap kraft 集群的节点 **controller 地址**，bootstrap kraft 集群的**第一个节点不能设置该变量**，broker 节点无需设置该变量
- 创建目录
```
grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
```
- 启动
```
docker-compose up -d
```
- 查看集群状态
```
docker exec kafka-controller2 kafka-metadata-quorum.sh \
--bootstrap-controller 127.10.11.3:9093 \
describe --status
docker exec kafka-broker2 kafka-metadata-quorum.sh \
--bootstrap-controller 127.10.11.2:9093 \
describe --replication
```

123
kafka/Demo/MultiNodes/docker-compose.yml Normal file
View File

@@ -0,0 +1,123 @@
services:
kafka-controller1:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka-controller1
restart: no
environment:
CONF: controller
CLUSTER_ID: xxxx
_CONF_node.id: 1
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: CONTROLLER://127.10.11.1:9093
_CONF_offsets.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.min.isr: 3
_CONF_transaction.state.log.replication.factor: 3
_CONF_transaction.state.log.min.isr: 3
network_mode: host
volumes:
- type: bind
source: ./kafka-controller1/data
target: /opt/kafka/data
- type: bind
source: ./kafka-controller1/logs
target: /opt/kafka/logs
kafka-controller2:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka-controller2
restart: no
environment:
CONF: controller
CLUSTER_ID: xxxx
LEAD_CONTROLLER: 127.10.11.1:9093
_CONF_node.id: 2
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: CONTROLLER://127.10.11.2:9093
_CONF_offsets.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.min.isr: 3
_CONF_transaction.state.log.replication.factor: 3
_CONF_transaction.state.log.min.isr: 3
network_mode: host
volumes:
- type: bind
source: ./kafka-controller2/data
target: /opt/kafka/data
- type: bind
source: ./kafka-controller2/logs
target: /opt/kafka/logs
kafka-controller3:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka-controller3
restart: no
environment:
CONF: controller
CLUSTER_ID: xxxx
LEAD_CONTROLLER: 127.10.11.1:9093
_CONF_node.id: 3
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: CONTROLLER://127.10.11.3:9093
_CONF_offsets.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.min.isr: 3
_CONF_transaction.state.log.replication.factor: 3
_CONF_transaction.state.log.min.isr: 3
network_mode: host
volumes:
- type: bind
source: ./kafka-controller3/data
target: /opt/kafka/data
- type: bind
source: ./kafka-controller3/logs
target: /opt/kafka/logs
kafka-broker10:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka-broker1
restart: no
environment:
CONF: broker
CLUSTER_ID: xxxx
_CONF_node.id: 10
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: PLAINTEXT://127.10.11.10:9092
_CONF_offsets.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.min.isr: 3
_CONF_transaction.state.log.replication.factor: 3
_CONF_transaction.state.log.min.isr: 3
network_mode: host
volumes:
- type: bind
source: ./kafka-broker1/data
target: /opt/kafka/data
- type: bind
source: ./kafka-broker1/logs
target: /opt/kafka/logs
kafka-broker11:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka-broker11
restart: no
environment:
CONF: broker
CLUSTER_ID: xxxx
_CONF_node.id: 11
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: PLAINTEXT://127.10.11.11:9092
_CONF_offsets.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.replication.factor: 3
_CONF_share.coordinator.state.topic.min.isr: 3
_CONF_transaction.state.log.replication.factor: 3
_CONF_transaction.state.log.min.isr: 3
network_mode: host
volumes:
- type: bind
source: ./kafka-broker11/data
target: /opt/kafka/data
- type: bind
source: ./kafka-broker11/logs
target: /opt/kafka/logs

8
kafka/Demo/SingleNode/README.md
View File

@@ -1,7 +1,11 @@
# 部署 kafka 单节点
- 根据实际环境修改
- docker-compose.yml
- 根据实际环境修改 docker-compose.yml
* CONF: 单节点只能用 server, 使用内置的 server 配置文件，该文件中默认配置的角色是 "controller,broker"
* CLUSTER_ID: 这里要替换成 uuid，生成命令
```
docker run --rm harbor.boyachain.cn:20443/general/kafka:4.0 kafka-storage.sh random-uuid
```
- 创建目录
```

41
kafka/Demo/SingleNode/docker-compose.yml
View File

@@ -1,38 +1,16 @@
version: "3.7"
services:
zk:
image: harbor.colben.cn/general/zookeeper:3.6
container_name: zk
restart: on-failure
environment:
MYID: 1
JVMFLAGS: "-Xmx1G -Xms1G"
_CONF_reconfigEnabled: "false"
_CONF_standaloneEnabled: "true"
_CONF_clientPort: 2181
networks:
kafka:
volumes:
- type: bind
source: ./zk/dataLog
target: /opt/zk/dataLog
- type: bind
source: ./zk/data
target: /opt/zk/data
- type: bind
source: ./zk/logs
target: /opt/zk/logs
kafka:
image: harbor.colben.cn/general/kafka:2.7
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka
restart: on-failure
environment:
KAFKA_OPTS: "-Xmx1G -Xms1G"
_CONF_zookeeper.connect: "zk:2181"
networks:
kafka:
KAFKA_HEAP_OPTS: '-Xmx1G -Xms1G'
CONF: server
CLUSTER_ID: xxxx
_CONF_node.id: 1
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093
_CONF_listeners: PLAINTEXT://127.10.11.1:9092,CONTROLLER://127.10.11.1:9093
network_mode: host
volumes:
- type: bind
source: ./kafka/data
@@ -41,6 +19,3 @@ services:
source: ./kafka/logs
target: /opt/kafka/logs
networks:
kafka:

15
kafka/Demo/ThreeNodes/README.md
View File

@@ -1,15 +0,0 @@
# 部署 kafka 集群
- 根据实际环境修改
- docker-compose.yml
- 创建目录
```
grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
```
- 启动
```
docker-compose up -d
```

138
kafka/Demo/ThreeNodes/docker-compose.yml
View File

@@ -1,138 +0,0 @@
version: "3.7"
services:
zk1:
image: harbor.colben.cn/general/zookeeper:3.6
container_name: zk1
restart: on-failure
environment:
MYID: 1
JVMFLAGS: "-Xmx1G -Xms1G"
_CONF_reconfigEnabled: "false"
_CONF_standaloneEnabled: "false"
_CONF_server.1: "zk1:2888:3888;2181"
_CONF_server.2: "zk2:2888:3888;2181"
_CONF_server.3: "zk3:2888:3888;2181"
networks:
kafka:
volumes:
- type: bind
source: ./zk1/dataLog
target: /opt/zk/dataLog
- type: bind
source: ./zk1/data
target: /opt/zk/data
- type: bind
source: ./zk1/logs
target: /opt/zk/logs
zk2:
image: harbor.colben.cn/general/zookeeper:3.6
container_name: zk2
restart: on-failure
environment:
MYID: 2
JVMFLAGS: "-Xmx1G -Xms1G"
_CONF_reconfigEnabled: "false"
_CONF_standaloneEnabled: "false"
_CONF_server.1: "zk1:2888:3888;2181"
_CONF_server.2: "zk2:2888:3888;2181"
_CONF_server.3: "zk3:2888:3888;2181"
networks:
kafka:
volumes:
- type: bind
source: ./zk2/dataLog
target: /opt/zk/dataLog
- type: bind
source: ./zk2/data
target: /opt/zk/data
- type: bind
source: ./zk2/logs
target: /opt/zk/logs
zk3:
image: harbor.colben.cn/general/zookeeper:3.6
container_name: zk3
restart: on-failure
environment:
MYID: 3
JVMFLAGS: "-Xmx1G -Xms1G"
_CONF_reconfigEnabled: "false"
_CONF_standaloneEnabled: "false"
_CONF_server.1: "zk1:2888:3888;2181"
_CONF_server.2: "zk2:2888:3888;2181"
_CONF_server.3: "zk3:2888:3888;2181"
networks:
kafka:
volumes:
- type: bind
source: ./zk3/dataLog
target: /opt/zk/dataLog
- type: bind
source: ./zk3/data
target: /opt/zk/data
- type: bind
source: ./zk3/logs
target: /opt/zk/logs
kafka1:
image: harbor.colben.cn/general/kafka:2.7
container_name: kafka1
restart: on-failure
environment:
KAFKA_OPTS: "-Xmx1G -Xms1G"
_CONF_broker.id: 1
_CONF_listeners: "PLAINTEXT://kafka1:9092"
_CONF_zookeeper.connect: "zk1:2181,zk2:2181,zk3:2181"
networks:
kafka:
volumes:
- type: bind
source: ./kafka1/data
target: /opt/kafka/data
- type: bind
source: ./kafka1/logs
target: /opt/kafka/logs
kafka2:
image: harbor.colben.cn/general/kafka:2.7
container_name: kafka2
restart: on-failure
environment:
KAFKA_OPTS: "-Xmx1G -Xms1G"
_CONF_broker.id: 2
_CONF_listeners: "PLAINTEXT://kafka2:9092"
_CONF_zookeeper.connect: "zk1:2181,zk2:2181,zk3:2181"
networks:
kafka:
volumes:
- type: bind
source: ./kafka2/data
target: /opt/kafka/data
- type: bind
source: ./kafka2/logs
target: /opt/kafka/logs
kafka3:
image: harbor.colben.cn/general/kafka:2.7
container_name: kafka3
restart: on-failure
environment:
KAFKA_OPTS: "-Xmx1G -Xms1G"
_CONF_broker.id: 3
_CONF_listeners: "PLAINTEXT://kafka3:9092"
_CONF_zookeeper.connect: "zk1:2181,zk2:2181,zk3:2181"
networks:
kafka:
volumes:
- type: bind
source: ./kafka3/data
target: /opt/kafka/data
- type: bind
source: ./kafka3/logs
target: /opt/kafka/logs
networks:
kafka:

31
kafka/Demo/ThreeServerNodes/README.md Normal file
View File

@@ -0,0 +1,31 @@
# 部署 kafka 集群
- 根据实际环境修改 docker-compose.yml
* CONF: 这里启动三个 server 节点，简单试用集群环境
* CLUSTER_ID: 这里要替换成完全一样的 uuid，生成命令
```
docker run --rm harbor.boyachain.cn:20443/general/kafka:4.0 kafka-storage.sh random-uuid
```
* LEAD_CONTROLLER: 指定 bootstrap kraft 集群的节点 **controller 地址**，bootstrap kraft 集群的**第一个节点不能设置该变量**
- 创建目录
```
grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
```
- 启动
```
docker-compose up -d
```
- 查看集群状态
```
docker exec kafka1 kafka-metadata-quorum.sh \
--bootstrap-controller 127.10.11.1:9093 \
describe --status
docker exec kafka1 kafka-metadata-quorum.sh \
--bootstrap-controller 127.10.11.1:9093 \
describe --replication
```

75
kafka/Demo/ThreeServerNodes/docker-compose.yml Normal file
View File

@@ -0,0 +1,75 @@
services:
kafka1:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka1
restart: no
environment:
CONF: server
CLUSTER_ID: xxxx
_CONF_node.id: 1
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: PLAINTEXT://127.10.11.1:9092,CONTROLLER://127.10.11.1:9093
_CONF_offsets.topic.replication.factor: 2
_CONF_share.coordinator.state.topic.replication.factor: 2
_CONF_share.coordinator.state.topic.min.isr: 2
_CONF_transaction.state.log.replication.factor: 2
_CONF_transaction.state.log.min.isr: 2
network_mode: host
volumes:
- type: bind
source: ./kafka1/data
target: /opt/kafka/data
- type: bind
source: ./kafka1/logs
target: /opt/kafka/logs
kafka2:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka2
restart: no
environment:
CONF: server
CLUSTER_ID: xxxx
LEAD_CONTROLLER: 127.10.11.1:9093
_CONF_node.id: 2
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: PLAINTEXT://127.10.11.2:9092,CONTROLLER://127.10.11.2:9093
_CONF_offsets.topic.replication.factor: 2
_CONF_share.coordinator.state.topic.replication.factor: 2
_CONF_share.coordinator.state.topic.min.isr: 2
_CONF_transaction.state.log.replication.factor: 2
_CONF_transaction.state.log.min.isr: 2
network_mode: host
volumes:
- type: bind
source: ./kafka2/data
target: /opt/kafka/data
- type: bind
source: ./kafka2/logs
target: /opt/kafka/logs
kafka3:
image: harbor.boyachain.cn:20443/general/kafka:4.0
container_name: kafka3
restart: no
environment:
CONF: server
CLUSTER_ID: xxxx
LEAD_CONTROLLER: 127.10.11.1:9093
_CONF_node.id: 3
_CONF_controller.quorum.bootstrap.servers: 127.10.11.1:9093,127.10.11.2:9093,127.10.11.3:9093
_CONF_listeners: PLAINTEXT://127.10.11.3:9092,CONTROLLER://127.10.11.3:9093
_CONF_offsets.topic.replication.factor: 2
_CONF_share.coordinator.state.topic.replication.factor: 2
_CONF_share.coordinator.state.topic.min.isr: 2
_CONF_transaction.state.log.replication.factor: 2
_CONF_transaction.state.log.min.isr: 2
network_mode: host
volumes:
- type: bind
source: ./kafka3/data
target: /opt/kafka/data
- type: bind
source: ./kafka3/logs
target: /opt/kafka/logs