This commit is contained in:
2022-04-18 11:21:20 +08:00
commit 45a7af638f
210 changed files with 8997 additions and 0 deletions

View File

@@ -0,0 +1,27 @@
# 部署多进程 datax
- 两组 job
- 第一组每天 5 点执行一次,每次最多并行 3 个 job每个 job 超时时间十五分钟
- 第二组每天 6 点执行一次,每次最多并行 4 个 job每个 job 超时时间一小时
- 根据实际环境修改
- docker-compose.yml
- datax/job/xxxx.json
- 创建目录
```
grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
```
- 把第一组任务的 job 配置文件(xxxx.json)放在 /compose/datax1/job/ 下
- 把第二组任务的 job 配置文件(xxxx.json)放在 /compose/datax2/job/ 下
- 启动
```
docker-compose up -d
```
- 添加定时任务
```
* 5 * * * docker-compose -f /compose/docker-compose.yml up -d datax1
* 6 * * * docker-compose -f /compose/docker-compose.yml up -d datax2
```

View File

@@ -0,0 +1,43 @@
version: "3.7"
services:
datax1:
image: harbor.colben.cn/general/datax
container_name: datax1
restart: "no"
stop_grace_period: 1m
environment:
TIMEOUT: 15m
MAX_PROCS: 3
network_mode: "host"
volumes:
- type: bind
source: ./datax1/job
target: /opt/datax/job
- type: bind
source: ./datax1/log
target: /opt/datax/log
- type: bind
source: ./datax1/log_perf
target: /opt/datax/log_perf
datax2:
image: harbor.colben.cn/general/datax
container_name: datax2
restart: "no"
stop_grace_period: 1m
environment:
TIMEOUT: 1h
MAX_PROCS: 4
network_mode: "host"
volumes:
- type: bind
source: ./datax2/job
target: /opt/datax/job
- type: bind
source: ./datax2/log
target: /opt/datax/log
- type: bind
source: ./datax2/log_perf
target: /opt/datax/log_perf

View File

@@ -0,0 +1,23 @@
# 部署单进程 datax
- 每天 6 点执行一次,逐个执行 job每个 job 超时时间十分钟
- 根据实际环境修改
- docker-compose.yml
- datax/job/xxxx.json
- 创建目录
```
grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
```
- 上传需要的 job 文件到 datax/job/ 下
- 启动
```
docker-compose up -d
```
- 添加定时任务
```
* 6 * * * docker-compose -f /compose/docker-compose.yml up -d
```

View File

@@ -0,0 +1,20 @@
version: "3.7"
services:
datax:
image: harbor.colben.cn/general/datax
container_name: datax
restart: "no"
stop_grace_period: 1m
network_mode: "host"
volumes:
- type: bind
source: ./datax/job
target: /opt/datax/job
- type: bind
source: ./datax/log
target: /opt/datax/log
- type: bind
source: ./datax/log_perf
target: /opt/datax/log_perf