update

2022-04-18 11:21:20 +08:00
commit 45a7af638f
210 changed files with 8997 additions and 0 deletions
--- a/kafka/ADD/ccmd
+++ b/kafka/ADD/ccmd
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+##################################################
+# Mount dir                                      #
+# - /opt/kafka/data                              #
+# - /opt/kafka/logs                              #
+# ENV                                            #
+# - KAFKA_OPTS                                   #
+# - _CONF_*                                      #
+##################################################
+
+set -euo pipefail
+export LANG=en_US.UTF-8
+trap Quit EXIT
+
+PIDS=
+GOT_SIGTERM=
+
+function Print {
+    local file=/dev/null
+    [ '-f' = "$1" ] && file=$2 && shift && shift
+    date +"[%F %T] $*" | tee -a $file
+}
+
+function Quit {
+    while :; do
+        pkill -f java && Print killing java ... || break
+        sleep 1
+    done
+    Print Container stopped.
+    test -n "$GOT_SIGTERM"
+}
+
+function Usage {
+    Print 'This container should run with
+    **root user**
+    **/opt/kafka/{data,logs} mounted from host**
+    '
+}
+
+function ModifyConf {
+    Print Modify server.properties ...
+    local kv=
+    local conf='config/server.properties'
+    while read kv; do
+        [ -z "$kv" ] && break
+        Print Modify property: ${kv%%=*} ...
+        sed -i "/^${kv%%=*} *=/d" $conf
+        echo "$kv" >> $conf
+    done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')"
+}
+
+function StartProc {
+    Print Start kafka ...
+    ./bin/kafka-server-start.sh ./config/server.properties --override log.dirs=./data \
+        &>> logs/kafka.out &
+    PIDS="$PIDS $!"
+}
+
+function Main {
+    local pid=
+    cd /opt/kafka
+    Usage
+    ModifyConf
+    StartProc
+    trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM
+    while [ -z "$GOT_SIGTERM" ] && sleep 1; do
+        for pid in $PIDS; do
+            [ ! -e /proc/$pid ] && Print Unexpected error! && exit
+        done
+    done
+}
+
+# Start here
+Main
+
--- a/kafka/Demo/SingleNode/README.md
+++ b/kafka/Demo/SingleNode/README.md
@@ -0,0 +1,15 @@
+# 部署 kafka 单节点
+
+- 根据实际环境修改
+    - docker-compose.yml
+
+- 创建目录
+    ```
+    grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
+    ```
+
+- 启动
+    ```
+    docker-compose up -d
+    ```
+
--- a/kafka/Demo/SingleNode/docker-compose.yml
+++ b/kafka/Demo/SingleNode/docker-compose.yml
@@ -0,0 +1,46 @@
+version: "3.7"
+
+services:
+  zk:
+    image: harbor.colben.cn/general/zookeeper:3.6
+    container_name: zk
+    restart: on-failure
+    environment:
+      MYID: 1
+      JVMFLAGS: "-Xmx1G -Xms1G"
+      _CONF_reconfigEnabled: "false"
+      _CONF_standaloneEnabled: "true"
+      _CONF_clientPort: 2181
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./zk/dataLog
+      target: /opt/zk/dataLog
+    - type: bind
+      source: ./zk/data
+      target: /opt/zk/data
+    - type: bind
+      source: ./zk/logs
+      target: /opt/zk/logs
+
+  kafka:
+    image: harbor.colben.cn/general/kafka:2.7
+    container_name: kafka
+    restart: on-failure
+    environment:
+      KAFKA_OPTS: "-Xmx1G -Xms1G"
+      _CONF_zookeeper.connect: "zk:2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./kafka/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka/logs
+      target: /opt/kafka/logs
+
+networks:
+  kafka:
+
--- a/kafka/Demo/ThreeNodes/README.md
+++ b/kafka/Demo/ThreeNodes/README.md
@@ -0,0 +1,15 @@
+# 部署 kafka 集群
+
+- 根据实际环境修改
+    - docker-compose.yml
+
+- 创建目录
+    ```
+    grep '\<source:' docker-compose.yml | cut -d: -f2 | xargs mkdir -p
+    ```
+
+- 启动
+    ```
+    docker-compose up -d
+    ```
+
--- a/kafka/Demo/ThreeNodes/docker-compose.yml
+++ b/kafka/Demo/ThreeNodes/docker-compose.yml
@@ -0,0 +1,138 @@
+version: "3.7"
+
+services:
+  zk1:
+    image: harbor.colben.cn/general/zookeeper:3.6
+    container_name: zk1
+    restart: on-failure
+    environment:
+      MYID: 1
+      JVMFLAGS: "-Xmx1G -Xms1G"
+      _CONF_reconfigEnabled: "false"
+      _CONF_standaloneEnabled: "false"
+      _CONF_server.1: "zk1:2888:3888;2181"
+      _CONF_server.2: "zk2:2888:3888;2181"
+      _CONF_server.3: "zk3:2888:3888;2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./zk1/dataLog
+      target: /opt/zk/dataLog
+    - type: bind
+      source: ./zk1/data
+      target: /opt/zk/data
+    - type: bind
+      source: ./zk1/logs
+      target: /opt/zk/logs
+
+  zk2:
+    image: harbor.colben.cn/general/zookeeper:3.6
+    container_name: zk2
+    restart: on-failure
+    environment:
+      MYID: 2
+      JVMFLAGS: "-Xmx1G -Xms1G"
+      _CONF_reconfigEnabled: "false"
+      _CONF_standaloneEnabled: "false"
+      _CONF_server.1: "zk1:2888:3888;2181"
+      _CONF_server.2: "zk2:2888:3888;2181"
+      _CONF_server.3: "zk3:2888:3888;2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./zk2/dataLog
+      target: /opt/zk/dataLog
+    - type: bind
+      source: ./zk2/data
+      target: /opt/zk/data
+    - type: bind
+      source: ./zk2/logs
+      target: /opt/zk/logs
+
+  zk3:
+    image: harbor.colben.cn/general/zookeeper:3.6
+    container_name: zk3
+    restart: on-failure
+    environment:
+      MYID: 3
+      JVMFLAGS: "-Xmx1G -Xms1G"
+      _CONF_reconfigEnabled: "false"
+      _CONF_standaloneEnabled: "false"
+      _CONF_server.1: "zk1:2888:3888;2181"
+      _CONF_server.2: "zk2:2888:3888;2181"
+      _CONF_server.3: "zk3:2888:3888;2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./zk3/dataLog
+      target: /opt/zk/dataLog
+    - type: bind
+      source: ./zk3/data
+      target: /opt/zk/data
+    - type: bind
+      source: ./zk3/logs
+      target: /opt/zk/logs
+
+  kafka1:
+    image: harbor.colben.cn/general/kafka:2.7
+    container_name: kafka1
+    restart: on-failure
+    environment:
+      KAFKA_OPTS: "-Xmx1G -Xms1G"
+      _CONF_broker.id: 1
+      _CONF_listeners: "PLAINTEXT://kafka1:9092"
+      _CONF_zookeeper.connect: "zk1:2181,zk2:2181,zk3:2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./kafka1/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka1/logs
+      target: /opt/kafka/logs
+
+  kafka2:
+    image: harbor.colben.cn/general/kafka:2.7
+    container_name: kafka2
+    restart: on-failure
+    environment:
+      KAFKA_OPTS: "-Xmx1G -Xms1G"
+      _CONF_broker.id: 2
+      _CONF_listeners: "PLAINTEXT://kafka2:9092"
+      _CONF_zookeeper.connect: "zk1:2181,zk2:2181,zk3:2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./kafka2/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka2/logs
+      target: /opt/kafka/logs
+
+  kafka3:
+    image: harbor.colben.cn/general/kafka:2.7
+    container_name: kafka3
+    restart: on-failure
+    environment:
+      KAFKA_OPTS: "-Xmx1G -Xms1G"
+      _CONF_broker.id: 3
+      _CONF_listeners: "PLAINTEXT://kafka3:9092"
+      _CONF_zookeeper.connect: "zk1:2181,zk2:2181,zk3:2181"
+    networks:
+      kafka:
+    volumes:
+    - type: bind
+      source: ./kafka3/data
+      target: /opt/kafka/data
+    - type: bind
+      source: ./kafka3/logs
+      target: /opt/kafka/logs
+
+networks:
+  kafka:
+
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -0,0 +1,6 @@
+ARG             ARCH
+FROM            harbor.colben.cn/general/jdk$ARCH:8
+MAINTAINER      Colben colbenlee@gmail.com
+ADD             --chown=root:root /ADD/ /opt/
+CMD             ["/opt/ccmd"]
+
--- a/kafka/README.md
+++ b/kafka/README.md
@@ -0,0 +1,21 @@
+# 构建 kafka 镜像
+
+## 导入文件
+- [kafka_2.13-$VERSION.tgz](https://kafka.apache.org/downloads)
+
+## 定制
+- 修改 rootLogger
+- 创建日志目录和数据目录
+- 启动时指定数据目录和日志目录参数，覆盖配置文件
+
+## 外挂目录和文件
+- /opt/kafka/data: kafka 数据目录
+- /opt/kafka/logs: kafka 日志目录
+
+## 引入环境变量
+- KAFKA_OPTS: kafka 参数
+
+## 案例
+- [Demo/SingleNode/](Demo/SingleNode/): 部署 kafka 单节点
+- [Demo/ThreeNodes/](Demo/ThreeNodes/): 部署 kafka 集群
+
--- a/kafka/kafka.sh
+++ b/kafka/kafka.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+#=========================================
+# Author   : colben
+#=========================================
+
+set -euo pipefail
+export LANG=en_US.UTF-8
+trap Quit EXIT
+
+[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)"
+ROOT_DIR="$(cd $(dirname $0) && pwd)"
+VERSION=$1
+IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:${VERSION%.*}"
+
+if [ -t 0 ]; then
+    function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; }
+    function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; }
+    function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; }
+else
+    function Print { echo -e "$(date +'[%F %T INFO]') $*"; }
+    function Warn { echo -e "$(date +'[%F %T WARN]') $*"; }
+    function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; }
+fi
+
+function Quit {
+    local exitCode=$?
+    [ 0 -ne $exitCode ] && Error Failed to build or push image!
+    [ -z "${END:-}" ] && echo && Error Interrupted manually!
+    Print Succeeded to build and push image.
+}
+
+function YesOrNo {
+    Warn $*
+    local sw=
+    while :; do
+        read -p '(Yes/No/Quit) ' -n1 sw
+        [[ "$sw" =~ ^Y|y$ ]] && echo && return 0
+        [[ "$sw" =~ ^N|n$ ]] && echo && return 1
+        [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0
+        [ -n "$sw" ] && echo
+    done
+}
+
+function Update {
+    Warn Preparing kafka $VERSION ...
+    cd $ROOT_DIR/ADD
+    rm -rf $(ls | grep -v ccmd || true)
+    tar zxf /release/RUNTIME/kafka_2.13-$VERSION.tgz
+    mv kafka_2.13-$VERSION kafka
+    cp kafka/config/server.properties kafka/config/server.properties.sample
+    sed -i '/^log4j\.rootLogger/clog4j.rootLogger=INFO, connectAppender' kafka/config/connect-log4j.properties
+    sed -i '/^log4j\.rootLogger/clog4j.rootLogger=INFO, kafkaAppender' kafka/config/log4j.properties
+    mkdir kafka/{data,logs}
+    rm -rf kafka/site-docs
+}
+
+function Build {
+    local yn
+    cd $ROOT_DIR
+    docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \
+        && Warn Removing image $IMAGE ... \
+        && docker rmi $IMAGE
+    Warn Building image: $IMAGE ...
+    docker build --force-rm --build-arg ARCH="$ARCH" --build-arg VERSION="$VERSION" -t $IMAGE .
+    YesOrNo Push image: $IMAGE? && docker push $IMAGE
+}
+
+function Main {
+    Update
+    Build
+    END=1
+}
+
+# Start here
+Main
+