commit 45a7af638fbcab4c6bd2c1c3b306b05524cede7e Author: colben Date: Mon Apr 18 11:21:20 2022 +0800 update diff --git a/README.md b/README.md new file mode 100644 index 0000000..e7261d0 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# 通用镜像 + +## 镜像信息 +- 镜像构建脚本: xxxx/xxxx.sh +- 容器启动脚本: xxxx/ADD/docker-entrypoint + +## 基础镜像 +- alpine +- photon +- rocky +- centos +- ubuntu + diff --git a/alpine-python/Dockerfile b/alpine-python/Dockerfile new file mode 100644 index 0000000..e06643a --- /dev/null +++ b/alpine-python/Dockerfile @@ -0,0 +1,7 @@ +ARG ARCH +FROM harbor.colben.cn/general/alpine$ARCH +MAINTAINER Colben colbenlee@gmail.com +RUN apk update \ + && apk add --no-cache linux-headers libc-dev gcc python3 python3-dev py3-pip \ + && rm -rf /var/cache/apk/* + diff --git a/alpine-python/README.md b/alpine-python/README.md new file mode 100644 index 0000000..0dc47df --- /dev/null +++ b/alpine-python/README.md @@ -0,0 +1,5 @@ +# 构建 python 镜像 + +## 定制 +- 安装 python3 及其编译工具 + diff --git a/alpine-python/alpine-python.sh b/alpine-python/alpine-python.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/alpine-python/alpine-python.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/alpine/Dockerfile b/alpine/Dockerfile new file mode 100644 index 0000000..c498fd3 --- /dev/null +++ b/alpine/Dockerfile @@ -0,0 +1,17 @@ +FROM alpine:3.14 +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /etc/ +RUN echo -e 'https://mirrors.tuna.tsinghua.edu.cn/alpine/v3.14/main\n\ +https://mirrors.tuna.tsinghua.edu.cn/alpine/v3.14/community\n\ +https://mirrors.tuna.tsinghua.edu.cn/alpine/edge/community\n\ +https://mirrors.tuna.tsinghua.edu.cn/alpine/edge/main\n\ +https://mirrors.tuna.tsinghua.edu.cn/alpine/edge/testing\n\ +' > /etc/apk/repositories \ + && apk update \ + && apk add --no-cache bash curl coreutils iproute2 \ + && echo "alias ls='ls --color=auto'" >> /root/.bashrc \ + && rm -rf /var/cache/apk/* +ENV PS1='\[\e[33;1;1m\][\[\e[0m\]\[\e[35;1m\]\u\[\e[0m\]\[\e[33;1;1m\]@\[\e[0m\]\[\e[31;1;1m\]docker\[\e[0m\]\[\e[32;1;1m\](\h)\[\e[0m\]\[\e[33;1;1m\]:\[\e[0m\]\[\e[32m\]\w\[\e[0m\]\[\e[33;1;1m\]]\[\e[0m\]\[\e[36m\]\$\[\e[0m\] ' +ENV PS2='\[\e[36m\]>\[\e[0m\] ' +ENV LANG=en_US.UTF-8 + diff --git a/alpine/README.md b/alpine/README.md new file mode 100644 index 0000000..26d5d4e --- /dev/null +++ b/alpine/README.md @@ -0,0 +1,11 @@ +# 构建 alpine 镜像 + +## 导入文件 +- 本机时区 /etc/localtime + +## 定制 +- 使用 Asia/Shanghai 时区 +- 修改软件源，开启 edge +- 安装 bash curl coreutils iproute2 +- 默认语言 en_US.UTF-8 + diff --git a/alpine/alpine.sh b/alpine/alpine.sh new file mode 100755 index 0000000..2d3ef7a --- /dev/null +++ b/alpine/alpine.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + Warn Preparing localtime ... + cd $ROOT_DIR + cp -f /etc/localtime ADD/ +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/centos7-django2.2/Dockerfile b/centos7-django2.2/Dockerfile new file mode 100644 index 0000000..3f80acf --- /dev/null +++ b/centos7-django2.2/Dockerfile @@ -0,0 +1,26 @@ +ARG ARCH +FROM harbor.colben.cn/general/centos-python$ARCH:7-3.6 +MAINTAINER Colben colbenlee@gmail.com +ARG ARCH +RUN echo -e "[mysql80-community]\n\ +name=MySQL 8.0 Community Server\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-8.0-community-el7${ARCH:--x86_64}/\n\ +enabled=1\n\ +gpgcheck=0\n\ +" > /etc/yum.repos.d/mysql.repo \ + && yum makecache fast \ + && yum install mysql-community-devel -y \ + && pip3 install \ + -i https://pypi.tuna.tsinghua.edu.cn/simple \ + --trusted-host pypi.tuna.tsinghua.edu.cn \ + requests==2.18.3 \ + redis==3.4.1 \ + django==2.2.10 \ + djangorestframework==3.11.0 \ + mysqlclient==1.4.6 \ + sqlparse==0.3.0 \ + django-cors-headers==3.2.0 \ + uwsgi==2.0.18 \ + django-mysql==3.5.0 \ + && rm -rf /root/.cache/pip /var/cache/yum + diff --git a/centos7-django2.2/README.md b/centos7-django2.2/README.md new file mode 100644 index 0000000..1ff55ba --- /dev/null +++ b/centos7-django2.2/README.md @@ -0,0 +1,5 @@ +# 构建 django 镜像 + +## 定制 +- 安装 django 2.2 及其 mysql 依赖包 + diff --git a/centos7-django2.2/centos-django.sh b/centos7-django2.2/centos-django.sh new file mode 100755 index 0000000..ee48606 --- /dev/null +++ b/centos7-django2.2/centos-django.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:7-2.2" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/centos7-python3.6/Dockerfile b/centos7-python3.6/Dockerfile new file mode 100644 index 0000000..87b783a --- /dev/null +++ b/centos7-python3.6/Dockerfile @@ -0,0 +1,7 @@ +ARG ARCH +FROM harbor.colben.cn/general/centos$ARCH:7 +MAINTAINER Colben colbenlee@gmail.com +RUN yum makecache fast \ + && yum install gcc python36-devel -y \ + && rm -rf /var/cache/yum + diff --git a/centos7-python3.6/README.md b/centos7-python3.6/README.md new file mode 100644 index 0000000..798bea8 --- /dev/null +++ b/centos7-python3.6/README.md @@ -0,0 +1,5 @@ +# 构建 python 镜像 + +## 定制 +- 安装 python36 及其编译工具 + diff --git a/centos7-python3.6/centos-python.sh b/centos7-python3.6/centos-python.sh new file mode 100755 index 0000000..d9ff60c --- /dev/null +++ b/centos7-python3.6/centos-python.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:7-3.6" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/centos7/Dockerfile b/centos7/Dockerfile new file mode 100644 index 0000000..169b236 --- /dev/null +++ b/centos7/Dockerfile @@ -0,0 +1,20 @@ +FROM centos:7.9.2009 +MAINTAINER Colben colbenlee@gmail.com +ARG ARCH +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && rm -f /etc/yum.repos.d/CentOS-* \ + && echo -e "[centos7]\n\ +name=centos base\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos${ARCH:+-altarch}/$releasever/os/$basearch/\n\ +enabled=1\n\ +gpgcheck=0\n\ +" > /etc/yum.repos.d/centos7.repo \ + && yum makecache fast \ + && yum update -y \ + && yum -y install less iproute \ + && rm -rf /var/cache/yum \ + && rm -rf /var/log/* +ENV PS1='\[\e[33;1;1m\][\[\e[0m\]\[\e[35;1m\]\u\[\e[0m\]\[\e[33;1;1m\]@\[\e[0m\]\[\e[31;1;1m\]docker\[\e[0m\]\[\e[32;1;1m\](\h)\[\e[0m\]\[\e[33;1;1m\]:\[\e[0m\]\[\e[32m\]\w\[\e[0m\]\[\e[33;1;1m\]]\[\e[0m\]\[\e[36m\]\$\[\e[0m\] ' +ENV PS2='\[\e[36m\]>\[\e[0m\] ' +ENV LANG=en_US.UTF-8 + diff --git a/centos7/README.md b/centos7/README.md new file mode 100644 index 0000000..724d325 --- /dev/null +++ b/centos7/README.md @@ -0,0 +1,7 @@ +# 构建 centos7 镜像 + +## 定制 +- 使用 Asia/Shanghai 时区 +- 安装 less iproute +- 默认语言 en_US.UTF-8 + diff --git a/centos7/centos.sh b/centos7/centos.sh new file mode 100755 index 0000000..35cddbc --- /dev/null +++ b/centos7/centos.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:7" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/datax-web/ADD/ccmd b/datax-web/ADD/ccmd new file mode 100755 index 0000000..65b0b5c --- /dev/null +++ b/datax-web/ADD/ccmd @@ -0,0 +1,67 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /opt/datax-web-2.1.2/modules/datax-admin/logs# +# - /opt/datax-web-2.1.2/modules/datax-admin/bin/console.out# +# - /opt/datax-web-2.1.2/modules/datax-datax-executor/logs# +# - /opt/datax-web-2.1.2/modules/datax-datax-executor/bin/console.out# +# ENV # +# - JAVA_OPTS # +# - TIMEOUT # +# - MAX_PROCS # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +GOT_SIGTERM= +TIMEOUT="${TIMEOUT:-10m}" +MAX_PROCS=${MAX_PROCS:-1} + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f java && Print killing java ... || break + sleep 1 + done +# exec 1022<&- + Print Container stopped. + test -n "$GOT_SIGTERM" +} + + + +function ModifyConf { + local kv= + Print Modify bootstrap.properties ... + while read kv; do + [ -z "$kv" ] && return 0 + Print Modify property: ${kv%%=*} ... + sed -i "/^#${kv%%=*} *=/c$kv" /opt/datax-web-2.1.2/modules/datax-admin/conf/bootstrap.properties + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" +} + +function StartProc { + Print Start datax-web + cd /opt/datax-web-2.1.2 + /usr/bin/bash bin/start-all.sh + tail -f /dev/null +} + + +function Main { + ModifyConf + trap "GOT_SIGTERM=1; Print Got SIGTERM ...; exit" SIGTERM + StartProc +} + +# Start here +Main + diff --git a/datax-web/Dockerfile b/datax-web/Dockerfile new file mode 100644 index 0000000..25d7b0c --- /dev/null +++ b/datax-web/Dockerfile @@ -0,0 +1,11 @@ +ARG ARCH +FROM harbor.colben.cn/general/jdk$ARCH:8u202 +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /opt/ +RUN tar xf /opt/datax-web-2.1.2.tar.gz -C /opt/ && \ + cd /opt/datax-web-2.1.2 && \ + ./bin/install.sh -f && \ + yum makecache fast && yum install -y awk && rm -rf /var/cache/tdnf/ && \ + rm -rf /opt/datax-web-2.1.2.tar.gz +CMD ["/opt/ccmd"] + diff --git a/datax-web/README.md b/datax-web/README.md new file mode 100644 index 0000000..af951a0 --- /dev/null +++ b/datax-web/README.md @@ -0,0 +1,18 @@ +# 构建 datax-web 镜像 + +## 导入文件 +- [下载 datax-web.git]( https://github.com/WeiYe-Jing/datax-web.git) + + +## 外挂目录和文件 +- /opt/datax-web-2.1.2/modules/datax-admin/bin/console.out: admin 日志文件 +- /opt/datax-web-2.1.2/modules/datax-datax-executor/bin/console.out: 执行器日志文件 + +## db文件在容器里边 +- 启动容器后docker cp {datax-web}:/opt/datax-web-2.1.2/bin/db/datax_web.sql . +- 创建数据库: create database datax_web; +- 导入数据:use datax_web ; source datax_web.sql; +- 创建用户:create user admin@'%' identified by '123456'; +- 授权用户:grant all on datax_web.* to admin@'%'; +- 访问地址 http://服务器ip:9527/index.html 帐号:admin 密码:123456 +## docker-compose.yml案例 diff --git a/datax-web/datax-web.sh b/datax-web/datax-web.sh new file mode 100755 index 0000000..4f5fc78 --- /dev/null +++ b/datax-web/datax-web.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + Warn Preparing datax ... + cd $ROOT_DIR/ADD + rm -rf $(ls | grep -v ccmd || true) + tar zxf /release/RUNTIME/datax.tar.gz -C . + rm -rf datax/tmp datax/job/*.json + rm -f datax/plugin/writer/mysqlwriter/libs/mysql-connector-java-5.1.34.jar + rm -f datax/plugin/reader/mysqlreader/libs/mysql-connector-java-5.1.34.jar + cp /release/RUNTIME/mysql-connector-java-8.0.27.jar datax/plugin/reader/mysqlreader/libs/ + cp /release/RUNTIME/mysql-connector-java-8.0.27.jar datax/plugin/writer/mysqlwriter/libs/ + find datax/ -type f | xargs chmod 0644 + mkdir datax/{hook,log,log_perf} +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + # YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { +# Update + Build + END=1 +} + +# Start here +Main + diff --git a/datax-web/docker-compose.yml b/datax-web/docker-compose.yml new file mode 100644 index 0000000..9aff6f1 --- /dev/null +++ b/datax-web/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.7" + +services: + datax-web: + image: harbor.colben.cn/general/datax-web + container_name: datax-web + command: /opt/ccmd + restart: on-failure + privileged: true + stop_grace_period: 1m + network_mode: "host" + ports: + - "9527:9527" + environment: + _CONF_DB_HOST: 10.0.4.115 + _CONF_DB_PORT: 3336 + _CONF_DB_USERNAME: admin + _CONF_DB_PASSWORD: 123456 + _CONF_DB_DATABASE: datax_web + volumes: + - type: bind + source: ./data-admin.console.out + target: /opt/datax-web-2.1.2/modules/datax-admin/bin/console.out + - type: bind + source: ./executor-console.out + target: /opt/datax-web-2.1.2/modules/datax-datax-executor/bin/console.out diff --git a/datax/ADD/ccmd b/datax/ADD/ccmd new file mode 100755 index 0000000..eded8b1 --- /dev/null +++ b/datax/ADD/ccmd @@ -0,0 +1,112 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /opt/datax/log # +# - /opt/datax/log_perf # +# - /opt/datax/job # +# ENV # +# - JAVA_OPTS # +# - TIMEOUT # +# - MAX_PROCS # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +GOT_SIGTERM= +TIMEOUT="${TIMEOUT:-10m}" +MAX_PROCS=${MAX_PROCS:-1} + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f java && Print killing java ... || break + sleep 1 + done + exec 1022<&- + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **env TIMEOUT, default 10m(ten minutes)** + **env MAX_PROCS, default 1** + **/opt/datax/{log,log_perf,job} mounted from host** + ' +} + +function InitPipe { + Print Init named pipe ... + rm -rf pool.pipe + mkfifo pool.pipe + exec 1022<> pool.pipe + rm -rf pool.pipe + printf "%${MAX_PROCS}s" '' >&1022 +} + +function StartJob { + local job="$1" + local code=0 + Print Start job $job with timeout $TIMEOUT ... + timeout ${TIMEOUT} java \ + -server \ + -Xms1g \ + -Xmx1g \ + -Duser.timezone=GMT+08 \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath=$PWD/log \ + ${JAVA_OPTS:-} \ + -Dfile.encoding=UTF-8 \ + -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ + -Djava.security.egd=file:///dev/urandom \ + -Ddatax.home=$PWD \ + -Dlogback.configurationFile=$PWD/conf/logback.xml \ + -classpath "$PWD/lib/*:." \ + -Dlog.file.name=$job \ + com.alibaba.datax.core.Engine \ + -mode standalone \ + -jobid -1 \ + -job $PWD/job/$job.json \ + >/dev/null \ + 2>log/$job.error \ + || code=$? + if [ 0 -eq $code ]; then + Print Job $job finished. + elif [ 124 -eq $code ]; then + Print Job $job timeout! + else + Print Job $job stopped unexpectly! + fi + echo >&1022 +} + +function StartProc { + Print Start datax with max $MAX_PROCS parallel jobs ... + local job= + for job in $(ls job/ | grep '\.json$'); do + read -n 1 -u 1022 + StartJob "${job%.json}" & + done + wait + [ -n "$job" ] || Print Not found any job! +} + +function Main { + cd /opt/datax + Usage + InitPipe + trap "GOT_SIGTERM=1; Print Got SIGTERM ...; exit" SIGTERM + StartProc +} + +# Start here +Main + diff --git a/datax/Demo/MultiProc/README.md b/datax/Demo/MultiProc/README.md new file mode 100644 index 0000000..e81f9f7 --- /dev/null +++ b/datax/Demo/MultiProc/README.md @@ -0,0 +1,27 @@ +# 部署多进程 datax + +- 两组 job +- 第一组每天 5 点执行一次，每次最多并行 3 个 job，每个 job 超时时间十五分钟 +- 第二组每天 6 点执行一次，每次最多并行 4 个 job，每个 job 超时时间一小时 +- 根据实际环境修改 + - docker-compose.yml + - datax/job/xxxx.json + +- 创建目录 + ``` + grep '\> $conf + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" + Print Remove path.data and path.log in $conf ... + sed -i -e '/^path\.data/d' -e '/^path\.logs/d' $conf +} + +function InstallPlugin { + for f in $(ls -d offline-plugins/*.zip 2>/dev/null); do + Print Install plugins from offline file: $f ... + ./bin/elasticsearch-plugin install file://$f + mv $f $f.installed + done +} + +function ChangeOwner { + Print Change file owner ... + chown -R es.es config/ data/ logs/ plugins/ +} + +function ChangeSysConf { + Print Change system conf ... + echo 262144 > /proc/sys/vm/max_map_count || Print Not specified "--privileged". +} + +function StartProc { + Print Start elasticsearch ... + su - es -c " + export JAVA_HOME=$JAVA_HOME + export PATH=$PATH + export ES_JAVA_OPTS='${ES_JAVA_OPTS:-}' + /opt/es/bin/elasticsearch -Epath.data=/opt/es/data -Epath.logs=/opt/es/logs + " &> /dev/null & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /opt/es + Usage + RestoreConf + ModifyConf + InstallPlugin + ChangeOwner + ChangeSysConf + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/elasticsearch6/Demo/SingleNode/README.md b/elasticsearch6/Demo/SingleNode/README.md new file mode 100644 index 0000000..2be9a70 --- /dev/null +++ b/elasticsearch6/Demo/SingleNode/README.md @@ -0,0 +1,15 @@ +# 部署单节点 es + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\> config/elasticsearch.yml + tar zcf config.tgz config + rm -rf config/* +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/elasticsearch7/ADD/ccmd b/elasticsearch7/ADD/ccmd new file mode 100755 index 0000000..c15719d --- /dev/null +++ b/elasticsearch7/ADD/ccmd @@ -0,0 +1,157 @@ +#!/bin/bash + +################################################## +# Docker # +# -- privileged # +# Mount dir # +# - /opt/es/config # +# - /opt/es/data # +# - /opt/es/logs # +# - /opt/es/offline-plugins # +# - /opt/es/plugins # +# ENV # +# - _CONF_* # +# - ES_JAVA_OPTS # +# - ELASTIC_PASSWORD # +# - APM_SYSTEM_PASSWORD # +# - KIBANA_SYSTEM_PASSWORD # +# - LOGSTASH_SYSTEM_PASSWORD # +# - BEATS_SYSTEM_PASSWORD # +# - REMOTE_MONITORING_USER_PASSWORD # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= +BOOTSTRAP= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f java && Print killing java ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **root user** + **privileted** + **/opt/es/{config,data,logs,offline-plugins,plugins} mounted from host** + **elastic passwords in production** + ' +} + +function RestoreConf { + if [ -z "$(ls config/)" ]; then + Print Restore default config files and quit ... + tar zxf config.tgz + exit + fi +} + +function ModifyConf { + local kv= + local conf='config/elasticsearch.yml' + Print Modify $conf ... + while read kv; do + [ -z "$kv" ] && break + sed -i "/^${kv%%=*}: /d" $conf + echo "${kv/=/: }" >> $conf + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" + Print Remove path.data and path.log in $conf ... + sed -i -e '/^path\.data/d' -e '/^path\.logs/d' $conf + if grep -q '^cluster\.initial_master_nodes' $conf; then + [ -z "$(ls data/)" -a -n "${ELASTIC_PASSWORD:-}" ] && BOOTSTRAP=1 && return 0 + Print Remove cluster.initial_master_nodes in $conf ... + sed -i '/^cluster\.initial_master_nodes/d' $conf + fi +} + +function InstallPlugin { + for f in $(ls -d offline-plugins/*.zip 2>/dev/null); do + Print Install plugins from offline file: $f ... + ./bin/elasticsearch-plugin install file://$f + mv $f $f.installed + done +} + +function ChangeOwner { + Print Change file owner ... + chown -R es.es config/ data/ logs/ plugins/ +} + +function ChangeSysConf { + Print Change system conf ... + echo 262144 > /proc/sys/vm/max_map_count || Print Not specified "--privileged". +} + +function SetupPassword { + local count=0 + while Print Try to setup passwords of buildin users ...; do + sleep 15 + if /opt/es/bin/elasticsearch-setup-passwords interactive &> logs/setup.out <<< "y +$ELASTIC_PASSWORD +$ELASTIC_PASSWORD +${APM_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${APM_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${KIBANA_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${KIBANA_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${LOGSTASH_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${LOGSTASH_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${BEATS_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${BEATS_SYSTEM_PASSWORD:-$ELASTIC_PASSWORD} +${REMOTE_MONITORING_USER_PASSWORD:-$ELASTIC_PASSWORD} +${REMOTE_MONITORING_USER_PASSWORD:-$ELASTIC_PASSWORD} +"; then + grep -q '^Changed password for user ' logs/setup.out \ + && Print Succeeded to setup passwords of buildin users. \ + && return 0 + else + Print Failed to execute elasticsearch-setup-passwords! + fi + [ 4 -le $((++count)) ] && Print Failed to setup passwords of buildin users! && exit + done +} + +function StartProc { + Print Start elasticsearch ... + su - es -c " + export ES_JAVA_OPTS='${ES_JAVA_OPTS:-}' + /opt/es/bin/elasticsearch -Epath.data=/opt/es/data -Epath.logs=/opt/es/logs + " &> /dev/null & + PIDS="$PIDS $!" + [ -n "$BOOTSTRAP" ] && SetupPassword +} + +function Main { + local pid= + cd /opt/es + Usage + RestoreConf + ModifyConf + InstallPlugin + ChangeOwner + ChangeSysConf + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/elasticsearch7/Demo/MultiRoles/README.md b/elasticsearch7/Demo/MultiRoles/README.md new file mode 100644 index 0000000..fa18e1b --- /dev/null +++ b/elasticsearch7/Demo/MultiRoles/README.md @@ -0,0 +1,18 @@ +# 部署多角色 es 集群 + +- 部署集群，有两个 master 节点和三个 data 节点 +- 每个节点的 127.0.1.x 用于 http 请求 +- 每个节点的 127.0.3.x 用于节点间通信 +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\> config/elasticsearch.yml + tar zcf config.tgz config + rm -rf config/* +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/gitea/ADD/ccmd b/gitea/ADD/ccmd new file mode 100755 index 0000000..2516029 --- /dev/null +++ b/gitea/ADD/ccmd @@ -0,0 +1,76 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /var/lib/gitea # +# - /var/log/gitea # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f gitea && Print killing gitea ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **root user** + **/var/{lib,log}/gitea mounted from host** + ' +} + +function RestoreConf { + if [ -z "$(ls gitea/)" ]; then + Print Restore default config files and quit ... + tar zxf gitea.tgz + exit + fi +} + +function ChangeOwner { + Print Change file owner ... + chown -R gitea.www-data gitea/ /var/log/gitea/ +} + +function StartProc { + Print Start gitea ... + su - gitea -c ' + gitea web --config /var/lib/gitea/custom/conf/app.ini + ' &>> /var/log/gitea/gitea.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /var/lib + Usage + RestoreConf + ChangeOwner + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/gitea/Demo/GiteaWithNginx/README.md b/gitea/Demo/GiteaWithNginx/README.md new file mode 100644 index 0000000..43483a3 --- /dev/null +++ b/gitea/Demo/GiteaWithNginx/README.md @@ -0,0 +1,59 @@ +# 部署 gitea, 由 nginx 反代 + +- 配合 nginx 反代，并单独挂载 indexers 目录 +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\> $conf + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" +} + +function StartProc { + Print Start kafka ... + ./bin/kafka-server-start.sh ./config/server.properties --override log.dirs=./data \ + &>> logs/kafka.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /opt/kafka + Usage + ModifyConf + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/kafka/Demo/SingleNode/README.md b/kafka/Demo/SingleNode/README.md new file mode 100644 index 0000000..122c7b1 --- /dev/null +++ b/kafka/Demo/SingleNode/README.md @@ -0,0 +1,15 @@ +# 部署 kafka 单节点 + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\> $LOG_DIR/keepalived.log & + PIDS="$PIDS $!" +} + +function Main { + local pid= + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/keepalived/Dockerfile b/keepalived/Dockerfile new file mode 100644 index 0000000..55f9c65 --- /dev/null +++ b/keepalived/Dockerfile @@ -0,0 +1,11 @@ +ARG ARCH +FROM harbor.colben.cn/general/alpine$ARCH +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /opt/ +RUN apk update \ + && apk add --no-cache keepalived \ + && mkdir -p /var/log/keepalived \ + && rm -f /etc/keepalived/keepalived.conf \ + && rm -rf /var/cache/apk/* +CMD ["/opt/ccmd"] + diff --git a/keepalived/README.md b/keepalived/README.md new file mode 100644 index 0000000..a1c808a --- /dev/null +++ b/keepalived/README.md @@ -0,0 +1,13 @@ +# 构建 keepalived 镜像 + +## 定制 +- 安装 keepalived +- docker 参数: --privileged --net host + +## 外挂目录和文件 +- /etc/keepalived: keepalived 配置目录 +- /var/log/keepalived: keepalived 日志目录 + +## 案例 +- [/OPS/GeneralDocker/mysql/Demo/TowMasterNodes/](/OPS/GeneralDocker/mysql/Demo/TowMasterNodes/): 两台 mysql 高可用，不抢占模式 + diff --git a/keepalived/keepalived.sh b/keepalived/keepalived.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/keepalived/keepalived.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/kibana/ADD/ccmd b/kibana/ADD/ccmd new file mode 100755 index 0000000..28f0ec2 --- /dev/null +++ b/kibana/ADD/ccmd @@ -0,0 +1,104 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /opt/kibana/config # +# - /opt/kibana/data # +# - /opt/kibana/logs # +# - /opt/kibana/offline-plugins # +# - /opt/kibana/plugins # +# ENV # +# - _CONF_* # +# - NODE_OPTIONS # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f node && Print killing node ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **root user** + **/opt/kibana/{config,data,logs,offline-plugins,plugins} mounted from host** + ' +} + +function RestoreConf { + if [ -z "$(ls config/)" ]; then + Print Restore default config files and quit ... + tar zxf config.tgz + exit + fi +} + +function ModifyConf { + local kv= + local conf='config/kibana.yml' + Print Modify $conf ... + while read kv; do + [ -z "$kv" ] && break + sed -i "/^${kv%%=*}: /d" $conf + echo "${kv/=/: }" >> $conf + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" +} + +function InstallPlugin { + for f in $(ls -d offline-plugins/*.zip 2>/dev/null); do + Print Install plugins from offline file: $f ... + ./bin/kibana-plugin install file://$f + mv $f $f.installed + done +} + +function ChangeOwner { + Print Change file owner ... + chown -R kibana.kibana config/ data/ logs/ plugins/ +} + +function StartProc { + Print Start kibana ... + su - kibana -c " + export NODE_OPTIONS='${NODE_OPTIONS:-}' + /opt/kibana/bin/kibana + " &>> logs/kibana.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /opt/kibana + Usage + RestoreConf + ModifyConf + InstallPlugin + ChangeOwner + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/kibana/Demo/SingleNode/README.md b/kibana/Demo/SingleNode/README.md new file mode 100644 index 0000000..084cbad --- /dev/null +++ b/kibana/Demo/SingleNode/README.md @@ -0,0 +1,17 @@ +# 部署 kibana + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\> config/kibana.yml + tar zcf config.tgz config + rm -rf config/* +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/letsencrypt/ADD/ccmd b/letsencrypt/ADD/ccmd new file mode 100755 index 0000000..6f8d0d3 --- /dev/null +++ b/letsencrypt/ADD/ccmd @@ -0,0 +1,79 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /etc/letsencrypt # +# - /var/log/letsencrypt # +# ENV # +# - DOMAINS # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +GOT_SIGTERM= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f python && Print killing python ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **host network** + **env DOMAINS** + **/etc/letsencrypt and /var/log/letsencrypt mounted from host** + ' +} + +function StartProc { + if [ ! -e /etc/letsencrypt/accounts ]; then + Print Register ... + certbot register --register-unsafely-without-email --agree-tos + if echo "$DOMAINS" | grep -qo '^*'; then + Print Request wildcard certificate ... + certbot certonly -q --manual \ + --manual-auth-hook /etc/letsencrypt/manual-hook.sh \ + -d "$DOMAINS" --preferred-challenges dns \ + --server https://acme-v02.api.letsencrypt.org/directory + else + Print Request certificate ... + certbot certonly -q -n --standalone -d $DOMAINS + fi + Print Generate dhparam.pem ... + openssl dhparam -out /etc/letsencrypt/dhparam.pem 2048 \ + &>/var/log/letsencrypt/dhparam.out + else + if echo "$DOMAINS" | grep -qo '^*'; then + Print Renew wildcard certificate ... + certbot certonly --force-renewal -q --manual \ + --manual-auth-hook /etc/letsencrypt/manual-hook.sh \ + -d "$DOMAINS" --preferred-challenges dns \ + --server https://acme-v02.api.letsencrypt.org/directory + else + Print Renew certificate ... + certbot renew -q --force-renewal + fi + fi +} + +function Main { + Usage + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + StartProc +} + +# Start here +Main + diff --git a/letsencrypt/Demo/SingleNode/README.md b/letsencrypt/Demo/SingleNode/README.md new file mode 100644 index 0000000..3831108 --- /dev/null +++ b/letsencrypt/Demo/SingleNode/README.md @@ -0,0 +1,35 @@ +# 部署 letsencrypt + +- 为域名 x1.xx.com 和 x2.xx.com 申请 ssl 证书，并在每月的 31 号晚上十一点更新一次 +- 为域名 \*.xxx.com 申请 ssl 证书，并在每月的 31 号晚上十点更新一次 +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ ${0%.sh}.out + function Print { echo -e "$(date +'[%F %T] INFO') $*"; } + function Warn { echo -e "$(date +'[%F %T] WARN') $*"; } + function Error { echo -e "$(date +'[%F %T] ERROR') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to request aliyun api! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to request aliyun api and wait 30 seconds. + sleep 30 +} + +function GetSignature { + local uriEncoded="GET&%2F&$(echo "$1" | sed -e 's/=/%3D/g' -e 's/:/%253A/g' -e 's/&/%26/g')" + local sha1Str=$(echo -n "$uriEncoded" | openssl dgst -sha1 -hmac "$ACCESS_KEY_SECRET&" -binary) + echo -n "$sha1Str" | base64 | sed -e 's/=/%3D/g' -e 's/+/%2B/g' -e 's,/,%2F,g' +} + +function ListRecord { + Warn Get request uri ... + local sign= + local resp= + local uri="AccessKeyId=$ACCESS_KEY_ID" + uri="${uri}&Action=DescribeDomainRecords" + uri="${uri}&DomainName=$DOMAIN" + uri="${uri}&Format=JSON" + uri="${uri}&KeyWord=$SUB_DOMAIN" + uri="${uri}&SearchMode=EXACT" + uri="${uri}&SignatureMethod=HMAC-SHA1" + uri="${uri}&SignatureNonce=$RANDOM" + uri="${uri}&SignatureVersion=1.0" + uri="${uri}&Timestamp=$(date +'%FT%TZ' -d'8 hours ago')" + uri="${uri}&Type=TXT" + uri="${uri}&Version=2015-01-09" + sign=$(GetSignature "$uri") + Warn List record ... + resp=$(curl -sSL -XGET "http://alidns.aliyuncs.com/?$uri&Signature=$sign" | jq -eM .) + RECORD_ID=$(echo $resp | jq -crM .DomainRecords.Record[].RecordId) + [ 'null' == "$RECORD_ID" ] && echo "$resp" && exit 1 + return 0 +} + +function CreateRecord { + Warn Get request uri ... + local sign= + local resp= + local uri="AccessKeyId=$ACCESS_KEY_ID" + uri="${uri}&Action=AddDomainRecord" + uri="${uri}&DomainName=$DOMAIN" + uri="${uri}&Format=JSON" + uri="${uri}&RR=$SUB_DOMAIN" + uri="${uri}&SignatureMethod=HMAC-SHA1" + uri="${uri}&SignatureNonce=$RANDOM" + uri="${uri}&SignatureVersion=1.0" + uri="${uri}&Timestamp=$(date +'%FT%TZ' -d'8 hours ago')" + uri="${uri}&Type=TXT" + uri="${uri}&Value=$RECORD_VA" + uri="${uri}&Version=2015-01-09" + sign=$(GetSignature "$uri") + Warn Create sub_domain: $SUB_DOMAIN with value: $RECORD_VA ... + resp=$(curl -sSL -XGET "http://alidns.aliyuncs.com/?$uri&Signature=$sign" | jq -eM .) + [ 'null' != "$(echo $resp | jq -crM .Message)" ] && echo "$resp" && exit 1 + return 0 +} + +function ModifyRecord { + Warn Get request uri ... + local sign= + local resp= + local uri="AccessKeyId=$ACCESS_KEY_ID" + uri="${uri}&Action=UpdateDomainRecord" + uri="${uri}&DomainName=$DOMAIN" + uri="${uri}&Format=JSON" + uri="${uri}&RR=$SUB_DOMAIN" + uri="${uri}&RecordId=$RECORD_ID" + uri="${uri}&SignatureMethod=HMAC-SHA1" + uri="${uri}&SignatureNonce=$RANDOM" + uri="${uri}&SignatureVersion=1.0" + uri="${uri}&Timestamp=$(date +'%FT%TZ' -d'8 hours ago')" + uri="${uri}&Type=TXT" + uri="${uri}&Value=$RECORD_VA" + uri="${uri}&Version=2015-01-09" + sign=$(GetSignature "$uri") + Warn Modify record: $RECORD_ID with value: $RECORD_VA ... + resp=$(curl -sSL -XGET "http://alidns.aliyuncs.com/?$uri&Signature=$sign" | jq -eM .) + [ 'null' != "$(echo $resp | jq -crM .Message)" ] && echo "$resp" && exit 1 + return 0 +} + +function DeleteRecord { + Warn Get request uri ... + local sign= + local resp= + local uri="AccessKeyId=$ACCESS_KEY_ID" + uri="${uri}&Action=DeleteDomainRecord" + uri="${uri}&DomainName=$DOMAIN" + uri="${uri}&Format=JSON" + uri="${uri}&RecordId=$RECORD_ID" + uri="${uri}&SignatureMethod=HMAC-SHA1" + uri="${uri}&SignatureNonce=$RANDOM" + uri="${uri}&SignatureVersion=1.0" + uri="${uri}&Timestamp=$(date +'%FT%TZ' -d'8 hours ago')" + uri="${uri}&Version=2015-01-09" + sign=$(GetSignature "$uri") + Warn Delete record $RECORD_ID ... + resp=$(curl -sSL -XGET "http://alidns.aliyuncs.com/?$uri&Signature=$sign" | jq -eM .) + [ 'null' != "$(echo $resp | jq -crM .Message)" ] && echo "$resp" && exit 1 + return 0 +} + +function Main { + [ -e "$PID_FILE" ] && Error Pid file $PID_FILE already exists, quit! + echo $$ > $PID_FILE + ListRecord + [ -z "$RECORD_ID" ] && CreateRecord + [ -n "$RECORD_ID" ] && ModifyRecord + END=1 +} + +# Start here +Main + diff --git a/letsencrypt/Demo/SingleNode/docker-compose.yml b/letsencrypt/Demo/SingleNode/docker-compose.yml new file mode 100644 index 0000000..61b3319 --- /dev/null +++ b/letsencrypt/Demo/SingleNode/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3.7" + +services: + letsencrypt: + image: harbor.colben.cn/general/letsencrypt + container_name: letsencrypt + restart: "no" + stop_grace_period: 1m + environment: + DOMAINS: x1.xx.com,x2.xx.com + network_mode: host + volumes: + - type: bind + source: ./letsencrypt/etc + target: /etc/letsencrypt + - type: bind + source: ./letsencrypt/log + target: /var/log/letsencrypt + + letsencrypt-wildcard: + image: harbor.colben.cn/general/letsencrypt + container_name: letsencrypt-wildcard + restart: "no" + stop_grace_period: 1m + environment: + DOMAINS: "*.xxx.com" + network_mode: host + volumes: + - type: bind + source: ./letsencrypt-wildcard/etc + target: /etc/letsencrypt + - type: bind + source: ./letsencrypt-wildcard/log + target: /var/log/letsencrypt + diff --git a/letsencrypt/Demo/SingleNode/tencent-api.sh b/letsencrypt/Demo/SingleNode/tencent-api.sh new file mode 100755 index 0000000..b54d813 --- /dev/null +++ b/letsencrypt/Demo/SingleNode/tencent-api.sh @@ -0,0 +1,136 @@ +i#!/bin/bash +#========================================= +# Author : colben +# Create : 2022-04-04 10:12 +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +SECRET_ID='tencent secret id' +SECRET_KEY='tencent secret key' +DOMAIN=$CERTBOT_DOMAIN +SUB_DOMAIN=_acme-challenge +RECORD_ID= +RECORD_VA=$CERTBOT_VALIDATION +PID_FILE=/tmp/$(basename ${0%.sh}).pid + +if [ -t 0 ]; then + function Print { echo -e "\033[32;1m$(date +'[%F %T]') $*\033[0m"; } + function Warn { echo -e "\033[33;1m$(date +'[%F %T]') $*\033[0m"; } + function Error { echo -e "\033[31;1m$(date +'[%F %T]') $*\033[0m"; exit 1; } +else + #exec &> ${0%.sh}.out + function Print { echo -e "$(date +'[%F %T] INFO') $*"; } + function Warn { echo -e "$(date +'[%F %T] WARN') $*"; } + function Error { echo -e "$(date +'[%F %T] ERROR') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to request tencent api! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to request tencent api and wait 30 seconds. + sleep 30 +} + +function GetSignature { + local sha1Str=$(echo -n "GET$1" | openssl dgst -sha1 -hmac "$SECRET_KEY" -binary) + echo -n "$sha1Str" | base64 | sed -e 's/=/%3D/g' -e 's/+/%2B/g' +} + +function ListRecord { + Warn Get request url ... + local sign= + local resp= + local url='cns.api.qcloud.com/v2/index.php' + url="${url}?Action=RecordList" + url="${url}&Nonce=$RANDOM" + url="${url}&SecretId=$SECRET_ID" + url="${url}&Timestamp=$(date +%s)" + url="${url}&Version=2018-08-08" + url="${url}&domain=$DOMAIN" + sign=$(GetSignature "$url") + Warn List record ... + resp=$(curl -sSL -XGET "https://$url&Signature=$sign" | jq -eM .) + [ '0' != "$(echo $resp | jq -crM .code)" ] && echo "$resp" && exit 1 + RECORD_ID=$(echo $resp | jq -crM ".data.records[] | select(.name == \"$SUB_DOMAIN\") | .id") +} + +function CreateRecord { + Warn Get request url ... + local sign= + local resp= + local url='cns.api.qcloud.com/v2/index.php' + url="${url}?Action=RecordCreate" + url="${url}&Nonce=$RANDOM" + url="${url}&SecretId=$SECRET_ID" + url="${url}&Timestamp=$(date +%s)" + url="${url}&Version=2018-08-08" + url="${url}&domain=$DOMAIN" + url="${url}&recordLine=默认" + url="${url}&recordType=TXT" + url="${url}&subDomain=$SUB_DOMAIN" + url="${url}&value=$RECORD_VA" + sign=$(GetSignature "$url") + Warn Create sub_domain: $SUB_DOMAIN with value: $RECORD_VA ... + resp=$(curl -sSL -XGET "https://$url&Signature=$sign" | jq -eM .) + [ '0' != "$(echo $resp | jq -crM .code)" ] && echo "$resp" && exit 1 + return 0 +} + +function ModifyRecord { + Warn Get request url ... + local sign= + local resp= + local url='cns.api.qcloud.com/v2/index.php' + url="${url}?Action=RecordModify" + url="${url}&Nonce=$RANDOM" + url="${url}&SecretId=$SECRET_ID" + url="${url}&Timestamp=$(date +%s)" + url="${url}&Version=2018-08-08" + url="${url}&domain=$CERTBOT_DOMAIN" + url="${url}&recordId=$RECORD_ID" + url="${url}&recordLine=默认" + url="${url}&recordType=TXT" + url="${url}&subDomain=$SUB_DOMAIN" + url="${url}&value=$RECORD_VA" + sign=$(GetSignature "$url") + Warn Modify record: $RECORD_ID with value: $RECORD_VA ... + resp=$(curl -sSL -XGET "https://$url&Signature=$sign" | jq -eM .) + [ '0' != "$(echo $resp | jq -crM .code)" ] && echo "$resp" && exit 1 + return 0 +} + +function DeleteRecord { + Warn Get request url ... + local sign= + local resp= + local url='cns.api.qcloud.com/v2/index.php' + url="${url}?Action=RecordDelete" + url="${url}&Nonce=$RANDOM" + url="${url}&SecretId=$SECRET_ID" + url="${url}&Timestamp=$(date +%s)" + url="${url}&Version=2018-08-08" + url="${url}&domain=$DOMAIN" + url="${url}&recordId=$RECORD_ID" + sign=$(GetSignature "$url") + Warn Delete record $RECORD_ID ... + resp=$(curl -sSL -XGET "https://$url&Signature=$sign" | jq -eM .) + [ '0' != "$(echo $resp | jq -crM .code)" ] && echo "$resp" && exit 1 + return 0 +} + +function Main { + [ -e "$PID_FILE" ] && Error Pid file $PID_FILE already exists, quit! + echo $$ > $PID_FILE + ListRecord + [ -z "$RECORD_ID" ] && CreateRecord + [ -n "$RECORD_ID" ] && ModifyRecord + END=1 +} + +# Start here +Main + diff --git a/letsencrypt/Dockerfile b/letsencrypt/Dockerfile new file mode 100644 index 0000000..f4ae595 --- /dev/null +++ b/letsencrypt/Dockerfile @@ -0,0 +1,10 @@ +ARG ARCH +FROM harbor.colben.cn/general/alpine$ARCH +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /opt/ +RUN apk update \ + && apk add --no-cache certbot openssl jq \ + && mkdir -p /etc/letsencrypt /var/log/letsencrypt \ + && rm -rf /var/cache/apk/* +CMD ["/opt/ccmd"] + diff --git a/letsencrypt/README.md b/letsencrypt/README.md new file mode 100644 index 0000000..3c2c558 --- /dev/null +++ b/letsencrypt/README.md @@ -0,0 +1,17 @@ +# 构建 letsencrypt 镜像 + +## 定制 +- 安装 certbot 和 openssl +- 不支持通配域名 + +## 外挂目录和文件 +- /etc/letsencrypt: letsencrypt 数据目录 +- /var/log/letsencrypt: letsencrypt 日志目录 +- /etc/letsencrypt/manual-hook.sh: 手动获取证书时用到的钩子脚本 + +## 引入环境变量 +- DOMAINS: 待申请 ssl 证书的域名，多个域名用逗号间隔 + +## 案例 1 +- [Demo/SingleNode/](/Demo/SingleNode/): 部署 letsencrypt + diff --git a/letsencrypt/letsencrypt.sh b/letsencrypt/letsencrypt.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/letsencrypt/letsencrypt.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/logstash6/ADD/ccmd b/logstash6/ADD/ccmd new file mode 100755 index 0000000..79d91ea --- /dev/null +++ b/logstash6/ADD/ccmd @@ -0,0 +1,84 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /opt/logstash/config # +# - /opt/logstash/data # +# - /opt/logstash/logs # +# - /opt/logstash/offline-plugins # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f java && Print killing java ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **root user** + **/opt/logstash/{config,data,logs,offline-plugins} mounted from host** + ' +} + +function RestoreConf { + if [ -z "$(ls config/)" ]; then + Print Restore default config files and quit ... + tar zxf config.tgz + GOT_SIGTERM=1 + exit 0 + fi +} + +function InstallPlugin { + for f in $(ls -d offline-plugins/*.zip 2>/dev/null); do + Print Install plugins from offline file: $f ... + ./bin/logstash-plugin install file://$f + mv $f $f.installed + done +} + +function StartProc { + Print Start logstash ... + ./bin/logstash \ + --path.data /opt/logstash/data \ + --path.logs /opt/logstash/logs \ + --path.settings /opt/logstash/config \ + &>> logs/logstash.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /opt/logstash + Usage + RestoreConf + InstallPlugin + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/logstash6/Demo/SingleNode/README.md b/logstash6/Demo/SingleNode/README.md new file mode 100644 index 0000000..fb248ef --- /dev/null +++ b/logstash6/Demo/SingleNode/README.md @@ -0,0 +1,21 @@ +# 部署 logstash6 + +- 为域名 x1.xx.com 和 x2.xx.com 申请 ssl 证书，并在每月的 31 号晚上十一点更新一次 +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /dev/null + INIT_FLAG=1 + fi +} + +function StartProc { + local sql_file= + local sql_files= + Print Start mysql ... + mysqld -u mysql & + PIDS="$PIDS $!" + while sleep 1; do + [ -e $SOCK_FILE ] && break || echo -n . + [ ! -e /proc/$! ] && echo && Print unexpected error! && exit + done + echo + if [ -n "$INIT_FLAG" ]; then + Print Secure database ... + mysql_secure_installation <<< "$(echo -e '\nn\nn\n\n\n\n\n')" > /dev/null + mysql -e "CREATE USER docker@localhost IDENTIFIED BY 'China_19\$(10)!'" + mysql -e "GRANT SHUTDOWN ON *.* TO docker@localhost" + if sql_files="$(ls $DATA_DIR/init_sql/*.sql 2>/dev/null)"; then + Print Import the sql files ... + for sql_file in $sql_files; do + Print Importing $sql_file ... + mysql < $sql_file + done + Print Imported all sql files successfully. + fi + fi + Print MySQL is ready for connections. +} + +function Main { + local pid= + Init + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/mariadb/Demo/SingleNode/README.md b/mariadb/Demo/SingleNode/README.md new file mode 100644 index 0000000..775d133 --- /dev/null +++ b/mariadb/Demo/SingleNode/README.md @@ -0,0 +1,15 @@ +# 部署 mariadb 单点 + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /etc/my.cnf \ + && echo -e '[client]\n\ +socket = /run/mysqld/mysqld.sock\n\ +\n\ +[server]\n\ +datadir = /var/lib/mysql\n\ +socket = /run/mysqld/mysqld.sock\n\ +pid-file = /run/mysqld/mysqld.pid\n\ +log-error = /var/log/mysql/error.log\n\ +character-set-server = utf8mb4\n\ +default-storage-engine = innodb\n\ +slow-query-log = TRUE\n\ +slow-query-log-file = /var/log/mysql/slow.log\n\ +lower-case-table-names = 1\n\ +\n\ +[mysqladmin]\n\ +user = docker\n\ +password = China_19$(10)!\n\ +' > /etc/mysql/my.cnf \ + && sed -i 's/stty/#stty/' /usr/bin/mysql_secure_installation \ + && mkdir -p /var/log/mysql /var/lib/mysql-bin /run/mysqld \ + && chown -R mysql.mysql /var/log/mysql /var/lib/mysql-bin /run/mysqld +CMD ["/opt/ccmd"] + diff --git a/mariadb/README.md b/mariadb/README.md new file mode 100644 index 0000000..0086135 --- /dev/null +++ b/mariadb/README.md @@ -0,0 +1,19 @@ +# 构建 mariadb 镜像 + +## 定制 +- 安装 mariadb +- 固定一些常用配置 +- 第一次启动 mysql 时，会执行如下操作 + - 初始化数据目录后 + - 自动创建一个只有 shutdown 权限的普通用户，该用户用于优雅停止 mysql，__不要修改该用户任何信息__ + - 自动执行 {mysql-log}/init_sql/ 下的 xxxx.sql 文件 + +## 外挂目录和文件 +- /etc/my.cnf: mysql 配置文件 +- /var/lib/mysql: mysql 数据目录 +- /var/lib/mysql-bin: mysql binlog 目录 +- /var/log/mysql: mysql 日志目录 + +## 案例 +- [Demo/SingleNode/](Demo/SingleNode/): 部署 mariadb 单点 + diff --git a/mariadb/mariadb.sh b/mariadb/mariadb.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/mariadb/mariadb.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/mongodb/ADD/ccmd b/mongodb/ADD/ccmd new file mode 100755 index 0000000..ef9b04f --- /dev/null +++ b/mongodb/ADD/ccmd @@ -0,0 +1,75 @@ +#!/bin/bash + +################################################## +# ENV # +# - MONGO_OPTS # +# Mount dir # +# - LOG_DIR # +# - DATA_DIR # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= +MONGO_OPTS="${MONGO_OPTS:---bind_ip_all}" +LOG_DIR='/var/log/mongo' +DATA_DIR='/var/lib/mongo' + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + Print killing mongod ... + mongod --dbpath $DATA_DIR --shutdown || true + while :; do + pkill -f mongod && Print killing mongod ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Init { + Print Init mongod ... + chown -R mongod.mongod $LOG_DIR $DATA_DIR + rm -f /tmp/mongodb-27017.sock +} + +function StartProc { + Print Start mongodb ... + su - mongod -c "mongod \ + --port 27017 \ + --dbpath $DATA_DIR \ + --unixSocketPrefix /tmp \ + --logpath $LOG_DIR/mongod.log \ + --logappend \ + $MONGO_OPTS" & + PIDS="$PIDS $!" + while sleep 1; do + [ -e /tmp/mongodb-27017.sock ] && break || echo -n . + [ ! -e /proc/$! ] && echo && Print unexpected error! && exit + done + echo && Print Mongodb is ready for connections. +} + +function Main { + local pid= + Init + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/mongodb/Demo/SingleNode/README.md b/mongodb/Demo/SingleNode/README.md new file mode 100644 index 0000000..bb16130 --- /dev/null +++ b/mongodb/Demo/SingleNode/README.md @@ -0,0 +1,15 @@ +# 部署 mongodb 单点 + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /etc/yum.repos.d/mongodb.repo \ + && dnf makecache \ + && dnf install -y mongodb-org \ + && usermod -s /bin/bash mongod \ + && mv /etc/mongod.conf /etc/mongod.conf.origin \ + && mv /var/log/mongodb /var/log/mongo \ + && rm -f /var/log/mongo/mongod.log \ + && rm -rf /var/cache/dnf +CMD ["/opt/ccmd"] + diff --git a/mongodb/README.md b/mongodb/README.md new file mode 100644 index 0000000..ba08bc2 --- /dev/null +++ b/mongodb/README.md @@ -0,0 +1,14 @@ +# 构建 mongodb 镜像 + +## 定制 +- 安装 mongodb +- 启动时指定端口、数据目录和日志目录等参数，覆盖配置文件 + +## 外挂目录和文件 +- /etc/mongod.conf: mongodb 配置文件 +- /var/lib/mongo: mongodb 数据目录 +- /var/log/mongo: mongodb 日志目录 + +## 案例 +- [Demo/SingleNode/](Demo/SingleNode/): 部署 mongodb 单节点 + diff --git a/mongodb/mongodb.sh b/mongodb/mongodb.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/mongodb/mongodb.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/mysql/ADD/ccmd b/mysql/ADD/ccmd new file mode 100755 index 0000000..8c8a89e --- /dev/null +++ b/mysql/ADD/ccmd @@ -0,0 +1,218 @@ +#!/bin/bash + +################################################## +# EVN # +# - SERVER_ID # +# - GROUP_REPLICATION # +# - GROUP_REPLICATION_LOCAL_ADDRESS # +# - GROUP_REPLICATION_GROUP_SEEDS # +# - EXTRA_SCRIPTS # +# Mount file # +# - /etc/my.cnf # +# Mount dir # +# - LOG_DIR # +# - DATA_DIR # +# - BINLOG_DIR # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= +LOG_DIR='/var/log/mysql' +DATA_DIR='/var/lib/mysql' +BINLOG_DIR='/var/lib/mysql-bin' +SOCK_FILE='/run/mysqld/mysqld.sock' +PID_FILE='/run/mysqld/mysqld.pid' +INIT_FLAG= +GROUP_REPLICATION="${GROUP_REPLICATION:+1}" +EXTRA_SCRIPTS="${EXTRA_SCRIPTS:+1}" + +if [ -n "$GROUP_REPLICATION" ]; then + SERVER_ID=${SERVER_ID:?} + GROUP_REPILCATION_GROUP_NAME="${GROUP_REPILCATION_GROUP_NAME:-aaaabbbb-7777-8888-9999-ccccddddeeee}" + GROUP_REPLICATION_LOCAL_ADDRESS="${GROUP_REPLICATION_LOCAL_ADDRESS}" + GROUP_REPLICATION_GROUP_SEEDS="${GROUP_REPLICATION_GROUP_SEEDS}" + BOOTSTRAP_GROUP=1 +fi + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + Print killing mysqld ... + mysqladmin shutdown || true + while :; do + pkill -f mysqld && Print killing mysqld ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function ProbeSeeds { + local all_seeds=" ${GROUP_REPLICATION_GROUP_SEEDS//,/ } " + local other_seeds="${all_seeds/ $GROUP_REPLICATION_LOCAL_ADDRESS / }" + local seed= seed_return=0 + [ "$all_seeds" = "$other_seeds" ] \ + && Print Not found local_address in group_seeds! \ + && exit 1 + Print Probe connection to other seeds ... + for seed in $other_seeds; do + echo -n "Connecting $seed ... " + curl -s --connect-timeout 8 ftp://$seed || seed_return=$? + [ 6 = "$seed_return" ] && echo failed to resolve host:"${seed%:*}"! && exit 1 + [ 7 = "$seed_return" ] && echo offline. && continue + echo online. + BOOTSTRAP_GROUP= + break + done +} + +function Init { + rm -f ${SOCK_FILE}* ${PID_FILE} + chown -R mysql.mysql $LOG_DIR $BINLOG_DIR $DATA_DIR + if [ ! -d "$DATA_DIR/mysql" ]; then + Print Write essential server config to /etc/mysql/my.cnf ... + Print Init mysql db files ... + mysqld_pre_systemd + INIT_FLAG=1 + fi +} + +function InitGroupReplication { + if ! grep -i '^group[-_]replication' /etc/my.cnf; then + Print Write advisable group replication config to /etc/my.cnf ... + grep -i '^binlog[-_]expire[-_]logs[-_]seconds' /etc/my.cnf \ + || echo 'binlog-expire-logs-seconds = 172800' >> /etc/my.cnf + echo 'group-replication-consistency = BEFORE_ON_PRIMARY_FAILOVER +group-replication-member-expel-timeout = 2 +group-replication-unreachable-majority-timeout = 2 +group-replication-autorejoin-tries = 0 +group-replication-exit-state-action = OFFLINE_MODE +' >> /etc/my.cnf + fi + if ! grep -i '^group[-_]replication' /etc/mysql/my.cnf; then + Print Write essential group replication config to /etc/mysql/my.cnf ... + cat >> /etc/mysql/my.cnf <<-EOF +server-id = $SERVER_ID +gtid-mode = ON +enforce-gtid-consistency = TRUE +binlog-format = ROW +binlog-checksum = NONE +log-bin = /var/lib/mysql-bin/master +relay-log = /var/lib/mysql-bin/slave +log-slave-updates = TRUE +relay-log-recovery = TRUE +slave-parallel-type = LOGICAL_CLOCK +slave-preserve-commit-order = ON +disabled-storage-engines = "MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" +plugin-load-add = "group_replication.so;mysql_clone.so" +group-replication-group-name = "$GROUP_REPILCATION_GROUP_NAME" +group-replication-local-address = "$GROUP_REPLICATION_LOCAL_ADDRESS" +group-replication-group-seeds = "$GROUP_REPLICATION_GROUP_SEEDS" +group-replication-bootstrap-group = OFF +group-replication-start-on-boot = OFF +EOF + fi +} + +function ImportInitSql { + local sql_file= sql_files= + mysql -e "CREATE USER docker@localhost IDENTIFIED BY 'China_19\$(10)!'" + mysql -e "GRANT SHUTDOWN ON *.* TO docker@localhost" + if sql_files="$(ls $LOG_DIR/init_sql/*.sql 2>/dev/null)"; then + Print Import the sql files ... + for sql_file in $sql_files; do + Print Importing $sql_file ... + mysql < $sql_file + done + Print Imported all sql files successfully. + fi +} + +function StartGroupReplication { + if [ -n "$BOOTSTRAP_GROUP" ]; then + Print Bootstrap new group replication ... + mysql -e " + SET GLOBAL group_replication_bootstrap_group=ON; + START GROUP_REPLICATION; + SET GLOBAL group_replication_bootstrap_group=OFF; + " + else + Print Join a running group replication ... + mysql -e "START GROUP_REPLICATION;" + fi +} + +function CreateGroupReplicationChannel { + Print Create user and channel of group replication ... + mysql -e "SET SQL_LOG_BIN=0; + CREATE USER rpl@'%' IDENTIFIED BY 'Rpl_1234'; + GRANT REPLICATION SLAVE ON *.* TO rpl@'%'; + GRANT BACKUP_ADMIN ON *.* TO rpl@'%'; + FLUSH PRIVILEGES; + SET SQL_LOG_BIN=1; + CHANGE MASTER TO + MASTER_USER='rpl', + MASTER_PASSWORD='Rpl_1234' + FOR CHANNEL 'group_replication_recovery'; + " +} + +function StartExtraScripts { + Print Start extra scripts ... + while sleep 2; do + for script in $(find $LOG_DIR/extra_scripts/ -type f -executable \ + 2>/dev/null || true); do + $script & + done + done & +} + +function StartProc { + Print Start mysql ... + mysqld -u mysql & + PIDS="$PIDS $!" + while sleep 1; do + [ -e $SOCK_FILE ] && break || echo -n . + [ ! -e /proc/$! ] && echo && Print unexpected error! && exit + done + echo + if [ -n "$INIT_FLAG" ]; then + ImportInitSql + if [ -n "$GROUP_REPLICATION" ]; then + CreateGroupReplicationChannel + fi + fi + if [ -n "$GROUP_REPLICATION" ]; then + StartGroupReplication + fi + if [ -n "$EXTRA_SCRIPTS" ]; then + StartExtraScripts + fi + Print MySQL is ready for connections. +} + +function Main { + local pid= + [ -n "$GROUP_REPLICATION" ] && ProbeSeeds + Init + [ -n "$GROUP_REPLICATION" ] && InitGroupReplication + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/mysql/Demo/MGR/README.md b/mysql/Demo/MGR/README.md new file mode 100644 index 0000000..403b19a --- /dev/null +++ b/mysql/Demo/MGR/README.md @@ -0,0 +1,36 @@ +# 部署 MySQL Group Replication + +- 按实际环境修改 + - docker-compose.yml + - mysql{1..3}/my.cnf + +- 创建目录 + ``` + grep '\'" + interval 10 + weight 0 + fall 2 + rise 2 +} + +vrrp_instance VI_1 { + state BACKUP + virtual_router_id 13 + priority 150 # 在另一台服务器中，这里配置100 + advert_int 2 + nopreempt + interface eth0 # 这里的 eth0 是服务器的网卡名 + track_script { + chk_mysql + } + authentication { + auth_type PASS + auth_pass El_en_mysql_1234 + } + virtual_ipaddress { + 虚拟IP/掩码 dev eth0 # 这里的eth0是服务器的网卡名 + } +} + diff --git a/mysql/Demo/TwoMasterNodes/mysql/log/init_sql/0.elcs-encrypt-user.sql b/mysql/Demo/TwoMasterNodes/mysql/log/init_sql/0.elcs-encrypt-user.sql new file mode 100644 index 0000000..b53c418 --- /dev/null +++ b/mysql/Demo/TwoMasterNodes/mysql/log/init_sql/0.elcs-encrypt-user.sql @@ -0,0 +1,4 @@ +CREATE USER replicator@'%' IDENTIFIED BY 'China_19$(10)!'; +GRANT REPLICATION SLAVE ON *.* TO replicator@'%'; +FLUSH PRIVILEGES; + diff --git a/mysql/Demo/TwoMasterNodes/mysql/my.cnf b/mysql/Demo/TwoMasterNodes/mysql/my.cnf new file mode 100644 index 0000000..170f2ce --- /dev/null +++ b/mysql/Demo/TwoMasterNodes/mysql/my.cnf @@ -0,0 +1,30 @@ +[mysqld] +mysqlx = OFF +skip-name-resolve = 1 +max-user-connections = 600 + +## this should be less than 80% of total memory +innodb-buffer-pool-size = xxxxG + +## this should be half of "innodb-buffer-pool-size" +innodb-buffer-pool-instances = xxxx + +# binlog +## this should be an unsigned tinyint and different with the other mysql server. +server-id = xxxx +log-bin = /var/lib/mysql-bin/master +binlog-format = ROW +binlog-expire-logs-seconds = 172800 +gtid-mode = ON +enforce-gtid-consistency = TRUE + +# replication +replicate-wild-ignore-table = information_schema.% +replicate-wild-ignore-table = mysql.% +replicate-wild-ignore-table = performance_schema.% +replicate-wild-ignore-table = sys.% +slave-parallel-workers = 2 +log-slave-updates = FALSE +relay-log = /var/lib/mysql-bin/slave +relay-log-recovery = TRUE + diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 0000000..2852e9f --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1,63 @@ +ARG ARCH +FROM harbor.colben.cn/general/rocky$ARCH:8 +MAINTAINER Colben colbenlee@gmail.com +ARG ARCH +ADD --chown=root:root /ADD/ /opt/ +RUN echo -e "[mysql-connectors-community]\n\ +name=MySQL Connectors Community\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-connectors-community-el8$ARCH/\n\ +enabled=1\n\ +gpgcheck=0\n\ +\n\ +[mysql-tools-community]\n\ +name=MySQL Tools Community\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-tools-community-el8$ARCH/\n\ +enabled=1\n\ +gpgcheck=0\n\ +\n\ +[mysql80-community]\n\ +name=MySQL 8.0 Community Server\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-8.0-community-el8${ARCH:--x86_64}/\n\ +enabled=1\n\ +gpgcheck=0\n\ +" > /etc/yum.repos.d/mysql-8.0.repo \ + && dnf makecache \ + && dnf install mysql-community-server -y --disablerepo=AppStream \ + && rm -rf /usr/sbin/mysqld-debug \ + /var/cache/yum \ + /var/lib/yum \ + /var/lib/rpm \ + /var/log/* \ + /etc/my.cnf.d \ + && mkdir -p /var/log/mysql \ + /var/lib/mysql-bin \ + /etc/mysql \ + && chown -R mysql.mysql \ + /var/log/mysql \ + /var/lib/mysql-bin \ + && sed -i -e 's,--initialize,&-insecure,g' \ + -e 's,/usr/sbin/mysqld ,&--defaults-file=/etc/mysql/my.cnf ,g' \ + /usr/bin/mysqld_pre_systemd \ + && echo -e '[mysqld]\nmysqlx = OFF\n' > /etc/my.cnf \ + && echo -e '[client]\n\ +socket = /run/mysqld/mysqld.sock\n\ +\n\ +[mysqld]\n\ +datadir = /var/lib/mysql\n\ +socket = /run/mysqld/mysqld.sock\n\ +pid-file = /run/mysqld/mysqld.pid\n\ +log-timestamps = SYSTEM\n\ +log-error = /var/log/mysql/error.log\n\ +character-set-server = utf8mb4\n\ +default-storage-engine = innodb\n\ +slow-query-log = TRUE\n\ +slow-query-log-file = /var/log/mysql/slow.log\n\ +default-authentication-plugin = mysql_native_password\n\ +lower-case-table-names = 1\n\ +\n\ +[mysqladmin]\n\ +user = docker\n\ +password = China_19$(10)!\n\ +' > /etc/mysql/my.cnf +CMD ["/opt/ccmd"] + diff --git a/mysql/Dockerfile-centos7 b/mysql/Dockerfile-centos7 new file mode 100644 index 0000000..ae037fa --- /dev/null +++ b/mysql/Dockerfile-centos7 @@ -0,0 +1,63 @@ +ARG ARCH +FROM harbor.colben.cn/general/centos$ARCH:7 +MAINTAINER Colben colbenlee@gmail.com +ARG ARCH +ADD --chown=root:root /ADD/ /opt/ +RUN echo -e "[mysql-connectors-community]\n\ +name=MySQL Connectors Community\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-connectors-community-el7$ARCH/\n\ +enabled=1\n\ +gpgcheck=0\n\ +\n\ +[mysql-tools-community]\n\ +name=MySQL Tools Community\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-tools-community-el7$ARCH/\n\ +enabled=1\n\ +gpgcheck=0\n\ +\n\ +[mysql80-community]\n\ +name=MySQL 8.0 Community Server\n\ +baseurl=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql-8.0-community-el7${ARCH:--x86_64}/\n\ +enabled=1\n\ +gpgcheck=0\n\ +" > /etc/yum.repos.d/mysql-8.0.repo \ + && yum makecache fast \ + && yum install mysql-community-server -y \ + && rm -rf /usr/sbin/mysqld-debug \ + /var/cache/yum \ + /var/lib/yum \ + /var/lib/rpm \ + /var/log/* \ + /etc/my.cnf.d \ + && mkdir -p /var/log/mysql \ + /var/lib/mysql-bin \ + /etc/mysql \ + && chown -R mysql.mysql \ + /var/log/mysql \ + /var/lib/mysql-bin \ + && sed -i -e 's,--initialize,&-insecure,g' \ + -e 's,/usr/sbin/mysqld ,&--defaults-file=/etc/mysql/my.cnf ,g' \ + /usr/bin/mysqld_pre_systemd \ + && echo -e '[mysqld]\nmysqlx = OFF\n' > /etc/my.cnf \ + && echo -e '[client]\n\ +socket = /run/mysqld/mysqld.sock\n\ +\n\ +[mysqld]\n\ +datadir = /var/lib/mysql\n\ +socket = /run/mysqld/mysqld.sock\n\ +pid-file = /run/mysqld/mysqld.pid\n\ +log-timestamps = SYSTEM\n\ +log-error = /var/log/mysql/error.log\n\ +character-set-server = utf8mb4\n\ +default-storage-engine = innodb\n\ +slow-query-log = TRUE\n\ +slow-query-log-file = /var/log/mysql/slow.log\n\ +default-authentication-plugin = mysql_native_password\n\ +lower-case-table-names = 1\n\ +\n\ +[mysqladmin]\n\ +user = docker\n\ +password = China_19$(10)!\n\ +' > /etc/mysql/my.cnf +CMD ["/opt/ccmd"] + diff --git a/mysql/README.md b/mysql/README.md new file mode 100644 index 0000000..0f0223d --- /dev/null +++ b/mysql/README.md @@ -0,0 +1,29 @@ +# 构建 mysql 镜像 + +## 定制 +- 安装 mysql 8.0 +- 固定一些常用配置 +- 第一次启动 mysql 时，会执行如下操作 + - 初始化数据目录 + - 自动创建一个只有 shutdown 权限的普通用户，该用户用于优雅停止 mysql，__不要修改该用户任何信息__ + - 自动执行 {mysql-log}/init_sql/ 下的 xxxx.sql 文件 +- 每两秒检查 {mysql-log}/extra_scripts/ 下的可执行文件，发现后立即执行 + +## 外挂目录和文件 +- /etc/my.cnf: mysql 配置文件 +- /var/lib/mysql: mysql 数据目录 +- /var/lib/mysql-bin: mysql binlog 目录 +- /var/log/mysql: mysql 日志目录 + +## 引入环境变量 +- SERVER_ID: mysql server id +- GROUP_REPLICATION: 是否开启组复制 +- GROUP_REPLICATION_LOCAL_ADDRESS: 指定本地地址 +- GROUP_REPLICATION_GROUP_SEEDS: 指定组复制的全部节点，用逗号间隔 +- EXTRA_SCRIPTS: 是否执行 {mysql-log}/extra_scripts/ 下的可执行文件，默认为空，不执行 + +## 案例 +- [Demo/SingleNode/](Demo/SingleNode/): 单节点 +- [Demo/TwoMasterNodes/](Demo/TwoMasterNodes/): 两节点互为主从+高可用 +- [Demo/MGR/](Demo/MGR/): 组复制 + diff --git a/mysql/mysql.sh b/mysql/mysql.sh new file mode 100755 index 0000000..773bc26 --- /dev/null +++ b/mysql/mysql.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:8" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/nginx-php/ADD/ccmd b/nginx-php/ADD/ccmd new file mode 100755 index 0000000..0ed95f0 --- /dev/null +++ b/nginx-php/ADD/ccmd @@ -0,0 +1,84 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /etc/nginx/stream.d # +# - /etc/nginx/http.d # +# - /var/lib/nginx/html # +# - /var/log/nginx # +# - /var/log/php7 # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + local running + while running= ; do + pkill -f php-fpm7 && running=1 && Print killing php-fpm7 ... + pkill -f sleep && running=1 && Print killing sleep ... + pkill -f nginx && running=1 && Print killing nginx ... + [ -z "$running" ] && break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function SideCar { + local day= last_day=$(date +%d) + local md5= last_md5=$(find /etc/nginx/ -type f -name "*.conf" \ + | xargs -I ^ md5sum ^ | md5sum) + while sleep 10; do + day=$(date +%d) \ + && [ "$day" != "$last_day" ] \ + && last_day=$day \ + && find /var/log/nginx/ -type f -name "*.log" \ + | xargs -I ^ mv -f ^ ^.$(date +%F -d yesterday) \ + && nginx -s reopen + md5=$(find /etc/nginx/ -type f -name "*.conf" | xargs -I ^ md5sum ^ \ + | md5sum) \ + && [ "$md5" != "$last_md5" ] \ + && last_md5=$md5 \ + && nginx -tq \ + && Print Reload nginx conf ... \ + && nginx -s reload + done +} + +function StartProc { + Print Start php ... + php-fpm7 -F -y /etc/php7/php-fpm.conf & + PIDS="$PIDS $!" + Print Start nginx ... + nginx -g 'daemon off;pid /run/nginx/nginx.pid;' & + PIDS="$PIDS $!" + Print Start nginx sidecar ... + SideCar & + PIDS="$PIDS $!" +} + +function Main { + local pid= + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/nginx-php/Demo/SingleNode/README.md b/nginx-php/Demo/SingleNode/README.md new file mode 100644 index 0000000..29dbe9c --- /dev/null +++ b/nginx-php/Demo/SingleNode/README.md @@ -0,0 +1,17 @@ +# 部署单节点 nginx-php + +- 根据实际环境修改 + - docker-compose.yml + - nginx/http.d/80.conf + +- 创建目录 + ``` + grep '\'" + interval 10 + weight 0 + fall 2 + rise 2 +} + +vrrp_instance VI_1 { + state BACKUP + virtual_router_id 14 + priority 150 # 在另一台服务器中，这里配置100 + advert_int 2 + nopreempt + interface eth0 # 这里的 eth0 是服务器的网卡名 + track_script { + chk_nginx + } + authentication { + auth_type PASS + auth_pass El_en_nginx_1234 + } + virtual_ipaddress { + 虚拟IP/掩码 dev eth0 # 这里的eth0是服务器的网卡名 + } +} + diff --git a/nginx/Demo/TwoNodes/nginx/http.d/80.conf b/nginx/Demo/TwoNodes/nginx/http.d/80.conf new file mode 100644 index 0000000..46b4e36 --- /dev/null +++ b/nginx/Demo/TwoNodes/nginx/http.d/80.conf @@ -0,0 +1,5 @@ +server { + listen 80; + location / {} +} + diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 0000000..8025100 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,17 @@ +ARG ARCH +FROM harbor.colben.cn/general/alpine$ARCH +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /opt/ +RUN apk update \ + && apk add --no-cache nginx nginx-mod-stream \ + && sed -i \ + -e '1a\\n# Added by Dockerfile' \ + -e '1adaemon off;' \ + -e '1apid /run/nginx/nginx.pid;' \ + -e '/^user /,/^worker_processes /d' \ + -e '/^#include /s/^#//' \ + /etc/nginx/nginx.conf \ + && rm -rf /var/cache/apk/* /etc/nginx/http.d/* \ + && chown nginx.nginx /run/nginx +CMD ["/opt/ccmd"] + diff --git a/nginx/README.md b/nginx/README.md new file mode 100644 index 0000000..c838605 --- /dev/null +++ b/nginx/README.md @@ -0,0 +1,20 @@ +# 构建 nginx 镜像 + +## 定制 +- 安装 nginx +- 固定一些常用配置 +- 每 10 秒扫描一次配置文件，有变更会立即 reload + +## 外挂目录和文件 +- /etc/nginx/stream.d: nginx stream 配置文件 +- /etc/nginx/http.d: nginx http 配置文件 +- /var/lib/nginx/html: nginx 前端文件存放目录 +- /var/log/nginx: nginx 日志目录 + +## 引入环境变量 +- GLOBAL_DIRECTIVES: 一般用不到 + +## 案例 +- [Demo/SingleNode/](Demo/SingleNode/): 单节点 +- [Demo/TwoNodes/](Demo/TwoNodes/): 两节点+高可用 + diff --git a/nginx/nginx.sh b/nginx/nginx.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/nginx/nginx.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/ops/Dockerfile b/ops/Dockerfile new file mode 100644 index 0000000..217da4b --- /dev/null +++ b/ops/Dockerfile @@ -0,0 +1,53 @@ +ARG ARCH +FROM harbor.colben.cn/general/jdk$ARCH:8 +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /opt/ +ENV MAVEN_HOME=/opt/maven +ENV PATH=${MAVEN_HOME}/bin:$PATH +RUN tdnf makecache \ + && tdnf -y install git openssh-clients \ + && mkdir -m 0600 /root/.ssh \ + && echo '-----BEGIN OPENSSH PRIVATE KEY-----\n\ +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\n\ +NhAAAAAwEAAQAAAYEAtOCNZp6gKBU737xzMzSSpThDxDBCx9D5hQ3ROeW1eps+9xBNQ6nB\n\ +xWXf5lJnMbQPGLYpAyCfhHtnSv7lUWCdfSP68TtzJPwfqJ5forx/KdlRs0KMLAisjvnaDQ\n\ +eqIAwWgXyt0Asdm8sYR7SHbBf9n4MyBX/qhBimI4eHsxLNdXibMUMXa0G5pztNsLh91K+i\n\ +QWyz4ul5xW9iTFOyqRtljnX9qS6atk3dj5fTiQRMTlmqAeV5YM7mBaskhsZY4/ugBS7M07\n\ +gRA1wxzDZ5zNR+u0XoSVcuyxefNOkGHxHk1qPZsxYli/OU2va/63plIzzCbubRK/3+JMLc\n\ +JsLL7tuUGoQVCx/Fon0Td6np9YDjhliaYUkQhFiOZbIh8qTA6ATUmTRbAZ26kAF5cVqCdL\n\ +tETkl4MYXhBeMRRURL/0KuEFasZDgIyfyeswBHUPG8/4jcytQePB5Vsmqb3/DLiz2W+baE\n\ +jPJoTbty74sHzlpeLnj+KMHhqowrNn/43jCq9IAPAAAFkNIv+orSL/qKAAAAB3NzaC1yc2\n\ +EAAAGBALTgjWaeoCgVO9+8czM0kqU4Q8QwQsfQ+YUN0TnltXqbPvcQTUOpwcVl3+ZSZzG0\n\ +Dxi2KQMgn4R7Z0r+5VFgnX0j+vE7cyT8H6ieX6K8fynZUbNCjCwIrI752g0HqiAMFoF8rd\n\ +ALHZvLGEe0h2wX/Z+DMgV/6oQYpiOHh7MSzXV4mzFDF2tBuac7TbC4fdSvokFss+LpecVv\n\ +YkxTsqkbZY51/akumrZN3Y+X04kETE5ZqgHleWDO5gWrJIbGWOP7oAUuzNO4EQNcMcw2ec\n\ +zUfrtF6ElXLssXnzTpBh8R5Naj2bMWJYvzlNr2v+t6ZSM8wm7m0Sv9/iTC3CbCy+7blBqE\n\ +FQsfxaJ9E3ep6fWA44ZYmmFJEIRYjmWyIfKkwOgE1Jk0WwGdupABeXFagnS7RE5JeDGF4Q\n\ +XjEUVES/9CrhBWrGQ4CMn8nrMAR1DxvP+I3MrUHjweVbJqm9/wy4s9lvm2hIzyaE27cu+L\n\ +B85aXi54/ijB4aqMKzZ/+N4wqvSADwAAAAMBAAEAAAGAfcfhzMjmSrlpZ47e9W7Lw4U66s\n\ +U8y0MxISuYoZByAQ7QXHLDqBf2ndTYPIx1IoU6Mk8KehhHlZNTIz3aLhrnqcxJh1N2IPQK\n\ +9/EaREqci3eKbwQKAd0OUmbBEWqRDbQaOnV/UTtJnbs/6S1LNwn/3tsW/+gSJ0YU0oQ/A3\n\ +JB2jzCfyFOudF8zyRYRFondq2kQabb+SGvAnMQgk954Esi9kxq1ymHgNrpBh/ohFGu1pVX\n\ +ufXf0EhQgQSgY/Cug+vIEkenCjXX9yuVR69MNpFhY9xdZp0PKR52SIHOvPN5WykI86IWkP\n\ +DQnF201lLtwAtLl9XPIDGOADQup7bxAT8/fmYD6wiQxDmj7qwlRhmdQ5/9GbXV76z3IItW\n\ +1kn73vprUN54dw68ubLMbn4c/FyJcHlBwSHyAbjUZaJX82vJAxZiB0HHK81BGg8sv12hZf\n\ +PkAfo3PFS01hyBRrao3ewdQ/ug6KE9sR3WmQA+MN7pr232tEWixfNIGtJl/g/Er7pRAAAA\n\ +wFrEd6Jni0q+hU0q+I2l/IpW7oheapDbloXF3ES6bOZnW9YObrjgpojJmKxH75/tiQnYVm\n\ +hyofU4xTKXW5xwxQBx6oQH6QlLP7biZsbi64F/S1uqqp8nEafJukUglK+3arTNDu7bLNNI\n\ +3FTvTNgNjef37kTyg7zmZsODgEhX13mkrQ9kyOnXnRWBmNNElI32dW2jhLhLX9SR75u5+C\n\ +6bOekyNJ2L0MR4HgZE1Iorkgy/NfUgc+9Gy45uyX2M0lhuTgAAAMEA2Rp0og8QU6fNLDoi\n\ +Bn8gjn/sEmJvDI+raaeQmFkrzASen8gOMhRCtcAjr4mwZkEdegIPmxCJ5I/XarsQsYxN1J\n\ +ychw8QXklF1i7GArjIOWwmnBFYiu3Qx9f34qwxUJcj1D+klgmQ5VwCV8yM3fjoxOPVIg2N\n\ +1azzTR7PUnzEQT/TRZiMGLEZiUiJAPFGeTlqEHRmRHhVxFjWR29GGWeheEKdZxCCy6ze/3\n\ +8DBGI3qDl8js0gSoDm2aFnmXZkd19rAAAAwQDVSJDwqexEggEAcRHUY9xPAZaX5VBo2+wS\n\ +CEqENDAflVUiuWEcYNh96GC8dscfo2ZLrAoBicIeSAiLJrko+U3buttctUtVBKD41CEGqW\n\ +UpAt9GV2nAWhvxq4WxXz48vf4tzBerIvzlDpujaON0K8KyfSNg4HCOnwkA0LbH6sVtQ0kS\n\ +vO4kAgIG2wmgGfvBsySjTuqQe4m/+mKXNt7bSc1gJhOikrEQQRy2dE04+3NGw5bYLpLG4l\n\ +QLpw2nbIGP/u0AAAAUcm9vdEB2cG4tMTAtMC0xNi0xMjQBAgMEBQYH\n\ +-----END OPENSSH PRIVATE KEY-----\n\ +' > /root/.ssh/id_rsa \ + && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC04I1mnqAoFTvfvHMzNJKlOEPEMELH0PmFDdE55bV6mz73EE1DqcHFZd/mUmcxtA8YtikDIJ+Ee2dK/uVRYJ19I/rxO3Mk/B+onl+ivH8p2VGzQowsCKyO+doNB6ogDBaBfK3QCx2byxhHtIdsF/2fgzIFf+qEGKYjh4ezEs11eJsxQxdrQbmnO02wuH3Ur6JBbLPi6XnFb2JMU7KpG2WOdf2pLpq2Td2Pl9OJBExOWaoB5XlgzuYFqySGxljj+6AFLszTuBEDXDHMNnnM1H67RehJVy7LF5806QYfEeTWo9mzFiWL85Ta9r/remUjPMJu5tEr/f4kwtwmwsvu25QahBULH8WifRN3qen1gOOGWJphSRCEWI5lsiHypMDoBNSZNFsBnbqQAXlxWoJ0u0ROSXgxheEF4xFFREv/Qq4QVqxkOAjJ/J6zAEdQ8bz/iNzK1B48HlWyapvf8MuLPZb5toSM8mhNu3LviwfOWl4ueP4oweGqjCs2f/jeMKr0gA8= root@ops\n\ +' > /root/.ssh/id_rsa.pub \ + && chmod 0600 /root/.ssh/id_rsa \ + && rm -rf /var/cache/tdnf + diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 0000000..3888381 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,9 @@ +# 构建运维镜像 + +## 导入文件 +- [下载 apache-maven-$VERSION-bin.tar.gz](https://archive.apache.org/dist/maven/maven-3/) + +## 定制 +- ops.sh: 构建 ops 镜像 +- 设置 maven 环境变量 + diff --git a/ops/ops.sh b/ops/ops.sh new file mode 100755 index 0000000..d05237c --- /dev/null +++ b/ops/ops.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/ops$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + Warn Preparing maven ... + cd $ROOT_DIR + rm -rf ADD && mkdir ADD + cd ADD + tar zxf $(ls /release/RUNTIME/apache-maven-*-bin.tar.gz|tail -1) + mv apache-maven-3.6.3 maven +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/photon/Dockerfile b/photon/Dockerfile new file mode 100644 index 0000000..3c7d85e --- /dev/null +++ b/photon/Dockerfile @@ -0,0 +1,10 @@ +FROM photon:4.0 +MAINTAINER Colben colbenlee@gmail.com +ADD --chown=root:root /ADD/ /etc/ +RUN tdnf makecache \ + && tdnf -y install less vim iproute2 shadow tar libstdc++ coreutils \ + && rm -rf /var/cache/tdnf \ + && echo "export PS1='\[\e[33;1;1m\][\[\e[0m\]\[\e[35;1m\]\u\[\e[0m\]\[\e[33;1;1m\]@\[\e[0m\]\[\e[31;1;1m\]docker\[\e[0m\]\[\e[32;1;1m\](\h)\[\e[0m\]\[\e[33;1;1m\]:\[\e[0m\]\[\e[32m\]\w\[\e[0m\]\[\e[33;1;1m\]]\[\e[0m\]\[\e[36m\]\$\[\e[0m\] '" >> /etc/bash.bashrc \ + && echo "export PS2='\[\e[36m\]>\[\e[0m\] '" >> /etc/bash.bashrc +ENV LANG=en_US.UTF-8 + diff --git a/photon/README.md b/photon/README.md new file mode 100644 index 0000000..4d8740f --- /dev/null +++ b/photon/README.md @@ -0,0 +1,10 @@ +# 构建 photon 镜像 + +## 导入文件 +- 本机时区 /etc/localtime + +## 定制 +- 使用 Asia/Shanghai 时区 +- 安装 less vim iproute2 shadow tar libstdc++ coreutils +- 默认语言 en_US.UTF-8 + diff --git a/photon/photon.sh b/photon/photon.sh new file mode 100755 index 0000000..2d3ef7a --- /dev/null +++ b/photon/photon.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + Warn Preparing localtime ... + cd $ROOT_DIR + cp -f /etc/localtime ADD/ +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/prometheus/ADD/ccmd b/prometheus/ADD/ccmd new file mode 100755 index 0000000..6846357 --- /dev/null +++ b/prometheus/ADD/ccmd @@ -0,0 +1,250 @@ +#!/bin/bash + +################################################## +# ENV # +# - PROMETHEUS_OPTS # +# - ALERTMANAGER_OPTS # +# - GRAFANA_OPTS # +# - LOKI_OPTS # +# Mount dir # +# - /etc/prometheus # +# - /var/log/prometheus # +# - /var/lib/prometheus # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= +LOG_DIR='/var/log/prometheus' +DATA_DIR='/var/lib/prometheus' +CONF_DIR='/etc/prometheus' + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + local running + while running= ; do + pkill -f sleep && running=1 && Print killing sleep ... + pkill -f grafana-server && running=1 && Print killing grafana-server ... + pkill -f prometheus && running=1 && Print killing prometheus ... + pkill -f alertmanager && running=1 && Print killing alertmanager ... + pkill -f loki && running=1 && Print killing loki ... + [ -z "$running" ] && break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function SideCar { + local md5= last_md5=$(find $CONF_DIR -maxdepth 1 -type f \ + -regex ".*\.yml\|.*\.tmpl" | xargs -I ^ md5sum ^ | md5sum) + while sleep 10; do + md5=$(find $CONF_DIR -maxdepth 1 -type f \ + -regex ".*\.yml\|.*\.tmpl" | xargs -I ^ md5sum ^ | md5sum) + [ "$md5" != "$last_md5" ] \ + && last_md5=$md5 \ + && Print Reload conf ... \ + && pkill -HUP -f prometheus \ + && pkill -HUP -f alertmanager + done +} + +function Init { + mkdir -p $DATA_DIR/{tsdb,alertmanager,grafana,loki} + [ -f $CONF_DIR/prometheus.yml ] || echo 'global: + scrape_interval: 15s + evaluation_interval: 15s + scrape_timeout: 10s + +alerting: + alertmanagers: + - static_configs: + - targets: + - 127.0.0.1:9093 + +rule_files: +# - node_rules.yml + +scrape_configs: +#- job_name: prometheus +# honor_labels: true +# static_configs: +# - targets: +# - 127.0.0.1:9100 +# labels: +# host: 127.0.0.1 + +#- job_name: nodes +# static_configs: +# - targets: +# - ip_1:9100 +# - ip_2:9100 +# metric_relabel_configs: +# - source_labels: [instance] +# target_label: host +# regex: "([^:]+):.+" +' > $CONF_DIR/prometheus.yml + [ -f $CONF_DIR/alertmanager.yml ] || echo 'global: + resolve_timeout: 10m + +templates: +# - xxxx.tmpl + +route: + group_by: [alertname] + group_wait: 10s + group_interval: 10s + repeat_interval: 1m + receiver: alert + +receivers: +- name: alert + +inhibit_rules: + - source_match: + severity: emergency + target_match_re: + severity: "warning|critical" + equal: [host] +' > $CONF_DIR/alertmanager.yml + [ -f $CONF_DIR/grafana.ini ] \ + || cp -af /usr/share/grafana/conf/sample.ini $CONF_DIR/grafana.ini + [ -d $CONF_DIR/provisioning ] \ + || cp -af /usr/share/grafana/conf/provisioning $CONF_DIR/provisioning + [ -f $CONF_DIR/loki.yml ] || echo " +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +ingester: + wal: + enabled: true + dir: $DATA_DIR/loki/db/wal + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h + max_chunk_age: 1h + chunk_target_size: 1048576 + chunk_retain_period: 30s + max_transfer_retries: 0 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: $DATA_DIR/loki/db/boltdb-shipper-active + cache_location: $DATA_DIR/loki/db/boltdb-shipper-cache + cache_ttl: 24h + shared_store: filesystem + filesystem: + directory: $DATA_DIR/loki/db/chunks + +compactor: + working_directory: $DATA_DIR/loki/db/boltdb-shipper-compactor + shared_store: filesystem + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + trapype: local + local: + directory: $DATA_DIR/loki/db/rules + rule_path: $DATA_DIR/loki/db/rules-temp + alertmanager_url: http://127.0.0.1:9093 + ring: + kvstore: + store: inmemory + enable_api: true +" > $CONF_DIR/loki.yml +} + +function StartProc { + Print Start alertmanager ... + alertmanager \ + --config.file=$CONF_DIR/alertmanager.yml \ + --storage.path=$DATA_DIR/alertmanager \ + --web.external-url=http://0.0.0.0:9093/alertmanager/ \ + ${ALERTMANAGER_OPTS:-} &>> $LOG_DIR/alertmanager.out & + PIDS="$PIDS $!" + sleep 2 + + Print Start prometheus ... + prometheus \ + --config.file=$CONF_DIR/prometheus.yml \ + --web.external-url=prometheus \ + --web.console.templates=/usr/share/prometheus/consoles \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --storage.tsdb.path=$DATA_DIR/tsdb \ + ${PROMETHEUS_OPTS:-} &>> $LOG_DIR/prometheus.out & + PIDS="$PIDS $!" + sleep 2 + + Print Start grafana-server ... + grafana-server \ + -homepath /usr/share/grafana \ + -config $CONF_DIR/grafana.ini \ + ${GRAFANA_OPTS:-} web &>> $LOG_DIR/grafana.out & + PIDS="$PIDS $!" + sleep 2 + + Print Start loki ... + loki \ + --config.file=$CONF_DIR/loki.yml \ + ${LOKI_OPTS:-} &>> $LOG_DIR/loki.out & + PIDS="$PIDS $!" + sleep 2 + + Print Start sidecar ... + SideCar & + PIDS="$PIDS $!" +} + +function Main { + local pid= + Init + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/prometheus/Demo/SingleNode/README.md b/prometheus/Demo/SingleNode/README.md new file mode 100644 index 0000000..5ccf916 --- /dev/null +++ b/prometheus/Demo/SingleNode/README.md @@ -0,0 +1,21 @@ +# 部署单节点 prometheus + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /etc/redis.conf + while read kv; do + [ 'cluster-enabled=yes' == "$kv" ] && clusterEnabled=1 + [[ "$kv" =~ ^requirepass= ]] && REQUIREPASS=${kv#*=} + echo "${kv/=/ }" >> /etc/redis.conf + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" + [ -e "$CLUSTER_CONFIG_FILE" -o -z "$clusterEnabled" -o -z "${MASTER_NODES[*]}" ] \ + || DEPLOY_CLUSTER=1 +} + +function DeployCluster { + Print Deploy redis cluster ... + local i= + local node= + local nodesId= + local clusterEnabled= + local redisCliCmd="redis-cli ${REQUIREPASS:+-a $REQUIREPASS --no-auth-warning}" + [ 0 -ne ${#SLAVE_NODES[@]} -a ${#SLAVE_NODES[@]} -ne ${#MASTER_NODES[@]} ] \ + && Print The num of slave nodes not equal to the num of master nodes! \ + && exit 100 + for node in ${MASTER_NODES[@]} ${SLAVE_NODES[@]}; do + for i in {0..5}; do + [ 5 -eq $i ] \ + && Print Failed to connect to $node! \ + && exit 100 + sleep 6 + clusterEnabled=$($redisCliCmd --raw -d , -h ${node/:/ -p } \ + CONFIG GET cluster-enabled) \ + || continue + [[ "$clusterEnabled" =~ ,yes$ ]] && break + Print $node not enable cluster mode! && exit 100 + done + done + Print Create redis cluster with master nodes: ${MASTER_NODES[@]} + $redisCliCmd --cluster-yes --cluster create ${MASTER_NODES[@]} + nodesId="$($redisCliCmd -s $SOCK_FILE cluster nodes | cut -d@ -f1)" + for i in ${!SLAVE_NODES[@]}; do + Print Add slave node ${SLAVE_NODES[$i]} with master: ${MASTER_NODES[$i]} + $redisCliCmd --cluster-yes --cluster add-node ${SLAVE_NODES[$i]} \ + ${MASTER_NODES[0]} --cluster-slave \ + --cluster-master-id $(echo "$nodesId" \ + | grep "${MASTER_NODES[$i]}$" \ + | awk '{print $1}') + done +} + +function Init { + echo 1024 > /proc/sys/net/core/somaxconn \ + && echo 1 > /proc/sys/vm/overcommit_memory \ + || Print Not specified "--privileged". + mkdir -p $LOG_DIR $DATA_DIR +} + +function StartProc { + Print Start Redis ... + redis-server /etc/redis.conf \ + --daemonize no \ + --dir $DATA_DIR \ + --cluster-config-file $CLUSTER_CONFIG_FILE \ + --logfile $LOG_DIR/redis.log \ + --unixsocket $SOCK_FILE & + PIDS="$PIDS $!" + while sleep 1; do + [ -e $SOCK_FILE ] && break || echo -n . + [ ! -e /proc/$! ] && echo && Print Unexpected error! && exit + done + echo + [ -n "$DEPLOY_CLUSTER" ] && DeployCluster + Print Redis is ready for connections. +} + +function Main { + local pid= + ModifyConf + Init + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/redis/Demo/SingleNode/README.md b/redis/Demo/SingleNode/README.md new file mode 100644 index 0000000..872a65e --- /dev/null +++ b/redis/Demo/SingleNode/README.md @@ -0,0 +1,15 @@ +# 部署 redis 单点 + +- 按实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /etc/yum.repos.d/rocky8.repo \ + && dnf makecache \ + && dnf install -y ncurses iproute procps-ng glibc-langpack-en \ + && echo 'alias ls="ls --color=auto"' > /etc/profile.d/docker.sh \ + && rm -rf /var/cache/dnf \ + && rm -rf /var/log/* +ENV PS1='\[\e[33;1;1m\][\[\e[0m\]\[\e[35;1m\]\u\[\e[0m\]\[\e[33;1;1m\]@\[\e[0m\]\[\e[31;1;1m\]docker\[\e[0m\]\[\e[32;1;1m\](\h)\[\e[0m\]\[\e[33;1;1m\]:\[\e[0m\]\[\e[32m\]\w\[\e[0m\]\[\e[33;1;1m\]]\[\e[0m\]\[\e[36m\]\$\[\e[0m\] ' +ENV PS2='\[\e[36m\]>\[\e[0m\] ' +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 + diff --git a/rocky8/README.md b/rocky8/README.md new file mode 100644 index 0000000..fbc8917 --- /dev/null +++ b/rocky8/README.md @@ -0,0 +1,10 @@ +# 构建 rockylinux 8 镜像 + +## 导入文件 +- 本机时区 /etc/localtime + +## 定制 +- 使用 Asia/Shanghai 时区 +- 安装 ncurses iproute +- 默认语言 en_US.UTF-8 + diff --git a/rocky8/rocky.sh b/rocky8/rocky.sh new file mode 100755 index 0000000..b01402c --- /dev/null +++ b/rocky8/rocky.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:8" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + Warn Preparing localtime ... + cd $ROOT_DIR + mkdir -p ADD && cp -f /etc/localtime ADD/ +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/rsync/ADD/ccmd b/rsync/ADD/ccmd new file mode 100755 index 0000000..9aceee9 --- /dev/null +++ b/rsync/ADD/ccmd @@ -0,0 +1,60 @@ +#!/bin/bash + +################################################## +# Mount file # +# - /etc/rsyncd.conf # +# Mount dir # +# - LOG_DIR # +# - PATH specified in every module # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= +LOG_DIR='/var/log/rsync' + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + Print killing rsync ... + while :; do + pkill -f rsync && Print killing rsync ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function StartProc { + Print Start rsync ... + rm -f /var/run/rsyncd.pid + rsync --daemon \ + --no-detach \ + --port=873 \ + --log-file=$LOG_DIR/rsyncd.log \ + --dparam=pidfile=/var/run/rsyncd.pid \ + &>> $LOG_DIR/rsyncd.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/rsync/Demo/SingleNode/README.md b/rsync/Demo/SingleNode/README.md new file mode 100644 index 0000000..cc4b4de --- /dev/null +++ b/rsync/Demo/SingleNode/README.md @@ -0,0 +1,16 @@ +# 部署 rsync + +- 分享服务器 /data1 和 /data2 目录 +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\> $LOG_DIR/svn.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/svn/Demo/SingleNode/README.md b/svn/Demo/SingleNode/README.md new file mode 100644 index 0000000..c10547a --- /dev/null +++ b/svn/Demo/SingleNode/README.md @@ -0,0 +1,16 @@ +# 部署 svn + +- 在服务器的 /data 目录下存放 svn 数据 +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /dev/null & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /opt/tomcat + RestoreConf + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/tomcat/Demo/SingleNode/README.md b/tomcat/Demo/SingleNode/README.md new file mode 100644 index 0000000..0645a45 --- /dev/null +++ b/tomcat/Demo/SingleNode/README.md @@ -0,0 +1,16 @@ +# 部署 tomcat + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /etc/bash.bashrc \ + && ln -sf /bin/bash /bin/sh \ + && apt update -y \ + && apt install -y ca-certificates \ + && sed -i '/^deb /s,http://ports.ubuntu.com,https://mirrors.tuna.tsinghua.edu.cn,' /etc/apt/sources.list \ + && apt update -y \ + && apt install -y vim-tiny less curl iproute2 \ + && rm -rf /var/lib/apt/lists/* /var/lib/dpkg/info/* /var/cache/apt/archives /root/.bashrc /root/.profile + diff --git a/ubuntu/README.md b/ubuntu/README.md new file mode 100644 index 0000000..45bcee9 --- /dev/null +++ b/ubuntu/README.md @@ -0,0 +1,10 @@ +# 构建 ubuntu 镜像 + +## 导入文件 +- 本机时区 /etc/localtime + +## 定制 +- 使用 Asia/Shanghai 时区 +- 安装 ca-certificates vim-tiny less curl iproute2 +- 默认语言 en_US.UTF-8 + diff --git a/ubuntu/ubuntu.sh b/ubuntu/ubuntu.sh new file mode 100755 index 0000000..5bc768e --- /dev/null +++ b/ubuntu/ubuntu.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:20" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + Warn Preparing localtime ... + cd $ROOT_DIR + cp -f /etc/localtime ADD/ +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/zabbix/ADD/ccmd b/zabbix/ADD/ccmd new file mode 100755 index 0000000..7467097 --- /dev/null +++ b/zabbix/ADD/ccmd @@ -0,0 +1,161 @@ +#!/bin/bash + +################################################## +# Mount file # +# - /etc/my.cnf # +# - /etc/zabbix/zabbix_server.conf # +# Mount dir # +# - /etc/nginx/http.d # +# - /var/log/mysql # +# - /var/lib/mysql # +# - /var/lib/mysql-bin # +# - /var/log/nginx # +# - /var/log/php7 # +# - /var/log/zabbix # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= +LOG_DIR='/var/log/mysql' +DATA_DIR='/var/lib/mysql' +BINLOG_DIR='/var/lib/mysql-bin' +INIT_FLAG=${INIT_FLAG:-} +SOCK_FILE='/run/mysqld/mysqld.sock' +PID_FILE='/run/mysqld/mysqld.pid' + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + local running + while running= ; do + pkill -f sleep && running=1 && Print killing sleep ... + pkill -f nginx && running=1 && Print killing nginx ... + pkill -f php-fpm7 && running=1 && Print killing php-fpm7 ... + pkill -f zabbix_server && running=1 && Print killing zabbix_server ... + pkill -f mysqld && running=1 && Print killing mysqld ... + [ -z "$running" ] && break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function RestoreConf { + ! ls /etc/nginx/conf.d/*.conf 2>/dev/null | grep -Eq '(zabbix|zbx)' \ + && Print Restore /etc/nginx/conf.d/zabbix.conf ... \ + && cp /usr/share/zabbix/nginx.conf /etc/nginx/conf.d/zabbix.conf + [ ! -e /etc/zabbix/zabbix_proxy.conf ] \ + && Print Restore /etc/zabbix/zabbix_proxy.conf ... \ + && cp /usr/share/zabbix/zabbix_proxy.conf /etc/zabbix/zabbix_proxy.conf + [ ! -e /etc/zabbix/zabbix_server.conf ] \ + && Print Restore /etc/zabbix/zabbix_server.conf ... \ + && cp /usr/share/zabbix/zabbix_server.conf /etc/zabbix/zabbix_server.conf + return 0 +} + +function SideCar { + local day= last_day=$(date +%d) + local md5= last_md5=$(find /etc/nginx/ -type f -name "*.conf" \ + | xargs -I ^ md5sum ^ | md5sum) + while sleep 10; do + day=$(date +%d) \ + && [ "$day" != "$last_day" ] \ + && last_day=$day \ + && find /var/log/nginx/ -type f -name "*.log" \ + | xargs -I ^ mv ^ ^.$(date +%F -d yesterday) \ + && nginx -s reopen + md5=$(find /etc/nginx/ -type f -name "*.conf" | xargs -I ^ md5sum ^ \ + | md5sum) \ + && [ "$md5" != "$last_md5" ] \ + && last_md5=$md5 \ + && nginx -tq \ + && Print Reload nginx conf ... \ + && nginx -s reload + done +} + +function InitDB { + rm -f $SOCK_FILE $PID_FILE + chown -R mysql.mysql $LOG_DIR $BINLOG_DIR $DATA_DIR + if [ ! -d "$DATA_DIR/mysql" ]; then + Print Install database ... + mysql_install_db --user=mysql > /dev/null + INIT_FLAG=1 + fi +} + +function StartProc { + Print Start mysql ... + mysqld -u mysql & + PIDS="$PIDS $!" + while sleep 1; do + [ -e $SOCK_FILE ] && break || echo -n . + [ ! -e /proc/$! ] && echo && Print unexpected error! && exit + done + echo + if [ -n "$INIT_FLAG" ]; then + Print Secure database ... + mysql_secure_installation <<< "$(echo -e '\nn\nn\n\n\n\n\n')" > /dev/null + Print Create zabbix db and user ... + mysql -e "CREATE DATABASE zabbix DEFAULT CHARSET UTF8 COLLATE UTF8_BIN" + mysql -e "CREATE USER zabbix@localhost" + mysql -e "GRANT ALL ON zabbix.* TO zabbix@localhost" + Print Import zabbix schema.sql ... + mysql -Dzabbix < /usr/share/zabbix/database/mysql/schema.sql + Print Import zabbix images.sql ... + mysql -Dzabbix < /usr/share/zabbix/database/mysql/images.sql + Print Import zabbix data.sql ... + mysql -Dzabbix < /usr/share/zabbix/database/mysql/data.sql + if sql_files="$(ls $DATA_DIR/init_sql/*.sql 2>/dev/null)"; then + Print Import the sql files ... + for sql_file in $sql_files; do + Print Importing $sql_file ... + mysql < $sql_file + done + Print Imported all sql files successfully. + fi + fi + Print MySQL is ready for connections. + + RestoreConf + + Print Start php ... + php-fpm7 -F -y /etc/php7/php-fpm.conf & + PIDS="$PIDS $!" + + Print Start zabbix ... + zabbix_server -f & + PIDS="$PIDS $!" + + Print Start nginx ... + nginx -g 'daemon off;pid /run/nginx/nginx.pid;' & + PIDS="$PIDS $!" + + Print Start nginx sidecar ... + SideCar & + PIDS="$PIDS $!" +} + +function Main { + local pid= + InitDB + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/zabbix/Demo/SingleNode/README.md b/zabbix/Demo/SingleNode/README.md new file mode 100644 index 0000000..7638a19 --- /dev/null +++ b/zabbix/Demo/SingleNode/README.md @@ -0,0 +1,23 @@ +# 部署 zabbix + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\ /etc/my.cnf \ + && echo -e '[client-server]\n\ +socket = /run/mysqld/mysqld.sock\n\ +[mysqld]\n\ +datadir = /var/lib/mysql\n\ +pid-file = /run/mysqld/mysqld.pid\n\ +log-error = /var/log/mysql/error.log\n\ +character-set-server = utf8mb4\n\ +default-storage-engine = innodb\n\ +slow-query-log = TRUE\n\ +slow-query-log-file = /var/log/mysql/slow.log\n\ +lower-case-table-names = 1\n\ +' > /etc/mysql/my.cnf \ + && sed -i 's/stty/#stty/' /usr/bin/mysql_secure_installation \ + && mkdir -p /var/log/mysql /var/lib/mysql-bin /run/mysqld \ + && chown -R mysql.mysql /var/log/mysql /var/lib/mysql-bin /run/mysqld \ + && chmod -R 0777 /usr/share/webapps/zabbix/conf \ + && sed -i '/^#* *AllowRoot *=/cAllowRoot=1' /etc/zabbix/zabbix_server.conf \ + && mv /etc/zabbix/* /usr/share/zabbix/ \ + && echo -e 'server {\n\ + listen 80;\n\ + server_name zabbix.colben.cn;\n\ + access_log /var/log/nginx/access-zabbix.log main;\n\ + error_log /var/log/nginx/error-zabbix.log;\n\ + location /zabbix/ {\n\ + root /usr/share/webapps;\n\ + index index.php index.html index.html;\n\ + }\n\ + location ~ ^/zabbix/.+\.php$ {\n\ + root /usr/share/webapps;\n\ + access_log /var/log/nginx/access-zabbix.log main;\n\ + index index.php index.html index.html;\n\ + expires -1s;\n\ + include fastcgi_params;\n\ + try_files $uri =404;\n\ + fastcgi_pass unix:/var/lib/php7/phpfpm.sock;\n\ + fastcgi_index index.php;\n\ + fastcgi_param PATH_INFO $fastcgi_path_info;\n\ + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;\n\ + fastcgi_split_path_info ^(.+\.php)(/.+)$;\n\ + break;\n\ + }\n\ + location / {\n\ + return 403;\n\ + }\n\ +}\n\ +' > /usr/share/zabbix/nginx.conf +CMD ["/opt/ccmd"] + diff --git a/zabbix/README.md b/zabbix/README.md new file mode 100644 index 0000000..c05c1bf --- /dev/null +++ b/zabbix/README.md @@ -0,0 +1,19 @@ +# 构建 zabbix 镜像 + +## 定制 +- 安装 mariadb zabbix + +## 外挂目录和文件 +- /etc/my.cnf: mysql 配置文件 +- /var/lib/mysql-bin: mysql binlog 目录 +- /var/lib/mysql: mysql 数据目录 +- /var/log/mysql: mysql 日志目录 +- /var/log/php7: php7 日志目录 +- /etc/nginx/http.d: nginx http 配置目录 +- /var/log/nginx: nginx 日志目录 +- /etc/zabbix/zabbix_server.conf: zabbix server 配置文件 +- /var/log/zabbix: zabbix 日志目录 + +## 案例 1 +- [Demo/SingleNode/](Demo/SingleNode/): 部署 zabbix + diff --git a/zabbix/zabbix.sh b/zabbix/zabbix.sh new file mode 100755 index 0000000..71b5083 --- /dev/null +++ b/zabbix/zabbix.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#========================================= +# Author : colben +#========================================= + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +[ 'x86_64' == "$(uname -m)" ] && ARCH='' || ARCH="-$(uname -m)" +ROOT_DIR="$(cd $(dirname $0) && pwd)" +IMAGE="harbor.colben.cn/general/$(basename ${0%.sh})$ARCH:latest" + +if [ -t 0 ]; then + function Print { echo -e "\033[36;1m$(date +'[%F %T]')\033[32;1m $*\033[0m"; } + function Warn { echo -e "\033[36;1m$(date +'[%F %T]')\033[33;1m $*\033[0m"; } + function Error { echo -e "\033[36;1m$(date +'[%F %T]')\033[31;1m $*\033[0m"; exit 1; } +else + function Print { echo -e "$(date +'[%F %T INFO]') $*"; } + function Warn { echo -e "$(date +'[%F %T WARN]') $*"; } + function Error { echo -e "$(date +'[%F %T ERROR]') $*"; exit 1; } +fi + +function Quit { + local exitCode=$? + [ 0 -ne $exitCode ] && Error Failed to build or push image! + [ -z "${END:-}" ] && echo && Error Interrupted manually! + Print Succeeded to build and push image. +} + +function YesOrNo { + Warn $* + local sw= + while :; do + read -p '(Yes/No/Quit) ' -n1 sw + [[ "$sw" =~ ^Y|y$ ]] && echo && return 0 + [[ "$sw" =~ ^N|n$ ]] && echo && return 1 + [[ "$sw" =~ ^Q|q$ ]] && echo && exit 0 + [ -n "$sw" ] && echo + done +} + +function Update { + : +} + +function Build { + local yn + cd $ROOT_DIR + docker images --format='{{.Repository}}:{{.Tag}}' | grep "^$IMAGE$" \ + && Warn Removing image $IMAGE ... \ + && docker rmi $IMAGE + Warn Building image: $IMAGE ... + docker build --force-rm --build-arg ARCH="$ARCH" -t $IMAGE . + YesOrNo Push image: $IMAGE? && docker push $IMAGE +} + +function Main { + Update + Build + END=1 +} + +# Start here +Main + diff --git a/zookeeper/ADD/ccmd b/zookeeper/ADD/ccmd new file mode 100755 index 0000000..abaac88 --- /dev/null +++ b/zookeeper/ADD/ccmd @@ -0,0 +1,83 @@ +#!/bin/bash + +################################################## +# Mount dir # +# - /opt/zk/data # +# - /opt/zk/dataLog # +# - /opt/zk/logs # +# ENV # +# - MYID # +# - JVMFLAGS # +# - _CONF_* # +################################################## + +set -euo pipefail +export LANG=en_US.UTF-8 +trap Quit EXIT + +PIDS= +GOT_SIGTERM= + +function Print { + local file=/dev/null + [ '-f' = "$1" ] && file=$2 && shift && shift + date +"[%F %T] $*" | tee -a $file +} + +function Quit { + while :; do + pkill -f java && Print killing java ... || break + sleep 1 + done + Print Container stopped. + test -n "$GOT_SIGTERM" +} + +function Usage { + Print 'This container should run with + **root user** + **/opt/zk/{data,dataLog,logs} mounted from host** + ' +} + +function ModifyConf { + Print Modify server.properties ... + local kv= + local conf='conf/zoo.cfg' + while read kv; do + [ -z "$kv" ] && break + Print Modify property: ${kv%%=*} ... + sed -i "/^${kv%%=*} *=/d" $conf + echo "$kv" >> $conf + done <<< "$(env | grep '^_CONF_' | sed 's/_CONF_//')" + sed -i -e '/^dataDir/d' -e '/^dataLogDir/d' $conf + echo -e 'dataDir=/opt/zk/data\ndataLogDir=/opt/zk/dataLog' >> $conf + if [ ! -e data/myid ]; then + Print Generate myid ... + echo $MYID > data/myid + fi +} + +function StartProc { + Print Start zookeeper ... + ./bin/zkServer.sh start-foreground &>> logs/zk.out & + PIDS="$PIDS $!" +} + +function Main { + local pid= + cd /opt/zk + Usage + ModifyConf + StartProc + trap "GOT_SIGTERM=1; Print Got SIGTERM ..." SIGTERM + while [ -z "$GOT_SIGTERM" ] && sleep 1; do + for pid in $PIDS; do + [ ! -e /proc/$pid ] && Print Unexpected error! && exit + done + done +} + +# Start here +Main + diff --git a/zookeeper/Demo/SingleNode/README.md b/zookeeper/Demo/SingleNode/README.md new file mode 100644 index 0000000..23be4d5 --- /dev/null +++ b/zookeeper/Demo/SingleNode/README.md @@ -0,0 +1,15 @@ +# 部署 zookeeper 单节点 + +- 根据实际环境修改 + - docker-compose.yml + +- 创建目录 + ``` + grep '\