Docker_文件数据备份实战

发表于 分类于 阅读次数：

Docker文件数据备份

该脚本运行需要要求环境有 scp 应用, 配置好远端及本地备份路径后, 直接运行即可,

配置文件目录:

config.sh

功能:

1
拉取远端服务器文件到本脚本所在目录:File_Bak

常用命令

1
2
docker build --no-cache -t file_back_up .
docker run --name c_file_back_up -v [主机备份目录]:/File/[配置容器备份目录] -d file_back_up

容器自启动

1
docker update --restart=always c_file_back_up

查看crond日志详情

1
cat /var/log/crond.log

自定义逻辑

1
2
3
4
5
6
1.备份日志为/File/log.txt
2.crond定时任务日志在/var/log/crond.log
3.最新一次拉取日志debug信息在/File/currentlog.txt
4.备份文件在/File/File_Bak, 可以通过修改config.sh中[local_path]更改
5.ssh证书密钥请放在env文件中, 并修改config.sh读取路径
6.当前设置为每5分钟定时拉取一次, 失败尝试三次, 三次后放弃, 需要请修改时长

Dockerfile文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
FROM alpine:3.12.1

WORKDIR File

COPY . .

RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories; \
    apk add --no-cache openssh tzdata; \
    cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime; \
    apk del tzdata; \
    apk add --no-cache dos2unix curl; \
    dos2unix *.sh; \
    dos2unix config; \
    apk del dos2unix; \
    chmod 600 ./env/id_rsa_190; \
    sed -i "s/#PermitRootLogin.*/PermitRootLogin yes/g" /etc/ssh/sshd_config; \
    ssh-keygen -t dsa -P "" -f /etc/ssh/ssh_host_dsa_key; \
    ssh-keygen -t rsa -P "" -f /etc/ssh/ssh_host_rsa_key; \
    ssh-keygen -t ecdsa -P "" -f /etc/ssh/ssh_host_ecdsa_key; \
    ssh-keygen -t ed25519 -P "" -f /etc/ssh/ssh_host_ed25519_key; \
    echo "root:admin" | chpasswd; \
    echo "*/2 * * * * cd /File && /bin/sh ./scp_back_up.sh > currentlog.txt" >> /var/spool/cron/crontabs/root;

EXPOSE 22

CMD crond -L /var/log/crond.log -f

scp_back_up.sh脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#! /bin/sh

######################### ↓↓↓ 方法区域  ↓↓↓ ############################
sendMsg(){
    if [ ${production} -eq 'true' ]; then
        MessageUrl=${alert_notice_product}
    else
        MessageUrl=${alert_notice_debug}
    fi
    PostData="{\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"### 项目: 文件备份\n >${IP}${NAME}\n${1}\n${2}\"}}"
    echo -e "\n ${current_time}" >> ${notice_file}
    echo "curl -H \"Content-Type: application/json;charset=utf-8\" -X POST -d \"${PostData}\" \"${MessageUrl}\"" >> ${notice_file}
    curl -H "Content-Type: application/json;charset=utf-8" -X POST -d "${PostData}" "${MessageUrl}" >> ${notice_file}
}

# 获取远端文件夹内创建时间在规定期间的文件
getFile(){
    echo "ssh -i ${identify_file} -o StrictHostKeyChecking=no -p ${PORT} ${USER}@${IP} \"find ${back_up_file_path} -type f -ctime ${back_time}\""
    ssh -i ${identify_file} -o StrictHostKeyChecking=no -p ${PORT} ${USER}@${IP} "find ${back_up_file_path} -type f -ctime ${back_time}" > ${current_time}.txt
}


######################### ↑↑↑ 方法区域  ↑↑↑ ############################

# 引入配置文件
source config
# 当前时间
current_time=`date "+%Y-%m-%d_%H:%M:%S"`

# 创建文件夹
if [[ ! -d ${local_tmp_path} ]]; then
        mkdir -p ${local_tmp_path}
fi

if [[ ! -d ${local_path} ]]; then
        mkdir -p ${local_path}
fi
if [[ ! -f ${log_file} ]]; then
        touch ${log_file}
fi
if [[ ! -f ${notice_file} ]]; then
        touch ${notice_file}
fi
if [[ ! -f ${current_time}.txt ]]; then
        touch ${current_time}.txt
fi
if [[ ! -f ${current_time}false.txt ]]; then
        touch ${current_time}false.txt
fi

getFile

if [ ! -s ${current_time}.txt ]; then
  rm -f ${current_time}.txt
  rm -f ${current_time}false.txt
  sendMsg "${IP}:${file_path}" "${current_time}备份文件为空,备份失败"
  exit 0
fi

cat ${current_time}.txt | while read LINE ; do
    file_path=${LINE}
    file_name=`basename ${LINE}`
    # 普通ssh登录
    # scp -r -v -p -P ${PORT} -l ${transfer_speed} ${USER}@${IP}:${back_up_file_path}${back_up_file_name} ${local_path}

    # 使用密钥文件登录
    count_times=0

    # 尝试最多三次拉取远端文件
    while [[ ${count_times} -lt 3 ]]; do
        count_times=`expr ${count_times} + 1`
        # echo "scp -r -v -p -i ${identify_file} -P ${PORT} -l ${transfer_speed} ${USER}@${IP}:${back_up_file_path}${back_up_file_name} ${local_tmp_path}"
        echo "scp -r -v -p -i ${identify_file} -o StrictHostKeyChecking=no -P ${PORT} -l ${transfer_speed} ${USER}@${IP}:${file_path} ${local_tmp_path}"
        scp -r -v -p -i ${identify_file} -o StrictHostKeyChecking=no -P ${PORT} -l ${transfer_speed} ${USER}@${IP}:${file_path} ${local_tmp_path}
        cmd_result=$?
        # 拉取成功后跳出
        if [[ ${cmd_result} -eq 0 ]]; then
                sh -c "echo '${current_time}第${count_times}次拉取数据${file_path}成功' >>  ${log_file}"
                break
        fi
        sleep 5
    done

    # 判读是否成功移动,写入日志
    if [[ ${cmd_result} -eq 0 ]]; then
        echo "${current_time}移动文件到缓存目录 ${local_tmp_path} 成功"
        sh -c "echo '${current_time}移动文件到缓存目录 ${local_tmp_path} 成功'  >> ${log_file}"
    else
        echo "${cmd_result}" > ${current_time}false.txt
        echo "!!! ${current_time}拉取远端文件到缓存目录 ${local_tmp_path} 失败"
        sh -c "echo '!!! ${current_time}拉取远端文件到缓存目录 ${local_tmp_path} 失败'  >> ${log_file}"
        # 调用通知函数发送失败通知
        sendMsg "${IP}${file_path}" "!!!${current_time}拉取远端文件到缓存目录${local_tmp_path}失败"
    fi

    # 从缓存目录移动文件到实际备份文件夹
    mv ${local_tmp_path}/${file_name} ${local_path}
    mv_result=$?
    if [[ ${mv_result} -ne 0 ]]; then
        echo "${mv_result}" > ${current_time}false.txt
        echo "!!! ${current_time}移动文件${local_tmp_path}${file_name} 到目录 ${local_path} 失败"
        sh -c "echo '!!! ${current_time}移动文件${local_tmp_path}${file_name} 到目录 ${local_path} 失败' >> ${log_file}"
        sendMsg "${IP}${file_name}" "!!!${current_time}移动文件${local_tmp_path}${file_name}到目录${local_path}失败"
    else
        echo "!!! ${current_time}移动文件${local_tmp_path}${file_name} 到目录 ${local_path} 成功"
    fi
    echo 'end'
done
# 删除文件清单
rm -r ${current_time}.txt

if [ ! -s ${current_time}false.txt ]; then
  sendMsg "${IP}:${file_path}" "${current_time}备份文件成功"
else
  sendMsg "${IP}:${file_path}" "!!!${current_time}备份文件有错误,请查看日志详情"
fi
rm -r ${current_time}false.txt

config配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 远端配置
NAME='日常备份'              # 项目名称
USER='[使用用户]'                    # 用户名
IP='xxx.xxx.xxx.xxx'              # IP地址
PORT='[端口]'                    # 端口

# 功能配置
identify_file='./env/[公钥文件名]'                # 登录公钥文件
transfer_speed='200000'                           # 最大传输速度 kb/s

# 项目文件配置
back_up_file_path='/var/www/[备份文件夹名]/'                # 远端备份路径
local_tmp_path='/var/www/LocalCFS/temp'                      # 本地缓存路径
local_path='/File/File_Bak/[项目名]/'  	        # 本地存储路径
back_time='-1'                                  # 遵循find命令格式,-1代表一天之内的文件

# 日志文件/通知配置
alert_notice_product='https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxxxxx'
alert_notice_debug='https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxxxxx'
log_file='log.txt'                              # 日志名称
notice_file='notice.txt'                        # 消息日志名称
production=false                                # 消息发送环境设定