Added restic-auto

2025-12-16 03:34:18 +01:00 · 2020-04-25 16:07:15 +02:00
parent ae65956de6
commit 044200f401
8 changed files with 222 additions and 16 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,25 +1,26 @@
 language: bash
 services: docker
 sudo: required
-   
+
 notifications:
  email: false
-   
+  slack:
+    rooms:
+      - secure: 3iUauEp7w5F6YDXXuJnKSLR9TZjndwhwWcDa+1KkAp9qv0NBAn1DWntl5iPUGzuZ0KdpQlI3QbvSMn0jYbsXgMDLgZRvCGaQ+TXrJblTPvlZXVxeGM3mCr6yEpt0foe8qqhiWAMer8ioM7QKncBiQfGbB2VctvuH8YqG2JuRSVe/dZxd+Jjn7J8SQyW97GQ0wpsYHQV7V2hg8KOF5/KNNk8Y93RGvnvEVX7XNVI+h+cdsxjBys9qxvC3gRpcngXw/v3mPM2sXN0LHjkobu9P9E7v9RHNwCz3i7FMzmOX82wcoPr06pnvnmecnU6pY3Uwmobed71KnTERSXt0S0Qpjvhod004vFBvzZovHKaucsUOzq0FdN+KaOhOZMr+LfCNty3iWZrEiYiuntFDAmr38xE0q4nb3dmljzEav8nTRO3/9DezWV98CBictzhwxknD/aPpWl7kXEl21K/jnprcJXypqNVscjJ/jylNiwebmFsv7oHlgeyzA9IbG8U7yO8kxCx86214FmPAJBkyAfevrQq9hOcigVwNuV7EoW5zqLTYb1pTyOkHSz/W9dKawjeG4c8ZTBsS1x6c+jQj735/MsQK0WMuaybOeTrVu6akKwXpiyZ4OGO7krHfCByXryZ56BNl7kb1J7aTjaLi0sifdc1X/eI6Y9nsHhc+Zyvv4AA=
+    on_success: change
+    
 env:
-  - IMAGE=test
+  - IMAGE=restic-auto

 install:
-  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
-  - sudo systemctl restart docker
-  - sudo docker run --privileged linuxkit/binfmt:v0.6
-  - sudo docker run -d --privileged -p 1234:1234 --name buildkit moby/buildkit:latest --addr tcp://0.0.0.0:1234 --oci-worker-platform linux/amd64 --oci-worker-platform linux/arm/v6 --oci-worker-platform linux/arm/v7 --oci-worker-platform linux/arm64
-  - sudo docker cp buildkit:/usr/bin/buildctl /usr/bin/
-  - export BUILDKIT_HOST=tcp://0.0.0.0:1234
-  
+  - docker run --rm --privileged multiarch/qemu-user-static:register --credential yes
+
 script:
+  - cd ${IMAGE}
+  - make build
+  - docker images
+
+after_success:
  - if [ "${TRAVIS_BRANCH}" == "master" ]; then
-      docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}" ;
-      buildctl build --frontend dockerfile.v0 --local context=${IMAGE} --local dockerfile=${IMAGE} --frontend-opt platform=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 --output type=image,name=docker.io/napnap75/${IMAGE},push=true ;
-    else
-      buildctl build --frontend dockerfile.v0 --local context=${IMAGE} --local dockerfile=${IMAGE} --frontend-opt platform=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 ;
+      make push ;
    fi
--- a/restic-auto/Dockerfile.amd64
+++ b/restic-auto/Dockerfile.amd64
@@ -0,0 +1,18 @@
+FROM alpine:latest AS builder
+
+RUN apk add --no-cache curl \
+	&& while [ "$DOWNLOAD_URL" == "" ] ; do DOWNLOAD_URL=$(curl -s https://api.github.com/repos/restic/restic/releases/latest | grep "browser_download_url" | grep "linux_amd64\." | cut -d\" -f4) ; done \
+	&& curl --retry 3 -L -s -o restic.bz2 ${DOWNLOAD_URL} \
+	&& bunzip2 restic.bz2 \
+	&& chmod +x restic
+
+FROM amd64/alpine:latest
+
+COPY --from=builder restic /usr/bin/
+
+RUN apk add --no-cache bash curl jq openssh-client dcron tzdata
+
+COPY restic-auto docker-entrypoint.sh docker-command.sh /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+CMD ["/usr/local/bin/docker-command.sh"]
--- a/restic-auto/Dockerfile.arm32v6
+++ b/restic-auto/Dockerfile.arm32v6
@@ -0,0 +1,21 @@
+FROM alpine:latest AS builder
+
+RUN apk add --no-cache curl \
+        && while [ "$QEMU_DOWNLOAD_URL" == "" ] ; do QEMU_DOWNLOAD_URL=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/releases/latest | grep "browser_download_url" | grep "\/qemu-arm-static.tar.gz" | cut -d\" -f4) ; done \
+        && curl --retry 3 -L -s -o /tmp/qemu-arm-static.tar.gz $QEMU_DOWNLOAD_URL \
+        && tar xzf /tmp/qemu-arm-static.tar.gz \
+	&& while [ "$DOWNLOAD_URL" == "" ] ; do DOWNLOAD_URL=$(curl -s https://api.github.com/repos/restic/restic/releases/latest | grep "browser_download_url" | grep "linux_arm\." | cut -d\" -f4) ; done \
+	&& curl --retry 3 -L -s -o restic.bz2 ${DOWNLOAD_URL} \
+	&& bunzip2 restic.bz2 \
+	&& chmod +x restic
+
+FROM arm32v6/alpine:latest
+
+COPY --from=builder qemu-arm-static restic /usr/bin/
+
+RUN apk add --no-cache bash curl jq openssh-client dcron tzdata
+
+COPY restic-auto docker-entrypoint.sh docker-command.sh /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+CMD ["/usr/local/bin/docker-command.sh"]
--- a/restic-auto/Makefile
+++ b/restic-auto/Makefile
@@ -0,0 +1,19 @@
+all: build push
+
+build:
+	docker build -t napnap75/restic-auto:latest-amd64 -f Dockerfile.amd64 .
+	docker build -t napnap75/restic-auto:latest-arm32v6 -f Dockerfile.arm32v6 .
+
+push:
+	if [ "${DOCKER_USERNAME}" != "" ]; then docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}" ; fi
+	docker push napnap75/restic-auto:latest-amd64
+	docker push napnap75/restic-auto:latest-arm32v6
+	env DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create napnap75/restic-auto:latest napnap75/restic-auto:latest-amd64 napnap75/restic-auto:latest-arm32v6
+	env DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push -p napnap75/restic-auto:latest
+	version=$$(docker run --rm -it napnap75/restic-auto:latest restic version | egrep -o "restic [.0-9]+ compiled" | egrep -o "[.0-9]+") ; \
+		docker tag napnap75/restic-auto:latest-amd64 napnap75/restic-auto:$$version-amd64 ; \
+		docker push napnap75/restic-auto:$$version-amd64 ; \
+		docker tag napnap75/restic-auto:latest-arm32v6 napnap75/restic-auto:$$version-arm32v6 ; \
+		docker push napnap75/restic-auto:$$version-arm32v6 ; \
+		env DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create napnap75/restic-auto:$$version napnap75/restic-auto:$$version-amd64 napnap75/restic-auto:$$version-arm32v6 ; \
+		env DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push -p napnap75/restic-auto:$$version
--- a/restic-auto/docker-command.sh
+++ b/restic-auto/docker-command.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+
+crond -b -L /var/log/cron.log && tail -f /var/log/cron.log
--- a/restic-auto/docker-entrypoint.sh
+++ b/restic-auto/docker-entrypoint.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+# When used with S3 and docker secrets, get the credentials from files
+if [[ "$AWS_ACCESS_KEY_ID" = /* && -f "$AWS_ACCESS_KEY_ID" ]] ; then
+	AWS_ACCESS_KEY_ID=$(cat $AWS_ACCESS_KEY_ID)
+fi
+if [[ "$AWS_SECRET_ACCESS_KEY" = /* && -f "$AWS_SECRET_ACCESS_KEY" ]] ; then
+	AWS_SECRET_ACCESS_KEY=$(cat $AWS_SECRET_ACCESS_KEY)
+fi
+
+# When used with SFTP set the SSH configuration file
+if [[ "$RESTIC_REPOSITORY" = sftp:* ]] ; then
+	# Copy the key and make it readable only by the current user to meet SSH security requirements
+	cp $SFTP_KEY /tmp/foreign_host_key
+	chmod 400 /tmp/foreign_host_key
+	SFTP_KEY=/tmp/foreign_host_key
+
+	# Initialize the SSH config file with the values provided in the environment
+	mkdir -p /root/.ssh
+	echo "Host $SFTP_HOST" > /root/.ssh/config
+	if [[ "$SFTP_PORT" != "" ]] ; then echo "Port $SFTP_PORT" >> /root/.ssh/config ; fi
+	echo "IdentityFile $SFTP_KEY" >> /root/.ssh/config
+	echo "StrictHostKeyChecking no" >> /root/.ssh/config
+fi
+
+# Install the crontabs
+if [ -d /crontabs ] ; then
+	for f in /crontabs/* ; do
+		crontab -u $(basename $f) $f
+	done
+fi
+
+"$@"
--- a/restic-auto/restic-auto
+++ b/restic-auto/restic-auto
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# Backup one directory
+function backup_dir {
+	# Check if the dir to backup is mounted as a subdirectory of /root inside this container
+	if [ -d "/root_fs$1" ]; then
+		restic backup /root_fs$1
+		return $?
+	else
+		echo "[ERROR] Directory $1 not found. Have you mounted the root fs from your host with the following option : '-v /:/root_fs:ro' ?"
+		return -1
+	fi
+}
+
+# Backup one file
+function backup_file {
+	cat $1 | restic backup --stdin --stdin-filename $2
+	return $?
+}
+
+# Firt, check the repository is unlocked and try to unlock it
+if [ "$(restic --no-lock list locks -q)" != "" ]; then
+	echo "[INFO] Repository locked, trying to unlock"
+	restic unlock
+	if [ $? -ne 0 ]; then
+		echo "[ERROR] Could not unlock repository"
+		return -1
+	fi
+fi
+
+count_success=0
+count_failure=0
+
+# List all the containers
+containers=$(curl -s --unix-socket /var/run/docker.sock http:/v1.26/containers/json)
+for container_id in $(echo $containers | jq ".[].Id") ; do
+	container=$(echo $containers | jq -c ".[] | select(.Id==$container_id)")
+
+	# Get the name and namespace (in case of a container run in a swarm stack)
+	container_name=$(echo $container | jq -r ".Names | .[0]" | cut -d'.' -f1 | cut -d'/' -f2)
+	namespace=$(echo $container | jq -r ".Labels | .[\"com.docker.stack.namespace\"]")
+
+	# Backup the dirs labelled with "napnap75.backup.dirs"
+	if $(echo $container | jq ".Labels | has(\"napnap75.backup.dirs\")") ; then
+		for dir_name in $(echo $container | jq -r ".Labels | .[\"napnap75.backup.dirs\"]") ; do
+			echo "[INFO] Backing up dir" $dir_name "for container" $container_name
+			backup_dir $dir_name
+			if [ $? -ne 0 ]; then
+				((++count_failure))
+			else
+				((++count_success))
+			fi
+		done
+	fi
+
+	# Backup the volumes labelled with "napnap75.backup.volumes"
+	if $(echo $container | jq ".Labels | has(\"napnap75.backup.volumes\")") ; then
+		for volume_name in $(echo $container | jq -r ".Labels | .[\"napnap75.backup.volumes\"]") ; do
+			if [ $namespace != "null" ] ; then volume_name="${namespace}_${volume_name}" ; fi
+			volume_mount=$(echo $container | jq -r ".Mounts[] | select(.Name==\"$volume_name\") | .Source")
+			echo "[INFO] Backing up volume" $volume_name "with mount" $volume_mount "for container" $container_name
+			backup_dir $volume_mount
+			if [ $? -ne 0 ]; then
+				((++count_failure))
+			else
+				((++count_success))
+			fi
+		done
+	fi
+
+	# Backup the databases labelled with "napnap75.backup.databases"
+	if $(echo $container | jq ".Labels | has(\"napnap75.backup.databases\")") ; then
+		container_id=$(echo $container_id | sed "s/\"//g")
+		database_password=$(curl -s --unix-socket /var/run/docker.sock http:/v1.26/containers/$container_id/json | jq -r ".Config.Env[] | match(\"MYSQL_ROOT_PASSWORD=(.*)\") | .captures[0].string")
+		for database_name in $(echo $container | jq -r ".Labels | .[\"napnap75.backup.databases\"]") ; do
+			echo "[INFO] Backing up database" $database_name "for container" $container_name
+			if [[ "$database_password" != "" ]] ; then
+				exec_id=$(curl -s --unix-socket /var/run/docker.sock -X POST -H "Content-Type: application/json" -d '{"AttachStdout":true,"AttachStderr":true,"Tty":true,"Cmd":["/bin/bash", "-c", "mysqldump -p'$database_password' --databases '$database_name'"]}' http:/v1.26/containers/$container_id/exec | jq ".Id" | sed "s/\"//g")
+			else
+				exec_id=$(curl -s --unix-socket /var/run/docker.sock -X POST -H "Content-Type: application/json" -d '{"AttachStdout":true,"AttachStderr":true,"Tty":true,"Cmd":["/bin/bash", "-c", "mysqldump --databases '$database_name'"]}' http:/v1.26/containers/$container_id/exec | jq ".Id" | sed "s/\"//g")
+			fi
+			curl -s --unix-socket /var/run/docker.sock -X POST -H "Content-Type: application/json" -d '{"Detach":false,"Tty":false}' http:/v1.26/exec/$exec_id/start | gzip > /tmp/database_backup.gz
+			exit_code=$(curl -s --unix-socket /var/run/docker.sock http:/v1.26/exec/$exec_id/json | jq ".ExitCode")
+			if [ $exit_code -ne 0 ]; then
+				echo "[ERROR] Unable to backup database $database_name from container $container_name"
+				cat /tmp/database_backup.gz | gzip -d
+				((++count_failure))
+			else
+				backup_file /tmp/database_backup.gz ${container_name}—${database_name}.sql.gz
+				if [ $? -ne 0 ]; then
+					((++count_failure))
+				else
+					((++count_success))
+				fi
+			fi
+			rm /tmp/database_backup.gz
+		done
+	fi
+done
+
+# Send a notification to Slack
+if [[ "$SLACK_URL" != "" ]] ; then
+	curl -s -X POST --data-urlencode "payload={\"username\": \"rpi-docker-backup\", \"icon_emoji\": \":dvd:\", \"text\": \"Backup finished on host $HOSTNAME : $count_success succeeded, $count_failure failed\"}" $SLACK_URL
+fi
+
+# Return a non zero exit code if an error happened
+if [ $count_failure -ne 0 ]; then
+	exit -1
+fi
--- a/test/Dockerfile
+++ b/test/Dockerfile
@@ -1,2 +0,0 @@
-FROM alpine
-RUN uname -a