romein/docker-volume-s3-backup
1
0
Fork 0
mirror of https://github.com/garraflavatra/docker-volume-s3-backup.git synced 2025-07-18 15:54:06 +00:00
Code Issues Activity

script cleanup, misc changes

Browse Source 
- --blobs by default
- always drop and recreate schemas, specified by DROP_SCHEMAS
This commit is contained in:
Elliott Shugerman
2019-11-29 22:17:47 -07:00
parent 1ba61ed95a
commit e8ad6ceba4
11 changed files with 188 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/10.Dockerfile
View File

@ -6,21 +6,21 @@ FROM alpine:3.8
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_DATABASE ''
ENV POSTGRES_HOST ''
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV POSTGRES_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV POSTGRES_USER ''
ENV POSTGRES_PASSWORD ''
ENV PGDUMP_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID ''
ENV S3_SECRET_ACCESS_KEY ''
ENV S3_BUCKET ''
ENV S3_REGION 'us-west-1'
ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ENV S3_ENDPOINT ''
ENV S3_S3V4 'no'
ENV SCHEDULE ''
ENV PASSPHRASE ''
ADD run.sh run.sh
ADD backup.sh backup.sh

26
src/11.Dockerfile
View File

@ -6,21 +6,21 @@ FROM alpine:3.10
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_DATABASE ''
ENV POSTGRES_HOST ''
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV POSTGRES_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV POSTGRES_USER ''
ENV POSTGRES_PASSWORD ''
ENV PGDUMP_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID ''
ENV S3_SECRET_ACCESS_KEY ''
ENV S3_BUCKET ''
ENV S3_REGION 'us-west-1'
ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ENV S3_ENDPOINT ''
ENV S3_S3V4 'no'
ENV SCHEDULE ''
ENV PASSPHRASE ''
ADD run.sh run.sh
ADD backup.sh backup.sh

26
src/12.Dockerfile
View File

@ -6,21 +6,21 @@ FROM alpine:edge
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_DATABASE ''
ENV POSTGRES_HOST ''
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV POSTGRES_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV POSTGRES_USER ''
ENV POSTGRES_PASSWORD ''
ENV PGDUMP_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID ''
ENV S3_SECRET_ACCESS_KEY ''
ENV S3_BUCKET ''
ENV S3_REGION 'us-west-1'
ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ENV S3_ENDPOINT ''
ENV S3_S3V4 'no'
ENV SCHEDULE ''
ENV PASSPHRASE ''
ADD run.sh run.sh
ADD backup.sh backup.sh

26
src/9.Dockerfile
View File

@ -6,21 +6,21 @@ FROM alpine:3.6
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_DATABASE ''
ENV POSTGRES_HOST ''
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV POSTGRES_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV POSTGRES_USER ''
ENV POSTGRES_PASSWORD ''
ENV PGDUMP_EXTRA_OPTS ''
ENV S3_ACCESS_KEY_ID ''
ENV S3_SECRET_ACCESS_KEY ''
ENV S3_BUCKET ''
ENV S3_REGION 'us-west-1'
ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ENV S3_ENDPOINT ''
ENV S3_S3V4 'no'
ENV SCHEDULE ''
ENV PASSPHRASE ''
ADD run.sh run.sh
ADD backup.sh backup.sh

59
src/backup.sh
View File

@ -1,30 +1,30 @@
#! /bin/sh
set -e
set -eu
set -o pipefail
if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then
if [ -z "$S3_ACCESS_KEY_ID" ]; then
echo "You need to set the S3_ACCESS_KEY_ID environment variable."
exit 1
fi
if [ "${S3_SECRET_ACCESS_KEY}" = "**None**" ]; then
if [ -z "$S3_SECRET_ACCESS_KEY" ]; then
echo "You need to set the S3_SECRET_ACCESS_KEY environment variable."
exit 1
fi
if [ "${S3_BUCKET}" = "**None**" ]; then
if [ -z "$S3_BUCKET" ]; then
echo "You need to set the S3_BUCKET environment variable."
exit 1
fi
if [ "${POSTGRES_DATABASE}" = "**None**" ]; then
if [ -z "$POSTGRES_DATABASE" ]; then
echo "You need to set the POSTGRES_DATABASE environment variable."
exit 1
fi
if [ "${POSTGRES_HOST}" = "**None**" ]; then
if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then
if [ -z "$POSTGRES_HOST" ]; then
if [ -n "$POSTGRES_PORT_5432_TCP_ADDR" ]; then
POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR
POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT
else
@ -33,36 +33,53 @@ if [ "${POSTGRES_HOST}" = "**None**" ]; then
fi
fi
if [ "${POSTGRES_USER}" = "**None**" ]; then
if [ -z "$POSTGRES_USER" ]; then
echo "You need to set the POSTGRES_USER environment variable."
exit 1
fi
if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then
echo "You need to set the POSTGRES_PASSWORD environment variable or link to a container named POSTGRES."
if [ -z "$POSTGRES_PASSWORD" ]; then
echo "You need to set the POSTGRES_PASSWORD environment variable."
exit 1
fi
if [ "${S3_ENDPOINT}" == "**None**" ]; then
AWS_ARGS=""
if [ -z "$S3_ENDPOINT" ]; then
aws_args=""
else
AWS_ARGS="--endpoint-url ${S3_ENDPOINT}"
aws_args="--endpoint-url $S3_ENDPOINT"
fi
# env vars needed for aws tools
export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY
export AWS_DEFAULT_REGION=$S3_REGION
export PGPASSWORD=$POSTGRES_PASSWORD
POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTGRES_EXTRA_OPTS"
echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..."
echo "Creating backup of $POSTGRES_DATABASE database..."
pg_dump --format=custom \
-h $POSTGRES_HOST \
-p $POSTGRES_PORT \
-U $POSTGRES_USER \
-d $POSTGRES_DATABASE \
$PGDUMP_EXTRA_OPTS \
> db.dump
pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | gzip > dump.sql.gz
timestamp=$(date +"%Y-%m-%dT%H:%M:%S")
s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}/${POSTGRES_DATABASE}_${timestamp}.dump"
echo "Uploading dump to $S3_BUCKET"
if [ -n "$PASSPHRASE" ]; then
echo "Encrypting backup..."
gpg --symmetric --batch --passphrase "$PASSPHRASE" db.dump
rm db.dump
local_file="db.dump.gpg"
s3_uri="${s3_uri_base}.gpg"
else
local_file="db.dump"
s3_uri="$s3_uri_base"
fi
cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2
echo "Uploading backup to $S3_BUCKET..."
aws $aws_args s3 cp "$local_file" "$s3_uri"
rm "$local_file"
echo "SQL backup uploaded successfully"
echo "Backup complete."

5
src/install.sh
View File

@ -1,13 +1,16 @@
#! /bin/sh
# exit if a command fails
set -e
set -o pipefail
apk update
# install pg_dump
apk add postgresql-client
# install gpg
apk add gnupg
# install s3 tools
apk add python py-pip
pip install awscli

76
src/restore.sh
View File

@ -1,30 +1,30 @@
#! /bin/sh
set -e
set -eu
set -o pipefail
if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then
if [ -z "$S3_ACCESS_KEY_ID" ]; then
echo "You need to set the S3_ACCESS_KEY_ID environment variable."
exit 1
fi
if [ "${S3_SECRET_ACCESS_KEY}" = "**None**" ]; then
if [ -z "$S3_SECRET_ACCESS_KEY" ]; then
echo "You need to set the S3_SECRET_ACCESS_KEY environment variable."
exit 1
fi
if [ "${S3_BUCKET}" = "**None**" ]; then
if [ -z "$S3_BUCKET" ]; then
echo "You need to set the S3_BUCKET environment variable."
exit 1
fi
if [ "${POSTGRES_DATABASE}" = "**None**" ]; then
if [ -z "$POSTGRES_DATABASE" ]; then
echo "You need to set the POSTGRES_DATABASE environment variable."
exit 1
fi
if [ "${POSTGRES_HOST}" = "**None**" ]; then
if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then
if [ -z "$POSTGRES_HOST" ]; then
if [ -n "$POSTGRES_PORT_5432_TCP_ADDR" ]; then
POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR
POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT
else
@ -33,49 +33,63 @@ if [ "${POSTGRES_HOST}" = "**None**" ]; then
fi
fi
if [ "${POSTGRES_USER}" = "**None**" ]; then
if [ -z "$POSTGRES_USER" ]; then
echo "You need to set the POSTGRES_USER environment variable."
exit 1
fi
if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then
echo "You need to set the POSTGRES_PASSWORD environment variable or link to a container named POSTGRES."
if [ -z "$POSTGRES_PASSWORD" ]; then
echo "You need to set the POSTGRES_PASSWORD environment variable" \
"or link to a container named POSTGRES."
exit 1
fi
if [ "${S3_ENDPOINT}" == "**None**" ]; then
AWS_ARGS=""
if [ -z "$S3_ENDPOINT" ]; then
aws_args=""
else
AWS_ARGS="--endpoint-url ${S3_ENDPOINT}"
aws_args="--endpoint-url $S3_ENDPOINT"
fi
# env vars needed for aws tools
export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY
export AWS_DEFAULT_REGION=$S3_REGION
export PGPASSWORD=$POSTGRES_PASSWORD
POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER"
echo "Finding latest backup"
s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}"
LATEST_BACKUP=$(aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }')
echo "Fetching ${LATEST_BACKUP} from S3"
aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz
gzip -d dump.sql.gz
if [ "${DROP_PUBLIC}" == "yes" ]; then
echo "Recreating the public schema"
psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE -c "drop schema public cascade; create schema public;"
if [ -z "$PASSPHRASE" ]; then
file_type=".dump"
else
file_type=".dump.gpg"
fi
echo "Restoring ${LATEST_BACKUP}"
if [ $# -eq 1 ]; then
timestamp="$1"
key_suffix="${POSTGRES_DATABASE}_${timestamp}${file_type}"
else
echo "Finding latest backup..."
key_suffix=$(
aws $aws_args s3 ls "${s3_uri_base}/" \
| sort \
| tail -n 1 \
| awk '{ print $4 }'
)
fi
psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE < dump.sql
echo "Fetching backup from S3..."
aws $aws_args s3 cp "${s3_uri_base}/${key_suffix}" "db${file_type}"
rm dump.sql
if [ -n "$PASSPHRASE" ]; then
echo "Decrypting backup..."
gpg --decrypt --batch --passphrase "$PASSPHRASE" db.dump.gpg > db.dump
rm db.dump.gpg
fi
echo "Restore complete"
conn_opts="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DATABASE"
echo "Restoring from backup..."
pg_restore $conn_opts --single-transaction --clean db.dump
rm db.dump
echo "Restore complete."

8
src/run.sh
View File

@ -1,13 +1,13 @@
#! /bin/sh
set -e
set -eu
if [ "${S3_S3V4}" = "yes" ]; then
if [ "$S3_S3V4" = "yes" ]; then
aws configure set default.s3.signature_version s3v4
fi
if [ "${SCHEDULE}" = "**None**" ]; then
sh backup.sh
if [ -z "$SCHEDULE" ]; then
echo "You need to set the SCHEDULE environment variable."
else
exec go-cron "$SCHEDULE" /bin/sh backup.sh
fi