commit 66a9a2b21967f1947593e672e056329adc221150 Author: Elliott Shugerman Date: Wed Nov 27 17:05:27 2019 -0700 initial commit diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d1988db --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Johannes Schickling + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f4405d3 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +This is a fork and restructuring of schickling's [postgres-backup-s3](https://github.com/schickling/dockerfiles/tree/master/postgres-backup-s3) and [postgres-restore-s3](https://github.com/schickling/dockerfiles/tree/master/postgres-restore-s3). + +See [`backup/README.md`](/backup/README.md) and [`restore/README.md`](/restore/README.md) for further instructions. + +Fork goals: + - [x] dedicated repository + - [x] automated builds + - [x] support multiple PostgreSQL versions + - [ ] support encrypted (password-protected) backups + - [ ] merge backup and restore images? diff --git a/backup/10.Dockerfile b/backup/10.Dockerfile new file mode 100644 index 0000000..651a19d --- /dev/null +++ b/backup/10.Dockerfile @@ -0,0 +1,27 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:3.8 + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV POSTGRES_EXTRA_OPTS '' +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV S3_ENDPOINT **None** +ENV S3_S3V4 no +ENV SCHEDULE **None** + +ADD run.sh run.sh +ADD backup.sh backup.sh + +CMD ["sh", "run.sh"] diff --git a/backup/11.Dockerfile b/backup/11.Dockerfile new file mode 100644 index 0000000..7bcbb40 --- /dev/null +++ b/backup/11.Dockerfile @@ -0,0 +1,27 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:3.10 + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV POSTGRES_EXTRA_OPTS '' +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV S3_ENDPOINT **None** +ENV S3_S3V4 no +ENV SCHEDULE **None** + +ADD run.sh run.sh +ADD backup.sh backup.sh + +CMD ["sh", "run.sh"] diff --git a/backup/12.Dockerfile b/backup/12.Dockerfile new file mode 100644 index 0000000..49e3768 --- /dev/null +++ b/backup/12.Dockerfile @@ -0,0 +1,27 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:edge + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV POSTGRES_EXTRA_OPTS '' +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV S3_ENDPOINT **None** +ENV S3_S3V4 no +ENV SCHEDULE **None** + +ADD run.sh run.sh +ADD backup.sh backup.sh + +CMD ["sh", "run.sh"] diff --git a/backup/9.Dockerfile b/backup/9.Dockerfile new file mode 100644 index 0000000..af8963d --- /dev/null +++ b/backup/9.Dockerfile @@ -0,0 +1,27 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:3.6 + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV POSTGRES_EXTRA_OPTS '' +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV S3_ENDPOINT **None** +ENV S3_S3V4 no +ENV SCHEDULE **None** + +ADD run.sh run.sh +ADD backup.sh backup.sh + +CMD ["sh", "run.sh"] diff --git a/backup/README.md b/backup/README.md new file mode 100644 index 0000000..a5339de --- /dev/null +++ b/backup/README.md @@ -0,0 +1,42 @@ +# postgres-backup-s3 + +Backup PostgresSQL to S3 (supports periodic backups) + +## Usage + +Docker: +```sh +$ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET=my-bucket -e S3_PREFIX=backup -e POSTGRES_DATABASE=dbname -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost schickling/postgres-backup-s3 +``` + +Docker Compose: +```yaml +postgres: + image: postgres + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + +pgbackups3: + image: schickling/postgres-backup-s3 + links: + - postgres + environment: + SCHEDULE: '@daily' + S3_REGION: region + S3_ACCESS_KEY_ID: key + S3_SECRET_ACCESS_KEY: secret + S3_BUCKET: my-bucket + S3_PREFIX: backup + POSTGRES_DATABASE: dbname + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_EXTRA_OPTS: '--schema=public --blobs' +``` + +### Automatic Periodic Backups + +You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE="@daily"` to run the backup automatically. + +More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). + diff --git a/backup/backup.sh b/backup/backup.sh new file mode 100644 index 0000000..6e5a7f0 --- /dev/null +++ b/backup/backup.sh @@ -0,0 +1,68 @@ +#! /bin/sh + +set -e +set -o pipefail + +if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then + echo "You need to set the S3_ACCESS_KEY_ID environment variable." + exit 1 +fi + +if [ "${S3_SECRET_ACCESS_KEY}" = "**None**" ]; then + echo "You need to set the S3_SECRET_ACCESS_KEY environment variable." + exit 1 +fi + +if [ "${S3_BUCKET}" = "**None**" ]; then + echo "You need to set the S3_BUCKET environment variable." + exit 1 +fi + +if [ "${POSTGRES_DATABASE}" = "**None**" ]; then + echo "You need to set the POSTGRES_DATABASE environment variable." + exit 1 +fi + +if [ "${POSTGRES_HOST}" = "**None**" ]; then + if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then + POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR + POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT + else + echo "You need to set the POSTGRES_HOST environment variable." + exit 1 + fi +fi + +if [ "${POSTGRES_USER}" = "**None**" ]; then + echo "You need to set the POSTGRES_USER environment variable." + exit 1 +fi + +if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then + echo "You need to set the POSTGRES_PASSWORD environment variable or link to a container named POSTGRES." + exit 1 +fi + +if [ "${S3_ENDPOINT}" == "**None**" ]; then + AWS_ARGS="" +else + AWS_ARGS="--endpoint-url ${S3_ENDPOINT}" +fi + +# env vars needed for aws tools +export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY +export AWS_DEFAULT_REGION=$S3_REGION + +export PGPASSWORD=$POSTGRES_PASSWORD +POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTGRES_EXTRA_OPTS" + +echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..." + +pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | gzip > dump.sql.gz + +echo "Uploading dump to $S3_BUCKET" + +cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2 + +echo "SQL backup uploaded successfully" diff --git a/backup/install.sh b/backup/install.sh new file mode 100644 index 0000000..16c7119 --- /dev/null +++ b/backup/install.sh @@ -0,0 +1,25 @@ +#! /bin/sh + +# exit if a command fails +set -e + + +apk update + +# install pg_dump +apk add postgresql + +# install s3 tools +apk add python py2-pip +pip install awscli +apk del py2-pip + +# install go-cron +apk add curl +curl -L --insecure https://github.com/odise/go-cron/releases/download/v0.0.6/go-cron-linux.gz | zcat > /usr/local/bin/go-cron +chmod u+x /usr/local/bin/go-cron +apk del curl + + +# cleanup +rm -rf /var/cache/apk/* diff --git a/backup/run.sh b/backup/run.sh new file mode 100644 index 0000000..e3fb3c8 --- /dev/null +++ b/backup/run.sh @@ -0,0 +1,13 @@ +#! /bin/sh + +set -e + +if [ "${S3_S3V4}" = "yes" ]; then + aws configure set default.s3.signature_version s3v4 +fi + +if [ "${SCHEDULE}" = "**None**" ]; then + sh backup.sh +else + exec go-cron "$SCHEDULE" /bin/sh backup.sh +fi diff --git a/backup/template.Dockerfile b/backup/template.Dockerfile new file mode 100644 index 0000000..55d68b4 --- /dev/null +++ b/backup/template.Dockerfile @@ -0,0 +1,24 @@ +FROM alpine:{alpine_version} + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV POSTGRES_EXTRA_OPTS '' +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV S3_ENDPOINT **None** +ENV S3_S3V4 no +ENV SCHEDULE **None** + +ADD run.sh run.sh +ADD backup.sh backup.sh + +CMD ["sh", "run.sh"] diff --git a/render.py b/render.py new file mode 100644 index 0000000..cc08955 --- /dev/null +++ b/render.py @@ -0,0 +1,26 @@ +#!/bin/python3 + +VERSIONS = ( + ('9', '3.6'), + ('10', '3.8'), + ('11', '3.10'), + ('12', 'edge'), +) + + +def render(dir_, postgres_version, alpine_version): + with open(f'{dir_}/template.Dockerfile') as f: + template = f.read() + + rendered = template.format(alpine_version=alpine_version) + + with open(f'{dir_}/{postgres_version}.Dockerfile', 'w') as f: + f.write('# This file is generated from template.Dockerfile. Do not edit it directly.\n') + f.write('###########################################################################\n\n') + f.write(rendered) + + +if __name__ == '__main__': + for versions in VERSIONS: + render('backup', *versions) + render('restore', *versions) diff --git a/restore/10.Dockerfile b/restore/10.Dockerfile new file mode 100644 index 0000000..59e090e --- /dev/null +++ b/restore/10.Dockerfile @@ -0,0 +1,23 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:3.8 + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV DROP_PUBLIC 'no' + +ADD restore.sh restore.sh + +CMD ["sh", "restore.sh"] diff --git a/restore/11.Dockerfile b/restore/11.Dockerfile new file mode 100644 index 0000000..79946b1 --- /dev/null +++ b/restore/11.Dockerfile @@ -0,0 +1,23 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:3.10 + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV DROP_PUBLIC 'no' + +ADD restore.sh restore.sh + +CMD ["sh", "restore.sh"] diff --git a/restore/12.Dockerfile b/restore/12.Dockerfile new file mode 100644 index 0000000..2fa28e0 --- /dev/null +++ b/restore/12.Dockerfile @@ -0,0 +1,23 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:edge + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV DROP_PUBLIC 'no' + +ADD restore.sh restore.sh + +CMD ["sh", "restore.sh"] diff --git a/restore/9.Dockerfile b/restore/9.Dockerfile new file mode 100644 index 0000000..5d75573 --- /dev/null +++ b/restore/9.Dockerfile @@ -0,0 +1,23 @@ +# This file is generated from template.Dockerfile. Do not edit it directly. +########################################################################### + +FROM alpine:3.6 + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV DROP_PUBLIC 'no' + +ADD restore.sh restore.sh + +CMD ["sh", "restore.sh"] diff --git a/restore/README.md b/restore/README.md new file mode 100644 index 0000000..98b0644 --- /dev/null +++ b/restore/README.md @@ -0,0 +1,27 @@ +# postgres-restore-s3 + +Restore a SQL backup from S3 to PostgresSQL + +## Warning + +This will potentially put your database in a very bad state or complete destroy your data, be very careful. + +## Limitations + +This is made to restore a backup made from postgres-backup-s3, if you backup came from somewhere else please check your format. + +* Your s3 bucket *must* only contain backups which you wish to restore - it will always grabs the 'latest' based on unix sort with no filtering +* They must be gzip encoded text sql files +* If your bucket has more than a 1000 files the latest may not be restore, only one s3 ls command is made + +## Usage + +Docker: +```sh +$ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET=my-bucket -e S3_PREFIX=backup -e POSTGRES_DATABASE=dbname -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost schickling/postgres-restore-s3 +``` + +## Dropping public + +If you wish to drop the public schema (drop schema public cascade; create schema public) then set the environment variable DROP_PUBLIC=yes. This is useful for situations where you wish to restore a database which currently has data / schemas in it. + diff --git a/restore/install.sh b/restore/install.sh new file mode 100644 index 0000000..be00ab0 --- /dev/null +++ b/restore/install.sh @@ -0,0 +1,17 @@ +#! /bin/sh + +# exit if a command fails +set -e + +apk update + +# install pg_dump +apk add postgresql + +# install s3 tools +apk add python py-pip +pip install awscli +apk del py-pip + +# cleanup +rm -rf /var/cache/apk/* diff --git a/restore/restore.sh b/restore/restore.sh new file mode 100644 index 0000000..495c237 --- /dev/null +++ b/restore/restore.sh @@ -0,0 +1,73 @@ +#! /bin/sh + +set -e +set -o pipefail + +if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then + echo "You need to set the S3_ACCESS_KEY_ID environment variable." + exit 1 +fi + +if [ "${S3_SECRET_ACCESS_KEY}" = "**None**" ]; then + echo "You need to set the S3_SECRET_ACCESS_KEY environment variable." + exit 1 +fi + +if [ "${S3_BUCKET}" = "**None**" ]; then + echo "You need to set the S3_BUCKET environment variable." + exit 1 +fi + +if [ "${POSTGRES_DATABASE}" = "**None**" ]; then + echo "You need to set the POSTGRES_DATABASE environment variable." + exit 1 +fi + +if [ "${POSTGRES_HOST}" = "**None**" ]; then + if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then + POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR + POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT + else + echo "You need to set the POSTGRES_HOST environment variable." + exit 1 + fi +fi + +if [ "${POSTGRES_USER}" = "**None**" ]; then + echo "You need to set the POSTGRES_USER environment variable." + exit 1 +fi + +if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then + echo "You need to set the POSTGRES_PASSWORD environment variable or link to a container named POSTGRES." + exit 1 +fi + +# env vars needed for aws tools +export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY +export AWS_DEFAULT_REGION=$S3_REGION + +export PGPASSWORD=$POSTGRES_PASSWORD +POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER" + +echo "Finding latest backup" + +LATEST_BACKUP=$(aws s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }') + +echo "Fetching ${LATEST_BACKUP} from S3" + +aws s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz +gzip -d dump.sql.gz + +if [ "${DROP_PUBLIC}" == "yes" ]; then + echo "Recreating the public schema" + psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE -c "drop schema public cascade; create schema public;" +fi + +echo "Restoring ${LATEST_BACKUP}" + +psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE < dump.sql + +echo "Restore complete" + diff --git a/restore/template.Dockerfile b/restore/template.Dockerfile new file mode 100644 index 0000000..1eeee1c --- /dev/null +++ b/restore/template.Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:{alpine_version} + +ADD install.sh install.sh +RUN sh install.sh && rm install.sh + +ENV POSTGRES_DATABASE **None** +ENV POSTGRES_HOST **None** +ENV POSTGRES_PORT 5432 +ENV POSTGRES_USER **None** +ENV POSTGRES_PASSWORD **None** +ENV S3_ACCESS_KEY_ID **None** +ENV S3_SECRET_ACCESS_KEY **None** +ENV S3_BUCKET **None** +ENV S3_REGION us-west-1 +ENV S3_PATH 'backup' +ENV DROP_PUBLIC 'no' + +ADD restore.sh restore.sh + +CMD ["sh", "restore.sh"]