diff --git a/backup/10.Dockerfile b/10.Dockerfile similarity index 82% rename from backup/10.Dockerfile rename to 10.Dockerfile index 651a19d..49b0e27 100644 --- a/backup/10.Dockerfile +++ b/10.Dockerfile @@ -3,7 +3,7 @@ FROM alpine:3.8 -ADD install.sh install.sh +ADD src/install.sh install.sh RUN sh install.sh && rm install.sh ENV POSTGRES_DATABASE **None** @@ -20,8 +20,10 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +ENV DROP_PUBLIC no -ADD run.sh run.sh -ADD backup.sh backup.sh +ADD src/run.sh run.sh +ADD src/backup.sh backup.sh +ADD src/restore.sh restore.sh CMD ["sh", "run.sh"] diff --git a/backup/11.Dockerfile b/11.Dockerfile similarity index 82% rename from backup/11.Dockerfile rename to 11.Dockerfile index 7bcbb40..824f6bc 100644 --- a/backup/11.Dockerfile +++ b/11.Dockerfile @@ -3,7 +3,7 @@ FROM alpine:3.10 -ADD install.sh install.sh +ADD src/install.sh install.sh RUN sh install.sh && rm install.sh ENV POSTGRES_DATABASE **None** @@ -20,8 +20,10 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +ENV DROP_PUBLIC no -ADD run.sh run.sh -ADD backup.sh backup.sh +ADD src/run.sh run.sh +ADD src/backup.sh backup.sh +ADD src/restore.sh restore.sh CMD ["sh", "run.sh"] diff --git a/backup/12.Dockerfile b/12.Dockerfile similarity index 82% rename from backup/12.Dockerfile rename to 12.Dockerfile index 49e3768..95ea474 100644 --- a/backup/12.Dockerfile +++ b/12.Dockerfile @@ -3,7 +3,7 @@ FROM alpine:edge -ADD install.sh install.sh +ADD src/install.sh install.sh RUN sh install.sh && rm install.sh ENV POSTGRES_DATABASE **None** @@ -20,8 +20,10 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +ENV DROP_PUBLIC no -ADD run.sh run.sh -ADD backup.sh backup.sh +ADD src/run.sh run.sh +ADD src/backup.sh backup.sh +ADD src/restore.sh restore.sh CMD ["sh", "run.sh"] diff --git a/backup/9.Dockerfile b/9.Dockerfile similarity index 82% rename from backup/9.Dockerfile rename to 9.Dockerfile index af8963d..34b7409 100644 --- a/backup/9.Dockerfile +++ b/9.Dockerfile @@ -3,7 +3,7 @@ FROM alpine:3.6 -ADD install.sh install.sh +ADD src/install.sh install.sh RUN sh install.sh && rm install.sh ENV POSTGRES_DATABASE **None** @@ -20,8 +20,10 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +ENV DROP_PUBLIC no -ADD run.sh run.sh -ADD backup.sh backup.sh +ADD src/run.sh run.sh +ADD src/backup.sh backup.sh +ADD src/restore.sh restore.sh CMD ["sh", "run.sh"] diff --git a/README.md b/README.md index f4405d3..b95764f 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,61 @@ Fork goals: - [x] automated builds - [x] support multiple PostgreSQL versions - [ ] support encrypted (password-protected) backups - - [ ] merge backup and restore images? + - [x] merge backup and restore images? + +------- + +# Usage +## Backup + +### Docker +```sh +$ docker run \ + -e S3_ACCESS_KEY_ID=key \ + -e S3_SECRET_ACCESS_KEY=secret \ + -e S3_BUCKET=my-bucket \ + -e S3_PREFIX=backup \ + -e POSTGRES_DATABASE=dbname \ + -e POSTGRES_USER=user \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_HOST=localhost \ + eeshugerman/postgres-backup-s3 +``` + +### Docker Compose +```yaml +postgres: + image: postgres + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + +pgbackups3: + image: eeshugerman/postgres-backup-s3 + container_name: pg-backup + links: + - postgres + environment: + SCHEDULE: '@daily' + S3_REGION: region + S3_ACCESS_KEY_ID: key + S3_SECRET_ACCESS_KEY: secret + S3_BUCKET: my-bucket + S3_PREFIX: backup + POSTGRES_DATABASE: dbname + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_EXTRA_OPTS: '--schema=public --blobs' +``` + +### Automatic Periodic Backups + +You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE="@daily"` to run the backup automatically. + +More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). + +## Restore +With the container running, +```sh +docker exec sh restore.sh +``` diff --git a/backup/README.md b/backup/README.md deleted file mode 100644 index a5339de..0000000 --- a/backup/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# postgres-backup-s3 - -Backup PostgresSQL to S3 (supports periodic backups) - -## Usage - -Docker: -```sh -$ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET=my-bucket -e S3_PREFIX=backup -e POSTGRES_DATABASE=dbname -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost schickling/postgres-backup-s3 -``` - -Docker Compose: -```yaml -postgres: - image: postgres - environment: - POSTGRES_USER: user - POSTGRES_PASSWORD: password - -pgbackups3: - image: schickling/postgres-backup-s3 - links: - - postgres - environment: - SCHEDULE: '@daily' - S3_REGION: region - S3_ACCESS_KEY_ID: key - S3_SECRET_ACCESS_KEY: secret - S3_BUCKET: my-bucket - S3_PREFIX: backup - POSTGRES_DATABASE: dbname - POSTGRES_USER: user - POSTGRES_PASSWORD: password - POSTGRES_EXTRA_OPTS: '--schema=public --blobs' -``` - -### Automatic Periodic Backups - -You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE="@daily"` to run the backup automatically. - -More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). - diff --git a/render.py b/render.py index cc08955..3dbbbb5 100644 --- a/render.py +++ b/render.py @@ -8,13 +8,13 @@ VERSIONS = ( ) -def render(dir_, postgres_version, alpine_version): - with open(f'{dir_}/template.Dockerfile') as f: +def render(postgres_version, alpine_version): + with open(f'template.Dockerfile') as f: template = f.read() rendered = template.format(alpine_version=alpine_version) - with open(f'{dir_}/{postgres_version}.Dockerfile', 'w') as f: + with open(f'{postgres_version}.Dockerfile', 'w') as f: f.write('# This file is generated from template.Dockerfile. Do not edit it directly.\n') f.write('###########################################################################\n\n') f.write(rendered) @@ -22,5 +22,4 @@ def render(dir_, postgres_version, alpine_version): if __name__ == '__main__': for versions in VERSIONS: - render('backup', *versions) - render('restore', *versions) + render(*versions) diff --git a/restore/10.Dockerfile b/restore/10.Dockerfile deleted file mode 100644 index 59e090e..0000000 --- a/restore/10.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# This file is generated from template.Dockerfile. Do not edit it directly. -########################################################################### - -FROM alpine:3.8 - -ADD install.sh install.sh -RUN sh install.sh && rm install.sh - -ENV POSTGRES_DATABASE **None** -ENV POSTGRES_HOST **None** -ENV POSTGRES_PORT 5432 -ENV POSTGRES_USER **None** -ENV POSTGRES_PASSWORD **None** -ENV S3_ACCESS_KEY_ID **None** -ENV S3_SECRET_ACCESS_KEY **None** -ENV S3_BUCKET **None** -ENV S3_REGION us-west-1 -ENV S3_PATH 'backup' -ENV DROP_PUBLIC 'no' - -ADD restore.sh restore.sh - -CMD ["sh", "restore.sh"] diff --git a/restore/11.Dockerfile b/restore/11.Dockerfile deleted file mode 100644 index 79946b1..0000000 --- a/restore/11.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# This file is generated from template.Dockerfile. Do not edit it directly. -########################################################################### - -FROM alpine:3.10 - -ADD install.sh install.sh -RUN sh install.sh && rm install.sh - -ENV POSTGRES_DATABASE **None** -ENV POSTGRES_HOST **None** -ENV POSTGRES_PORT 5432 -ENV POSTGRES_USER **None** -ENV POSTGRES_PASSWORD **None** -ENV S3_ACCESS_KEY_ID **None** -ENV S3_SECRET_ACCESS_KEY **None** -ENV S3_BUCKET **None** -ENV S3_REGION us-west-1 -ENV S3_PATH 'backup' -ENV DROP_PUBLIC 'no' - -ADD restore.sh restore.sh - -CMD ["sh", "restore.sh"] diff --git a/restore/12.Dockerfile b/restore/12.Dockerfile deleted file mode 100644 index 2fa28e0..0000000 --- a/restore/12.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# This file is generated from template.Dockerfile. Do not edit it directly. -########################################################################### - -FROM alpine:edge - -ADD install.sh install.sh -RUN sh install.sh && rm install.sh - -ENV POSTGRES_DATABASE **None** -ENV POSTGRES_HOST **None** -ENV POSTGRES_PORT 5432 -ENV POSTGRES_USER **None** -ENV POSTGRES_PASSWORD **None** -ENV S3_ACCESS_KEY_ID **None** -ENV S3_SECRET_ACCESS_KEY **None** -ENV S3_BUCKET **None** -ENV S3_REGION us-west-1 -ENV S3_PATH 'backup' -ENV DROP_PUBLIC 'no' - -ADD restore.sh restore.sh - -CMD ["sh", "restore.sh"] diff --git a/restore/9.Dockerfile b/restore/9.Dockerfile deleted file mode 100644 index 5d75573..0000000 --- a/restore/9.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# This file is generated from template.Dockerfile. Do not edit it directly. -########################################################################### - -FROM alpine:3.6 - -ADD install.sh install.sh -RUN sh install.sh && rm install.sh - -ENV POSTGRES_DATABASE **None** -ENV POSTGRES_HOST **None** -ENV POSTGRES_PORT 5432 -ENV POSTGRES_USER **None** -ENV POSTGRES_PASSWORD **None** -ENV S3_ACCESS_KEY_ID **None** -ENV S3_SECRET_ACCESS_KEY **None** -ENV S3_BUCKET **None** -ENV S3_REGION us-west-1 -ENV S3_PATH 'backup' -ENV DROP_PUBLIC 'no' - -ADD restore.sh restore.sh - -CMD ["sh", "restore.sh"] diff --git a/restore/README.md b/restore/README.md deleted file mode 100644 index 98b0644..0000000 --- a/restore/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# postgres-restore-s3 - -Restore a SQL backup from S3 to PostgresSQL - -## Warning - -This will potentially put your database in a very bad state or complete destroy your data, be very careful. - -## Limitations - -This is made to restore a backup made from postgres-backup-s3, if you backup came from somewhere else please check your format. - -* Your s3 bucket *must* only contain backups which you wish to restore - it will always grabs the 'latest' based on unix sort with no filtering -* They must be gzip encoded text sql files -* If your bucket has more than a 1000 files the latest may not be restore, only one s3 ls command is made - -## Usage - -Docker: -```sh -$ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET=my-bucket -e S3_PREFIX=backup -e POSTGRES_DATABASE=dbname -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost schickling/postgres-restore-s3 -``` - -## Dropping public - -If you wish to drop the public schema (drop schema public cascade; create schema public) then set the environment variable DROP_PUBLIC=yes. This is useful for situations where you wish to restore a database which currently has data / schemas in it. - diff --git a/restore/install.sh b/restore/install.sh deleted file mode 100644 index be00ab0..0000000 --- a/restore/install.sh +++ /dev/null @@ -1,17 +0,0 @@ -#! /bin/sh - -# exit if a command fails -set -e - -apk update - -# install pg_dump -apk add postgresql - -# install s3 tools -apk add python py-pip -pip install awscli -apk del py-pip - -# cleanup -rm -rf /var/cache/apk/* diff --git a/restore/template.Dockerfile b/restore/template.Dockerfile deleted file mode 100644 index 1eeee1c..0000000 --- a/restore/template.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM alpine:{alpine_version} - -ADD install.sh install.sh -RUN sh install.sh && rm install.sh - -ENV POSTGRES_DATABASE **None** -ENV POSTGRES_HOST **None** -ENV POSTGRES_PORT 5432 -ENV POSTGRES_USER **None** -ENV POSTGRES_PASSWORD **None** -ENV S3_ACCESS_KEY_ID **None** -ENV S3_SECRET_ACCESS_KEY **None** -ENV S3_BUCKET **None** -ENV S3_REGION us-west-1 -ENV S3_PATH 'backup' -ENV DROP_PUBLIC 'no' - -ADD restore.sh restore.sh - -CMD ["sh", "restore.sh"] diff --git a/backup/backup.sh b/src/backup.sh similarity index 100% rename from backup/backup.sh rename to src/backup.sh diff --git a/backup/install.sh b/src/install.sh similarity index 84% rename from backup/install.sh rename to src/install.sh index 16c7119..8d4f00a 100644 --- a/backup/install.sh +++ b/src/install.sh @@ -3,16 +3,15 @@ # exit if a command fails set -e - apk update # install pg_dump -apk add postgresql +apk add postgresql-client # install s3 tools -apk add python py2-pip +apk add python py-pip pip install awscli -apk del py2-pip +apk del py-pip # install go-cron apk add curl diff --git a/restore/restore.sh b/src/restore.sh similarity index 85% rename from restore/restore.sh rename to src/restore.sh index 495c237..a433d7a 100644 --- a/restore/restore.sh +++ b/src/restore.sh @@ -43,6 +43,12 @@ if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then exit 1 fi +if [ "${S3_ENDPOINT}" == "**None**" ]; then + AWS_ARGS="" +else + AWS_ARGS="--endpoint-url ${S3_ENDPOINT}" +fi + # env vars needed for aws tools export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY @@ -53,11 +59,11 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER" echo "Finding latest backup" -LATEST_BACKUP=$(aws s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }') +LATEST_BACKUP=$(aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }') echo "Fetching ${LATEST_BACKUP} from S3" -aws s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz +aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz gzip -d dump.sql.gz if [ "${DROP_PUBLIC}" == "yes" ]; then @@ -69,5 +75,7 @@ echo "Restoring ${LATEST_BACKUP}" psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE < dump.sql +rm dump.sql + echo "Restore complete" diff --git a/backup/run.sh b/src/run.sh similarity index 100% rename from backup/run.sh rename to src/run.sh diff --git a/backup/template.Dockerfile b/template.Dockerfile similarity index 77% rename from backup/template.Dockerfile rename to template.Dockerfile index 55d68b4..55ce6c8 100644 --- a/backup/template.Dockerfile +++ b/template.Dockerfile @@ -1,6 +1,6 @@ FROM alpine:{alpine_version} -ADD install.sh install.sh +ADD src/install.sh install.sh RUN sh install.sh && rm install.sh ENV POSTGRES_DATABASE **None** @@ -17,8 +17,10 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +ENV DROP_PUBLIC no -ADD run.sh run.sh -ADD backup.sh backup.sh +ADD src/run.sh run.sh +ADD src/backup.sh backup.sh +ADD src/restore.sh restore.sh CMD ["sh", "run.sh"]