romein/docker-volume-s3-backup
1
0
Fork 0
mirror of https://github.com/garraflavatra/docker-volume-s3-backup.git synced 2025-05-17 13:14:38 +00:00
Code Issues

merge restore and util containers

Browse Source
This commit is contained in:
Elliott Shugerman 2019-11-27 20:57:21 -07:00
parent 66a9a2b219
commit 3e05233a2a
19 changed files with 100 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backup/10.Dockerfile → 10.Dockerfile
View File

@ -3,7 +3,7 @@
FROM alpine:3.8
ADD install.sh install.sh
ADD src/install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
@ -20,8 +20,10 @@ ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ADD run.sh run.sh
ADD backup.sh backup.sh
ADD src/run.sh run.sh
ADD src/backup.sh backup.sh
ADD src/restore.sh restore.sh
CMD ["sh", "run.sh"]

8
backup/11.Dockerfile → 11.Dockerfile
View File

@ -3,7 +3,7 @@
FROM alpine:3.10
ADD install.sh install.sh
ADD src/install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
@ -20,8 +20,10 @@ ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ADD run.sh run.sh
ADD backup.sh backup.sh
ADD src/run.sh run.sh
ADD src/backup.sh backup.sh
ADD src/restore.sh restore.sh
CMD ["sh", "run.sh"]

8
backup/12.Dockerfile → 12.Dockerfile
View File

@ -3,7 +3,7 @@
FROM alpine:edge
ADD install.sh install.sh
ADD src/install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
@ -20,8 +20,10 @@ ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ADD run.sh run.sh
ADD backup.sh backup.sh
ADD src/run.sh run.sh
ADD src/backup.sh backup.sh
ADD src/restore.sh restore.sh
CMD ["sh", "run.sh"]

8
backup/9.Dockerfile → 9.Dockerfile
View File

@ -3,7 +3,7 @@
FROM alpine:3.6
ADD install.sh install.sh
ADD src/install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
@ -20,8 +20,10 @@ ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ADD run.sh run.sh
ADD backup.sh backup.sh
ADD src/run.sh run.sh
ADD src/backup.sh backup.sh
ADD src/restore.sh restore.sh
CMD ["sh", "run.sh"]

59
README.md
View File

@ -7,4 +7,61 @@ Fork goals:
- [x] automated builds
- [x] support multiple PostgreSQL versions
- [ ] support encrypted (password-protected) backups
- [ ] merge backup and restore images?
- [x] merge backup and restore images?
-------
# Usage
## Backup
### Docker
```sh
$ docker run \
-e S3_ACCESS_KEY_ID=key \
-e S3_SECRET_ACCESS_KEY=secret \
-e S3_BUCKET=my-bucket \
-e S3_PREFIX=backup \
-e POSTGRES_DATABASE=dbname \
-e POSTGRES_USER=user \
-e POSTGRES_PASSWORD=password \
-e POSTGRES_HOST=localhost \
eeshugerman/postgres-backup-s3
```
### Docker Compose
```yaml
postgres:
image: postgres
environment:
POSTGRES_USER: user
POSTGRES_PASSWORD: password
pgbackups3:
image: eeshugerman/postgres-backup-s3
container_name: pg-backup
links:
- postgres
environment:
SCHEDULE: '@daily'
S3_REGION: region
S3_ACCESS_KEY_ID: key
S3_SECRET_ACCESS_KEY: secret
S3_BUCKET: my-bucket
S3_PREFIX: backup
POSTGRES_DATABASE: dbname
POSTGRES_USER: user
POSTGRES_PASSWORD: password
POSTGRES_EXTRA_OPTS: '--schema=public --blobs'
```
### Automatic Periodic Backups
You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE="@daily"` to run the backup automatically.
More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules).
## Restore
With the container running,
```sh
docker exec <container name> sh restore.sh
```

42
backup/README.md
View File

@ -1,42 +0,0 @@
# postgres-backup-s3
Backup PostgresSQL to S3 (supports periodic backups)
## Usage
Docker:
```sh
$ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET=my-bucket -e S3_PREFIX=backup -e POSTGRES_DATABASE=dbname -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost schickling/postgres-backup-s3
```
Docker Compose:
```yaml
postgres:
image: postgres
environment:
POSTGRES_USER: user
POSTGRES_PASSWORD: password
pgbackups3:
image: schickling/postgres-backup-s3
links:
- postgres
environment:
SCHEDULE: '@daily'
S3_REGION: region
S3_ACCESS_KEY_ID: key
S3_SECRET_ACCESS_KEY: secret
S3_BUCKET: my-bucket
S3_PREFIX: backup
POSTGRES_DATABASE: dbname
POSTGRES_USER: user
POSTGRES_PASSWORD: password
POSTGRES_EXTRA_OPTS: '--schema=public --blobs'
```
### Automatic Periodic Backups
You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE="@daily"` to run the backup automatically.
More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules).

9
render.py
View File

@ -8,13 +8,13 @@ VERSIONS = (
)
def render(dir_, postgres_version, alpine_version):
with open(f'{dir_}/template.Dockerfile') as f:
def render(postgres_version, alpine_version):
with open(f'template.Dockerfile') as f:
template = f.read()
rendered = template.format(alpine_version=alpine_version)
with open(f'{dir_}/{postgres_version}.Dockerfile', 'w') as f:
with open(f'{postgres_version}.Dockerfile', 'w') as f:
f.write('# This file is generated from template.Dockerfile. Do not edit it directly.\n')
f.write('###########################################################################\n\n')
f.write(rendered)
@ -22,5 +22,4 @@ def render(dir_, postgres_version, alpine_version):
if __name__ == '__main__':
for versions in VERSIONS:
render('backup', *versions)
render('restore', *versions)
render(*versions)

23
restore/10.Dockerfile
View File

@ -1,23 +0,0 @@
# This file is generated from template.Dockerfile. Do not edit it directly.
###########################################################################
FROM alpine:3.8
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV S3_PATH 'backup'
ENV DROP_PUBLIC 'no'
ADD restore.sh restore.sh
CMD ["sh", "restore.sh"]

23
restore/11.Dockerfile
View File

@ -1,23 +0,0 @@
# This file is generated from template.Dockerfile. Do not edit it directly.
###########################################################################
FROM alpine:3.10
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV S3_PATH 'backup'
ENV DROP_PUBLIC 'no'
ADD restore.sh restore.sh
CMD ["sh", "restore.sh"]

23
restore/12.Dockerfile
View File

@ -1,23 +0,0 @@
# This file is generated from template.Dockerfile. Do not edit it directly.
###########################################################################
FROM alpine:edge
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV S3_PATH 'backup'
ENV DROP_PUBLIC 'no'
ADD restore.sh restore.sh
CMD ["sh", "restore.sh"]

23
restore/9.Dockerfile
View File

@ -1,23 +0,0 @@
# This file is generated from template.Dockerfile. Do not edit it directly.
###########################################################################
FROM alpine:3.6
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV S3_PATH 'backup'
ENV DROP_PUBLIC 'no'
ADD restore.sh restore.sh
CMD ["sh", "restore.sh"]

27
restore/README.md
View File

@ -1,27 +0,0 @@
# postgres-restore-s3
Restore a SQL backup from S3 to PostgresSQL
## Warning
This will potentially put your database in a very bad state or complete destroy your data, be very careful.
## Limitations
This is made to restore a backup made from postgres-backup-s3, if you backup came from somewhere else please check your format.
* Your s3 bucket *must* only contain backups which you wish to restore - it will always grabs the 'latest' based on unix sort with no filtering
* They must be gzip encoded text sql files
* If your bucket has more than a 1000 files the latest may not be restore, only one s3 ls command is made
## Usage
Docker:
```sh
$ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET=my-bucket -e S3_PREFIX=backup -e POSTGRES_DATABASE=dbname -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password -e POSTGRES_HOST=localhost schickling/postgres-restore-s3
```
## Dropping public
If you wish to drop the public schema (drop schema public cascade; create schema public) then set the environment variable DROP_PUBLIC=yes. This is useful for situations where you wish to restore a database which currently has data / schemas in it.

17
restore/install.sh
View File

@ -1,17 +0,0 @@
#! /bin/sh
# exit if a command fails
set -e
apk update
# install pg_dump
apk add postgresql
# install s3 tools
apk add python py-pip
pip install awscli
apk del py-pip
# cleanup
rm -rf /var/cache/apk/*

20
restore/template.Dockerfile
View File

@ -1,20 +0,0 @@
FROM alpine:{alpine_version}
ADD install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
ENV POSTGRES_HOST **None**
ENV POSTGRES_PORT 5432
ENV POSTGRES_USER **None**
ENV POSTGRES_PASSWORD **None**
ENV S3_ACCESS_KEY_ID **None**
ENV S3_SECRET_ACCESS_KEY **None**
ENV S3_BUCKET **None**
ENV S3_REGION us-west-1
ENV S3_PATH 'backup'
ENV DROP_PUBLIC 'no'
ADD restore.sh restore.sh
CMD ["sh", "restore.sh"]

0
backup/backup.sh → src/backup.sh
View File

7
backup/install.sh → src/install.sh
View File

@ -3,16 +3,15 @@
# exit if a command fails
set -e
apk update
# install pg_dump
apk add postgresql
apk add postgresql-client
# install s3 tools
apk add python py2-pip
apk add python py-pip
pip install awscli
apk del py2-pip
apk del py-pip
# install go-cron
apk add curl

12
restore/restore.sh → src/restore.sh
View File

@ -43,6 +43,12 @@ if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then
exit 1
fi
if [ "${S3_ENDPOINT}" == "**None**" ]; then
AWS_ARGS=""
else
AWS_ARGS="--endpoint-url ${S3_ENDPOINT}"
fi
# env vars needed for aws tools
export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY
@ -53,11 +59,11 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER"
echo "Finding latest backup"
LATEST_BACKUP=$(aws s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }')
LATEST_BACKUP=$(aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }')
echo "Fetching ${LATEST_BACKUP} from S3"
aws s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz
aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz
gzip -d dump.sql.gz
if [ "${DROP_PUBLIC}" == "yes" ]; then
@ -69,5 +75,7 @@ echo "Restoring ${LATEST_BACKUP}"
psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE < dump.sql
rm dump.sql
echo "Restore complete"

0
backup/run.sh → src/run.sh
View File

8
backup/template.Dockerfile → template.Dockerfile
View File

@ -1,6 +1,6 @@
FROM alpine:{alpine_version}
ADD install.sh install.sh
ADD src/install.sh install.sh
RUN sh install.sh && rm install.sh
ENV POSTGRES_DATABASE **None**
@ -17,8 +17,10 @@ ENV S3_PATH 'backup'
ENV S3_ENDPOINT **None**
ENV S3_S3V4 no
ENV SCHEDULE **None**
ENV DROP_PUBLIC no
ADD run.sh run.sh
ADD backup.sh backup.sh
ADD src/run.sh run.sh
ADD src/backup.sh backup.sh
ADD src/restore.sh restore.sh
CMD ["sh", "run.sh"]