From 513f9af69f180902de4fe9703866a18d507ec73c Mon Sep 17 00:00:00 2001 From: Romein van Buren Date: Sun, 4 May 2025 13:18:07 +0200 Subject: [PATCH] Adapt postgres-baskup-s3 to backup Docker volumes instead of Postgres databases --- template.env => .env.example | 0 ...ild-and-push-images.yml => build-push.yml} | 17 +--- .gitignore | 5 +- Dockerfile | 18 ++-- LICENSE.txt | 2 + README.md | 87 +++++++++---------- docker-compose.yaml | 35 +++----- src/backup.sh | 24 ++--- src/env.sh | 37 +++----- src/install.sh | 15 +--- src/{restore.sh => restore__wip__.sh} | 2 +- src/run.sh | 2 +- test_image/Dockerfile | 3 + test_image/data/file | 1 + test_image/data/nested/file | 1 + 15 files changed, 100 insertions(+), 149 deletions(-) rename template.env => .env.example (100%) rename .github/workflows/{build-and-push-images.yml => build-push.yml} (60%) rename src/{restore.sh => restore__wip__.sh} (98%) create mode 100644 test_image/Dockerfile create mode 100644 test_image/data/file create mode 100644 test_image/data/nested/file diff --git a/template.env b/.env.example similarity index 100% rename from template.env rename to .env.example diff --git a/.github/workflows/build-and-push-images.yml b/.github/workflows/build-push.yml similarity index 60% rename from .github/workflows/build-and-push-images.yml rename to .github/workflows/build-push.yml index 6b89df2..020f910 100644 --- a/.github/workflows/build-and-push-images.yml +++ b/.github/workflows/build-push.yml @@ -1,22 +1,13 @@ -name: build and push images +name: Build and push image on: push: - branches: ['master'] + branches: ['main'] jobs: build-and-push-image: runs-on: ubuntu-latest - strategy: - matrix: - include: - - { postgres: 12, alpine: '3.12' } - - { postgres: 13, alpine: '3.14' } - - { postgres: 14, alpine: '3.16' } - - { postgres: 15, alpine: '3.17' } - - { postgres: 16, alpine: '3.19' } - steps: - name: Checkout repository uses: actions/checkout@v2 @@ -38,9 +29,7 @@ jobs: with: context: . push: true - tags: ${{ github.repository }}:${{ matrix.postgres }} - build-args: | - ALPINE_VERSION=${{ matrix.alpine }} + tags: ${{ github.repository }}:latest platforms: | linux/amd64 linux/arm64 diff --git a/.gitignore b/.gitignore index 3bf780b..a43464e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .idea -.env \ No newline at end of file +.env +.DS_Store +.env* +!.env.example diff --git a/Dockerfile b/Dockerfile index 16915d7..439d5b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,14 @@ -ARG ALPINE_VERSION -FROM alpine:${ALPINE_VERSION} -ARG TARGETARCH +FROM alpine:3.21 + +RUN mkdir /data ADD src/install.sh install.sh RUN sh install.sh && rm install.sh -ENV POSTGRES_DATABASE '' -ENV POSTGRES_HOST '' -ENV POSTGRES_PORT 5432 -ENV POSTGRES_USER '' -ENV POSTGRES_PASSWORD '' -ENV PGDUMP_EXTRA_OPTS '' ENV S3_ACCESS_KEY_ID '' ENV S3_SECRET_ACCESS_KEY '' +ENV S3_ACCESS_KEY '' +ENV S3_SECRET_KEY '' ENV S3_BUCKET '' ENV S3_REGION 'us-west-1' ENV S3_PATH 'backup' @@ -25,6 +21,6 @@ ENV BACKUP_KEEP_DAYS '' ADD src/run.sh run.sh ADD src/env.sh env.sh ADD src/backup.sh backup.sh -ADD src/restore.sh restore.sh +# ADD src/restore.sh restore.sh -- not ready yet -CMD ["sh", "run.sh"] +CMD ['sh', 'run.sh'] diff --git a/LICENSE.txt b/LICENSE.txt index d1988db..f954654 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,8 @@ MIT License Copyright (c) 2017 Johannes Schickling +Copyright (c) 2022 Elliott Shugerman +Copyright (c) 2025 Romein van Buren Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 67e2f99..20f218b 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,24 @@ -# Introduction -This project provides Docker images to periodically back up a PostgreSQL database to AWS S3, and to restore from the backup as needed. +# docker-volume-s3-backup + +## Introduction + +This project provides a Docker image that periodically backs up a Docker volume to AWS S3, and can restore a backup as needed. + +## Usage + +### Backup + +Example `docker-compose.yml`: -# Usage -## Backup ```yaml services: - postgres: - image: postgres:16 - environment: - POSTGRES_USER: user - POSTGRES_PASSWORD: password + my_service: + image: ... + volumes: + data:/app/some-data-dir - backup: - image: eeshugerman/postgres-backup-s3:16 + volume_backup: + image: smartyellow/docker-volume-s3-backup environment: SCHEDULE: '@weekly' # optional BACKUP_KEEP_DAYS: 7 # optional @@ -22,66 +28,53 @@ services: S3_SECRET_ACCESS_KEY: secret S3_BUCKET: my-bucket S3_PREFIX: backup - POSTGRES_HOST: postgres - POSTGRES_DATABASE: dbname - POSTGRES_USER: user - POSTGRES_PASSWORD: password ``` -- Images are tagged by the major PostgreSQL version supported: `12`, `13`, `14`, `15` or `16`. - The `SCHEDULE` variable determines backup frequency. See go-cron schedules documentation [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). Omit to run the backup immediately and then exit. - If `PASSPHRASE` is provided, the backup will be encrypted using GPG. -- Run `docker exec sh backup.sh` to trigger a backup ad-hoc. +- Run `docker exec sh backup.sh` to trigger a backup ad-hoc. - If `BACKUP_KEEP_DAYS` is set, backups older than this many days will be deleted from S3. - Set `S3_ENDPOINT` if you're using a non-AWS S3-compatible storage provider. -## Restore +### Restore + > [!CAUTION] > DATA LOSS! All database objects will be dropped and re-created. -### ... from latest backup +#### ... from latest backup + ```sh -docker exec sh restore.sh +docker exec sh restore.sh ``` > [!NOTE] > If your bucket has more than a 1000 files, the latest may not be restored -- only one S3 `ls` command is used -### ... from specific backup +#### ... from specific backup + ```sh -docker exec sh restore.sh +docker exec sh restore.sh ``` -# Development -## Build the image locally -`ALPINE_VERSION` determines Postgres version compatibility. See [`build-and-push-images.yml`](.github/workflows/build-and-push-images.yml) for the latest mapping. -```sh -DOCKER_BUILDKIT=1 docker build --build-arg ALPINE_VERSION=3.14 . -``` -## Run a simple test environment with Docker Compose +## Development + +### Run a simple test environment with Docker Compose ```sh cp template.env .env # fill out your secrets/params in .env docker compose up -d ``` -# Acknowledgements -This project is a fork and re-structuring of @schickling's [postgres-backup-s3](https://github.com/schickling/dockerfiles/tree/master/postgres-backup-s3) and [postgres-restore-s3](https://github.com/schickling/dockerfiles/tree/master/postgres-restore-s3). +## Acknowledgements -## Fork goals -These changes would have been difficult or impossible merge into @schickling's repo or similarly-structured forks. - - dedicated repository - - automated builds - - support multiple PostgreSQL versions - - backup and restore with one image +This project is a modification of the excellent [`eeshugerman/postgres-backup-s3`](https://github.com/eeshugerman/postgres-backup-s3), which in turn is a fork and re-structuring of [`schickling/postgres-backup-s3`](https://github.com/schickling/dockerfiles/tree/master/postgres-backup-s3) and [`schickling/postgres-restore-s3`](https://github.com/schickling/dockerfiles/tree/master/postgres-restore-s3). -## Other changes and features - - some environment variables renamed or removed - - uses `pg_dump`'s `custom` format (see [docs](https://www.postgresql.org/docs/10/app-pgdump.html)) - - drop and re-create all database objects on restore - - backup blobs and all schemas by default - - no Python 2 dependencies - - filter backups on S3 by database name - - support encrypted (password-protected) backups - - support for restoring from a specific backup by timestamp - - support for auto-removal of old backups +## Copyright + +Copyright (c) 2017 Johannes Schickling + +Copyright (c) 2022 Elliott Shugerman + +Copyright (c) 2025 [Romein van Buren](mailto:romein@smartyellow.nl) + +Licensed under the MIT license. diff --git a/docker-compose.yaml b/docker-compose.yaml index 16ac481..134056c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,28 +1,19 @@ -# this file is here to facilitate development/testing +# This file is here to facilitate development/testing. +# See the readme for a list of configuration options. +# Copy the .env.example and add in your S3 credentials. # $ docker compose up -d --build --force-recreate services: - postgres: - image: postgres:14 - environment: - POSTGRES_USER: user - POSTGRES_PASSWORD: password + test: + build: test_image + volumes: + - data:/data backup: - build: - context: . - args: - ALPINE_VERSION: '3.16' + build: . + env_file: .env environment: - SCHEDULE: '@weekly' # optional - BACKUP_KEEP_DAYS: 7 # optional - PASSPHRASE: passphrase # optional - S3_REGION: - S3_ACCESS_KEY_ID: - S3_SECRET_ACCESS_KEY: - S3_BUCKET: - S3_PREFIX: backup - POSTGRES_HOST: postgres - POSTGRES_DATABASE: postgres - POSTGRES_USER: user - POSTGRES_PASSWORD: password + SCHEDULE: '* * * * *' + BACKUP_KEEP_DAYS: 7 + PASSPHRASE: passphrase + S3_PREFIX: backup-test diff --git a/src/backup.sh b/src/backup.sh index ff1bf15..76815be 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -1,31 +1,25 @@ -#! /bin/sh +#!/bin/sh set -eu set -o pipefail source ./env.sh -echo "Creating backup of $POSTGRES_DATABASE database..." -pg_dump --format=custom \ - -h $POSTGRES_HOST \ - -p $POSTGRES_PORT \ - -U $POSTGRES_USER \ - -d $POSTGRES_DATABASE \ - $PGDUMP_EXTRA_OPTS \ - > db.dump +echo "Creating backup..." +tar -xzf dump.tar.gz /data timestamp=$(date +"%Y-%m-%dT%H:%M:%S") -s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}/${POSTGRES_DATABASE}_${timestamp}.dump" +s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}/${BACKUP_NAME}_${timestamp}.dump" if [ -n "$PASSPHRASE" ]; then echo "Encrypting backup..." - rm -f db.dump.gpg - gpg --symmetric --batch --passphrase "$PASSPHRASE" db.dump - rm db.dump - local_file="db.dump.gpg" + rm -f dump.tar.gz.gpg + gpg --symmetric --batch --passphrase "$PASSPHRASE" dump.tar.gz + rm dump.tar.gz + local_file="dump.tar.gz.gpg" s3_uri="${s3_uri_base}.gpg" else - local_file="db.dump" + local_file="dump.tar.gz" s3_uri="$s3_uri_base" fi diff --git a/src/env.sh b/src/env.sh index af495e4..f3a9761 100644 --- a/src/env.sh +++ b/src/env.sh @@ -3,29 +3,8 @@ if [ -z "$S3_BUCKET" ]; then exit 1 fi -if [ -z "$POSTGRES_DATABASE" ]; then - echo "You need to set the POSTGRES_DATABASE environment variable." - exit 1 -fi - -if [ -z "$POSTGRES_HOST" ]; then - # https://docs.docker.com/network/links/#environment-variables - if [ -n "$POSTGRES_PORT_5432_TCP_ADDR" ]; then - POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR - POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT - else - echo "You need to set the POSTGRES_HOST environment variable." - exit 1 - fi -fi - -if [ -z "$POSTGRES_USER" ]; then - echo "You need to set the POSTGRES_USER environment variable." - exit 1 -fi - -if [ -z "$POSTGRES_PASSWORD" ]; then - echo "You need to set the POSTGRES_PASSWORD environment variable." +if [ -z "$BACKUP_NAME" ]; then + echo "You need to set the BACKUP_NAME environment variable." exit 1 fi @@ -35,12 +14,20 @@ else aws_args="--endpoint-url $S3_ENDPOINT" fi - if [ -n "$S3_ACCESS_KEY_ID" ]; then export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID fi + +if [ -n "$S3_ACCESS_KEY" ]; then + export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY +fi + if [ -n "$S3_SECRET_ACCESS_KEY" ]; then export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY fi + +if [ -n "$S3_SECRET_KEY" ]; then + export AWS_SECRET_ACCESS_KEY=$S3_SECRET_KEY +fi + export AWS_DEFAULT_REGION=$S3_REGION -export PGPASSWORD=$POSTGRES_PASSWORD diff --git a/src/install.sh b/src/install.sh index 5077ee2..5313293 100644 --- a/src/install.sh +++ b/src/install.sh @@ -1,19 +1,12 @@ -#! /bin/sh +#!/bin/sh set -eux set -o pipefail apk update +apk add gnupg aws-cli -# install pg_dump -apk add postgresql-client - -# install gpg -apk add gnupg - -apk add aws-cli - -# install go-cron +# Install go-cron apk add curl curl -L https://github.com/ivoronin/go-cron/releases/download/v0.0.5/go-cron_0.0.5_linux_${TARGETARCH}.tar.gz -O tar xvf go-cron_0.0.5_linux_${TARGETARCH}.tar.gz @@ -22,6 +15,4 @@ mv go-cron /usr/local/bin/go-cron chmod u+x /usr/local/bin/go-cron apk del curl - -# cleanup rm -rf /var/cache/apk/* diff --git a/src/restore.sh b/src/restore__wip__.sh similarity index 98% rename from src/restore.sh rename to src/restore__wip__.sh index 3040146..c54986c 100644 --- a/src/restore.sh +++ b/src/restore__wip__.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh set -u # `-e` omitted intentionally, but i can't remember why exactly :'( set -o pipefail diff --git a/src/run.sh b/src/run.sh index 90b5fcb..179bf8f 100644 --- a/src/run.sh +++ b/src/run.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh set -eu diff --git a/test_image/Dockerfile b/test_image/Dockerfile new file mode 100644 index 0000000..d7dea20 --- /dev/null +++ b/test_image/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch + +COPY data/ /data diff --git a/test_image/data/file b/test_image/data/file new file mode 100644 index 0000000..16b14f5 --- /dev/null +++ b/test_image/data/file @@ -0,0 +1 @@ +test file diff --git a/test_image/data/nested/file b/test_image/data/nested/file new file mode 100644 index 0000000..b5caa67 --- /dev/null +++ b/test_image/data/nested/file @@ -0,0 +1 @@ +nested test file