From 1ea38070f644fee4b349a51d97ecb010c5a0b57f Mon Sep 17 00:00:00 2001 From: Dmitriy Haidiuk Date: Tue, 12 Apr 2022 21:39:51 +0300 Subject: [PATCH 1/5] Added auto-remove for old backups. --- .gitignore | 1 + Dockerfile | 1 + README.md | 3 +++ src/backup.sh | 11 ++++++++++- 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/Dockerfile b/Dockerfile index e3552f8..4ab650b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ ENV S3_ENDPOINT '' ENV S3_S3V4 'no' ENV SCHEDULE '' ENV PASSPHRASE '' +ENV BACKUP_KEEP_DAYS 7 ADD src/run.sh run.sh ADD src/backup.sh backup.sh diff --git a/README.md b/README.md index 8a67c16..023df0f 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,14 @@ pg_backup_s3: POSTGRES_DATABASE: dbname POSTGRES_USER: user POSTGRES_PASSWORD: password + BACKUP_KEEP_DAYS: 7 ``` + - Images are tagged by the major PostgreSQL version they support: `10`, `11`, `12`, `13`, or `14`. - The `SCHEDULE` variable determines backup frequency. See go-cron schedules documentation [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). Omit to run the backup immediately and then exit. - If `PASSPHRASE` is provided, the backup will be encrypted using GPG. - Run `docker exec sh backup.sh` to trigger a backup ad-hoc +- Use `BACKUP_KEEP_DAYS` to set time for how long you want to keep backup. ## Restore > **WARNING:** DATA LOSS! All database objects will be dropped and re-created. diff --git a/src/backup.sh b/src/backup.sh index e5a243d..2fece34 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -79,7 +79,16 @@ else fi echo "Uploading backup to $S3_BUCKET..." -aws $aws_args s3 cp "$local_file" "$s3_uri" +aws "$aws_args" s3 cp "$local_file" "$s3_uri" rm "$local_file" echo "Backup complete." + +if [ "$BACKUP_KEEP_DAYS" -ne 0 ]; then + date_from_remove=$(date -v -"${BACKUP_KEEP_DAYS}"d +"%Y-%m-%d") + backups_query="Contents[?LastModified<='${date_from_remove} 00:00:00'].{Key: Key}" + remove_backups=$(aws s3api list-objects-v2 --bucket "${S3_BUCKET}" --prefix "${S3_PREFIX}" --query "${backups_query}" --output text | xargs -n1 -t -I 'KEY' aws s3 rm s3://"${S3_BUCKET}"/'KEY') + echo "Removing old backup from $S3_BUCKET..." + eval "$remove_backups"; + echo "Removing complete." +fi From b1b12e8a05800786a34b04490588f1071ba88536 Mon Sep 17 00:00:00 2001 From: Dmitriy Haidiuk Date: Wed, 13 Apr 2022 00:11:32 +0300 Subject: [PATCH 2/5] Updated backup remove cron date variable. --- src/backup.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backup.sh b/src/backup.sh index 2fece34..517e070 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -79,16 +79,17 @@ else fi echo "Uploading backup to $S3_BUCKET..." -aws "$aws_args" s3 cp "$local_file" "$s3_uri" +aws $aws_args s3 cp "$local_file" "$s3_uri" rm "$local_file" echo "Backup complete." if [ "$BACKUP_KEEP_DAYS" -ne 0 ]; then - date_from_remove=$(date -v -"${BACKUP_KEEP_DAYS}"d +"%Y-%m-%d") + ms=$((2*BACKUP_KEEP_DAYS+1)) + date_from_remove=$(date -d "@$(($(date +%s) - ms))" +%Y-%m-%d) backups_query="Contents[?LastModified<='${date_from_remove} 00:00:00'].{Key: Key}" - remove_backups=$(aws s3api list-objects-v2 --bucket "${S3_BUCKET}" --prefix "${S3_PREFIX}" --query "${backups_query}" --output text | xargs -n1 -t -I 'KEY' aws s3 rm s3://"${S3_BUCKET}"/'KEY') + echo "Removing old backup from $S3_BUCKET..." - eval "$remove_backups"; + aws s3api list-objects --bucket "${S3_BUCKET}" --prefix "${S3_PREFIX}" --query "${backups_query}" --output text | xargs -n1 -t -I 'KEY' aws s3 rm s3://"${S3_BUCKET}"/'KEY' echo "Removing complete." fi From 2e45dc55b8dda6668e44d44f0cec35179abcb8aa Mon Sep 17 00:00:00 2001 From: Dmitriy Haidiuk Date: Wed, 13 Apr 2022 00:21:34 +0300 Subject: [PATCH 3/5] Updated backup remove cron date variable. --- src/backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.sh b/src/backup.sh index 517e070..6bc6d83 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -85,7 +85,7 @@ rm "$local_file" echo "Backup complete." if [ "$BACKUP_KEEP_DAYS" -ne 0 ]; then - ms=$((2*BACKUP_KEEP_DAYS+1)) + ms=$((86400*BACKUP_KEEP_DAYS)) date_from_remove=$(date -d "@$(($(date +%s) - ms))" +%Y-%m-%d) backups_query="Contents[?LastModified<='${date_from_remove} 00:00:00'].{Key: Key}" From f08931bf808af2232971af8ec8f8865ce2c9961f Mon Sep 17 00:00:00 2001 From: Dmitriy Haidiuk Date: Wed, 13 Apr 2022 00:34:08 +0300 Subject: [PATCH 4/5] Updated README file. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 023df0f..ee3c7bc 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ This project is a fork and re-structuring of @schickling's [postgres-backup-s3]( - [x] backup and restore with one image - [x] support encrypted (password-protected) backups - [x] option to restore from specific backup by timestamp + - [x] auto-removal old backups ## Other changes - uses `pg_dump`'s `custom` format (see [docs](https://www.postgresql.org/docs/10/app-pgdump.html)) From 2b28f9f0ec18306f54876c53c8cfa7f5a5ed933d Mon Sep 17 00:00:00 2001 From: Dmitriy Haidiuk Date: Wed, 13 Apr 2022 11:16:23 +0300 Subject: [PATCH 5/5] Updated removing old backups logic. --- Dockerfile | 2 +- README.md | 2 +- src/backup.sh | 13 +++++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ab650b..8738374 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV S3_ENDPOINT '' ENV S3_S3V4 'no' ENV SCHEDULE '' ENV PASSPHRASE '' -ENV BACKUP_KEEP_DAYS 7 +ENV BACKUP_KEEP_DAYS '' ADD src/run.sh run.sh ADD src/backup.sh backup.sh diff --git a/README.md b/README.md index ee3c7bc..200b083 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ pg_backup_s3: POSTGRES_DATABASE: dbname POSTGRES_USER: user POSTGRES_PASSWORD: password - BACKUP_KEEP_DAYS: 7 + BACKUP_KEEP_DAYS: 7 // Disabled by default. ``` - Images are tagged by the major PostgreSQL version they support: `10`, `11`, `12`, `13`, or `14`. diff --git a/src/backup.sh b/src/backup.sh index 6bc6d83..2d84359 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -84,12 +84,17 @@ rm "$local_file" echo "Backup complete." -if [ "$BACKUP_KEEP_DAYS" -ne 0 ]; then - ms=$((86400*BACKUP_KEEP_DAYS)) - date_from_remove=$(date -d "@$(($(date +%s) - ms))" +%Y-%m-%d) +if [ -n "$BACKUP_KEEP_DAYS" ]; then + sec=$((86400*BACKUP_KEEP_DAYS)) + date_from_remove=$(date -d "@$(($(date +%s) - sec))" +%Y-%m-%d) backups_query="Contents[?LastModified<='${date_from_remove} 00:00:00'].{Key: Key}" echo "Removing old backup from $S3_BUCKET..." - aws s3api list-objects --bucket "${S3_BUCKET}" --prefix "${S3_PREFIX}" --query "${backups_query}" --output text | xargs -n1 -t -I 'KEY' aws s3 rm s3://"${S3_BUCKET}"/'KEY' + aws s3api list-objects \ + --bucket "${S3_BUCKET}" \ + --prefix "${S3_PREFIX}" \ + --query "${backups_query}" \ + --output text \ + | xargs -n1 -t -I 'KEY' aws s3 rm s3://"${S3_BUCKET}"/'KEY' echo "Removing complete." fi