From caee1d5655800579c56c0bd3e97683ab2fe7f1b1 Mon Sep 17 00:00:00 2001 From: momentum dev Date: Mon, 13 Jul 2026 16:39:26 -0400 Subject: [PATCH] backup: nightly pg_dump service on the postgres:18 image Production Postgres had NO automated backup (pg_dump appeared only in the one-time migration script). New db-backup service in both compose files fires 20:10 ET Mon-Fri: pg_dump | gzip with pipefail + gunzip -t + tmp-then-rename (a mid-stream death can't masquerade as a backup), prunes past BACKUP_KEEP_DAYS, writes last_backup_ok on success. postgres:18 image = version-matched client (app image has none). TrueNAS host is pull-only, so its copy inlines the script. Restore: gunzip -c dump | psql $DATABASE_URL. Co-Authored-By: Claude Fable 5 --- .env.example | 5 ++++ deploy/docker/backup_db.sh | 30 ++++++++++++++++++++++++ deploy/docker/truenas-compose.yml | 39 +++++++++++++++++++++++++++++++ docker-compose.yml | 33 ++++++++++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 deploy/docker/backup_db.sh diff --git a/.env.example b/.env.example index 4e1596d..e944d12 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,11 @@ TELEGRAM_CHAT_ID='2082795871' DEADMAN_STALE_MIN=5 # alert when no events for this many minutes during market hours DEADMAN_DAILY_PING=1 # nightly OK-ping after 20:30 ET — proves the watchdog itself is alive +# Nightly pg_dump (db-backup compose service, 20:10 ET Mon-Fri) — the only +# automated backup of the production Postgres. See deploy/docker/backup_db.sh. +BACKUP_DIR=/data/backups # inside the db-backup container (app-data volume) +BACKUP_KEEP_DAYS=14 # prune dumps older than this many days + #Market Data these are free plans so work within days/months capacity. Alpha_Vantage_Api_Key= Twelve_Data_Api_Key= diff --git a/deploy/docker/backup_db.sh b/deploy/docker/backup_db.sh new file mode 100644 index 0000000..ecc9d97 --- /dev/null +++ b/deploy/docker/backup_db.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Nightly logical backup of the momentum Postgres. Runs inside the db-backup +# compose service — the postgres:18 image, so pg_dump always matches the +# server major (the app image has NO pg_dump, and a Debian-stock client is an +# older major that refuses to dump a newer server). +# +# pg_dump | gzip -> $BACKUP_DIR/momentum-YYYY-MM-DD.sql.gz +# prune anything older than BACKUP_KEEP_DAYS (default 14) +# write $BACKUP_DIR/last_backup_ok on success (freshness marker) +# +# Restore: gunzip -c momentum-.sql.gz | psql "$DATABASE_URL" +# +# Run with bash (not sh): pipefail is what catches a mid-stream pg_dump death +# that would otherwise leave a truncated-but-valid gzip behind. +set -euo pipefail + +BACKUP_DIR="${BACKUP_DIR:-/backups}" +BACKUP_KEEP_DAYS="${BACKUP_KEEP_DAYS:-14}" +: "${DATABASE_URL:?DATABASE_URL is required}" + +mkdir -p "$BACKUP_DIR" +out="$BACKUP_DIR/momentum-$(date +%F).sql.gz" + +echo "[backup] $(date) -> $out" +pg_dump "$DATABASE_URL" | gzip > "$out.tmp" +gunzip -t "$out.tmp" +mv "$out.tmp" "$out" +find "$BACKUP_DIR" -name 'momentum-*.sql.gz*' -mtime +"$BACKUP_KEEP_DAYS" -delete # *: also sweep stale .tmp from failed runs +date > "$BACKUP_DIR/last_backup_ok" +echo "[backup] OK $(du -h "$out" | cut -f1)" diff --git a/deploy/docker/truenas-compose.yml b/deploy/docker/truenas-compose.yml index e9e0341..9404f5a 100644 --- a/deploy/docker/truenas-compose.yml +++ b/deploy/docker/truenas-compose.yml @@ -194,3 +194,42 @@ services: fi sleep 20 done + + db-backup: + # Nightly pg_dump at 20:10 ET Mon-Fri, on the postgres:18 image so the + # client always matches the server major (the app image has NO pg_dump). + # This host is pull-only (no repo checkout), so the backup logic is INLINE + # — keep it in sync with deploy/docker/backup_db.sh. Restore: + # gunzip -c momentum-.sql.gz | psql "$DATABASE_URL" + image: postgres:18 + restart: unless-stopped + environment: + DATABASE_URL: 'postgresql://momentum:@postgres:5432/momentum' + BACKUP_DIR: '/backups' + BACKUP_KEEP_DAYS: '14' + TZ: 'America/New_York' # postgres image defaults to UTC; the 20:10 gate is ET + volumes: + - /mnt/volume/Apps/momentum/data/backups:/backups + depends_on: + postgres: + condition: service_healthy + command: + - /bin/bash + - -c + - | + set -o pipefail + echo "[backup] up; fires 20:10 ET Mon-Fri" + while true; do + if [ "$$(date +%H:%M)" = "20:10" ] && [ "$$(date +%u)" -le 5 ]; then + out="$$BACKUP_DIR/momentum-$$(date +%F).sql.gz" + mkdir -p "$$BACKUP_DIR" \ + && pg_dump "$$DATABASE_URL" | gzip > "$$out.tmp" \ + && gunzip -t "$$out.tmp" && mv "$$out.tmp" "$$out" \ + && find "$$BACKUP_DIR" -name 'momentum-*.sql.gz*' -mtime +"$$BACKUP_KEEP_DAYS" -delete \ + && date > "$$BACKUP_DIR/last_backup_ok" \ + && echo "[backup] OK $$out" \ + || echo "[backup] FAILED" + sleep 120 + fi + sleep 20 + done diff --git a/docker-compose.yml b/docker-compose.yml index 12b59cd..d166d73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,6 +137,39 @@ services: sleep 20 done + db-backup: + # Nightly pg_dump at 20:10 ET Mon-Fri (after the 20:00 scheduler pass). + # Runs on the postgres:18 image — a version-matched pg_dump for free; the + # app image has none. Backups land in the app-data volume under backups/. + image: postgres:18 + container_name: momentum-db-backup + restart: unless-stopped + environment: + # in-stack postgres, same value the app services use (NOT .env's + # DATABASE_URL, which may point at the old LAN instance) + DATABASE_URL: postgresql://${POSTGRES_USER:-momentum}:${POSTGRES_PASSWORD:-momentum}@postgres:5432/${POSTGRES_DB:-momentum} + BACKUP_DIR: /data/backups + BACKUP_KEEP_DAYS: "14" + TZ: America/New_York # the postgres image defaults to UTC; the 20:10 gate is ET + volumes: + - ./deploy/docker/backup_db.sh:/backup_db.sh:ro + - app-data:/data + depends_on: + postgres: + condition: service_healthy + command: + - /bin/bash + - -c + - | + echo "[backup] up; fires 20:10 ET Mon-Fri" + while true; do + if [ "$$(date +%H:%M)" = "20:10" ] && [ "$$(date +%u)" -le 5 ]; then + bash /backup_db.sh || echo "[backup] FAILED" + sleep 120 + fi + sleep 20 + done + # ---- On-demand jobs: docker compose run --rm journal: <<: *app