Prometheus Backup and Recovery

Overview

A comprehensive backup strategy ensures data protection and rapid recovery from failures, data corruption, or disasters. This page covers Prometheus and Alertmanager backups along with the shared backup infrastructure used across the whole monitoring stack — backup storage (S3, MinIO, encryption), disaster recovery testing, backup monitoring, and the combined automation scripts. Grafana-specific backup and restore procedures are documented in Grafana Backup and Recovery.

Key Components to Backup:

Prometheus TSDB data and configuration
Grafana dashboards, data sources, and database
Alertmanager configuration and silences
Recording and alerting rules
Provisioning configurations

Backup Strategy Requirements:

Automated daily backups
Off-site storage (S3, MinIO, NFS)
Retention policies (7 daily, 4 weekly, 12 monthly)
Encryption at rest and in transit
Tested restore procedures
Documented recovery processes

Prometheus Backups

Snapshot API

Prometheus provides a snapshot API for creating point-in-time backups.

#!/bin/bash
# Create Prometheus snapshot

PROMETHEUS_URL="http://prometheus:9090"
SNAPSHOT_NAME="prometheus-snapshot-$(date +%Y%m%d-%H%M%S)"

# Create the snapshot with a single POST and capture its response
# (requires the --web.enable-admin-api flag). Calling the endpoint twice
# would create two separate snapshots and throw the first one away.
SNAPSHOT=$(curl -s -XPOST "${PROMETHEUS_URL}/api/v1/admin/tsdb/snapshot" | jq -r '.data.name')

echo "Snapshot created: ${SNAPSHOT}"

Automated Backup Script

#!/bin/bash
# prometheus-backup.sh
# Automated Prometheus backup with retention

set -euo pipefail

# Configuration
PROMETHEUS_URL="${PROMETHEUS_URL:-http://prometheus:9090}"
PROMETHEUS_DATA_DIR="${PROMETHEUS_DATA_DIR:-/prometheus}"
BACKUP_DIR="${BACKUP_DIR:-/backups/prometheus}"
S3_BUCKET="${S3_BUCKET:-s3://prometheus-backups}"
RETENTION_DAYS="${RETENTION_DAYS:-7}"
RETENTION_WEEKS="${RETENTION_WEEKS:-4}"
RETENTION_MONTHS="${RETENTION_MONTHS:-12}"

# Logging
Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}

Error()
{
    Log "ERROR: $*" >&2
    exit 1
}

# Create backup directory
Mkdir -p "${BACKUP_DIR}"

# Generate backup name
BACKUP_NAME="prometheus-backup-$(date +%Y%m%d-%H%M%S)"
BACKUP_PATH="${BACKUP_DIR}/${BACKUP_NAME}"

Log "Starting Prometheus backup: ${BACKUP_NAME}"

# Create snapshot via API
Log "Creating Prometheus snapshot..."
SNAPSHOT_RESPONSE=$(curl -s -XPOST "${PROMETHEUS_URL}/api/v1/admin/tsdb/snapshot")
SNAPSHOT_STATUS=$(echo "${SNAPSHOT_RESPONSE}" | jq -r '.status')

if [ "${SNAPSHOT_STATUS}" != "success" ]; then
    Error "Failed to create snapshot: ${SNAPSHOT_RESPONSE}"
fi

SNAPSHOT_NAME=$(echo "${SNAPSHOT_RESPONSE}" | jq -r '.data.name')
SNAPSHOT_PATH="${PROMETHEUS_DATA_DIR}/snapshots/${SNAPSHOT_NAME}"

Log "Snapshot created: ${SNAPSHOT_NAME}"

# Create compressed archive
Log "Creating compressed archive..."
cd "${PROMETHEUS_DATA_DIR}/snapshots"
tar -czf "${BACKUP_PATH}.tar.gz" "${SNAPSHOT_NAME}"

if [ ! -f "${BACKUP_PATH}.tar.gz" ]; then
    Error "Failed to create backup archive"
fi

BACKUP_SIZE=$(du -h "${BACKUP_PATH}.tar.gz" | cut -f1)
Log "Backup archive created: ${BACKUP_PATH}.tar.gz (${BACKUP_SIZE})"

# Calculate checksum
Log "Calculating checksum..."
sha256sum "${BACKUP_PATH}.tar.gz" > "${BACKUP_PATH}.tar.gz.sha256"

# Backup Prometheus configuration
Log "Backing up Prometheus configuration..."
mkdir -p "${BACKUP_PATH}-config"
cp -r /etc/prometheus/* "${BACKUP_PATH}-config/"
tar -czf "${BACKUP_PATH}-config.tar.gz" -C "${BACKUP_PATH}-config" .
rm -rf "${BACKUP_PATH}-config"

# Upload to S3 if configured
if [ -n "${S3_BUCKET}" ]; then
    Log "Uploading to S3..."
    aws s3 cp "${BACKUP_PATH}.tar.gz" "${S3_BUCKET}/$(date +%Y/%m/%d)/${BACKUP_NAME}.tar.gz"
    aws s3 cp "${BACKUP_PATH}.tar.gz.sha256" "${S3_BUCKET}/$(date +%Y/%m/%d)/${BACKUP_NAME}.tar.gz.sha256"
    aws s3 cp "${BACKUP_PATH}-config.tar.gz" "${S3_BUCKET}/$(date +%Y/%m/%d)/${BACKUP_NAME}-config.tar.gz"
    Log "Upload completed"
fi

# Clean up snapshot
Log "Cleaning up snapshot..."
rm -rf "${SNAPSHOT_PATH}"

# Apply retention policy
Log "Applying retention policy..."

# Keep daily backups for RETENTION_DAYS
find "${BACKUP_DIR}" -name "prometheus-backup-*.tar.gz" -type f -mtime +${RETENTION_DAYS} -delete

# Keep weekly backups (Sundays) for RETENTION_WEEKS
WEEKLY_RETENTION_DAYS=$((RETENTION_WEEKS * 7))
find "${BACKUP_DIR}" -name "prometheus-backup-*.tar.gz" -type f -mtime +${WEEKLY_RETENTION_DAYS} ! -name "*-Sun-*.tar.gz" -delete

# Keep monthly backups (1st of month) for RETENTION_MONTHS
MONTHLY_RETENTION_DAYS=$((RETENTION_MONTHS * 30))
find "${BACKUP_DIR}" -name "prometheus-backup-*.tar.gz" -type f -mtime +${MONTHLY_RETENTION_DAYS} ! -name "*-01-*.tar.gz" -delete

# Clean up checksums for deleted backups
find "${BACKUP_DIR}" -name "*.sha256" -type f | while read -r checksum_file; do
    backup_file="${checksum_file%.sha256}"
    if [ ! -f "${backup_file}" ]; then
        rm -f "${checksum_file}"
    fi
done

Log "Backup completed successfully: ${BACKUP_NAME}"

# Send success notification
curl -X POST https://healthchecks.io/ping/your-uuid

exit 0

Incremental Backups

#!/bin/bash
# prometheus-incremental-backup.sh
# Incremental backup using rsync

set -euo pipefail

PROMETHEUS_DATA_DIR="/prometheus"
BACKUP_BASE_DIR="/backups/prometheus-incremental"
CURRENT_BACKUP="${BACKUP_BASE_DIR}/current"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)

# Create backup directories
mkdir -p "${BACKUP_BASE_DIR}"

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}

Log "Starting incremental backup..."

# Perform incremental backup using rsync
if [ -d "${CURRENT_BACKUP}" ]; then
    Log "Performing incremental backup..."
    rsync -av --delete \
        --link-dest="${CURRENT_BACKUP}" \
        "${PROMETHEUS_DATA_DIR}/" \
        "${BACKUP_BASE_DIR}/${TIMESTAMP}/"
else
    Log "Performing full backup..."
    rsync -av \
        "${PROMETHEUS_DATA_DIR}/" \
        "${BACKUP_BASE_DIR}/${TIMESTAMP}/"
fi

# Update current symlink
rm -f "${CURRENT_BACKUP}"
ln -s "${BACKUP_BASE_DIR}/${TIMESTAMP}" "${CURRENT_BACKUP}"

Log "Incremental backup completed: ${TIMESTAMP}"

Retention Policies

#!/bin/bash
# prometheus-retention.sh
# Apply sophisticated retention policies

set -euo pipefail

BACKUP_DIR="/backups/prometheus"

# Retention rules
DAILY_RETENTION=7      # Keep 7 daily backups
WEEKLY_RETENTION=4     # Keep 4 weekly backups (Sundays)
MONTHLY_RETENTION=12   # Keep 12 monthly backups (1st of month)
YEARLY_RETENTION=5     # Keep 5 yearly backups (January 1st)

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}

Log "Applying retention policies..."

# Get current date
CURRENT_DATE=$(date +%s)

# Process all backups
find "${BACKUP_DIR}" -name "prometheus-backup-*.tar.gz" -type f | while read -r backup_file; do
    # Extract date from filename (format: prometheus-backup-YYYYMMDD-HHMMSS.tar.gz)
    backup_date=$(basename "${backup_file}" | sed 's/prometheus-backup-\([0-9]\{8\}\).*/\1/')
    backup_timestamp=$(date -d "${backup_date}" +%s)
    backup_age_days=$(( (CURRENT_DATE - backup_timestamp) / 86400 ))
    backup_dow=$(date -d "${backup_date}" +%u)  # Day of week (1=Monday, 7=Sunday)
    backup_dom=$(date -d "${backup_date}" +%d)  # Day of month
    backup_month=$(date -d "${backup_date}" +%m)
    
    # Determine if backup should be kept
    keep_backup=false
    reason=""
    
    # Keep if within daily retention
    if [ ${backup_age_days} -le ${DAILY_RETENTION} ]; then
        keep_backup=true
        reason="daily"
    # Keep if weekly backup (Sunday) within weekly retention
    elif [ ${backup_dow} -eq 7 ] && [ ${backup_age_days} -le $((WEEKLY_RETENTION * 7)) ]; then
        keep_backup=true
        reason="weekly"
    # Keep if monthly backup (1st) within monthly retention
    elif [ "${backup_dom}" = "01" ] && [ ${backup_age_days} -le $((MONTHLY_RETENTION * 30)) ]; then
        keep_backup=true
        reason="monthly"
    # Keep if yearly backup (Jan 1st) within yearly retention
    elif [ "${backup_dom}" = "01" ] && [ "${backup_month}" = "01" ] && [ ${backup_age_days} -le $((YEARLY_RETENTION * 365)) ]; then
        keep_backup=true
        reason="yearly"
    fi
    
    # Delete if not kept
    if [ "${keep_backup}" = false ]; then
        Log "Deleting old backup: $(basename "${backup_file}") (${backup_age_days} days old)"
        rm -f "${backup_file}"
        rm -f "${backup_file}.sha256"
    else
        Log "Keeping backup: $(basename "${backup_file}") (${reason}, ${backup_age_days} days old)"
    fi
done

Log "Retention policy applied"

Alertmanager Backups

#!/bin/bash
# alertmanager-backup.sh

set -euo pipefail

ALERTMANAGER_URL="${ALERTMANAGER_URL:-http://alertmanager:9093}"
ALERTMANAGER_DATA_DIR="${ALERTMANAGER_DATA_DIR:-/alertmanager}"
BACKUP_DIR="${BACKUP_DIR:-/backups/alertmanager}"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)

mkdir -p "${BACKUP_DIR}"

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}

Log "Starting Alertmanager backup..."

# Backup configuration
Log "Backing up configuration..."
cp /etc/alertmanager/alertmanager.yml "${BACKUP_DIR}/alertmanager-config-${TIMESTAMP}.yml"

# Backup silences
Log "Backing up silences..."
curl -s "${ALERTMANAGER_URL}/api/v2/silences" > "${BACKUP_DIR}/silences-${TIMESTAMP}.json"

# Backup notification log
Log "Backing up notification log..."
if [ -d "${ALERTMANAGER_DATA_DIR}" ]; then
    tar -czf "${BACKUP_DIR}/alertmanager-data-${TIMESTAMP}.tar.gz" -C "${ALERTMANAGER_DATA_DIR}" .
fi

Log "Alertmanager backup completed"

Backup Storage

The storage, encryption, and off-site upload procedures below are shared infrastructure used by every backup in the monitoring stack, including Grafana. The Grafana Backup and Recovery page references this section rather than duplicating it.

S3 Storage

#!/bin/bash
# upload-to-s3.sh

set -euo pipefail

SOURCE_DIR="/backups"
S3_BUCKET="s3://monitoring-backups"
STORAGE_CLASS="INTELLIGENT_TIERING"

# Upload with encryption
aws s3 sync "${SOURCE_DIR}" "${S3_BUCKET}" \
    --storage-class "${STORAGE_CLASS}" \
    --server-side-encryption AES256 \
    --exclude "*.tmp" \
    --exclude ".DS_Store"

# Set lifecycle policy
aws s3api put-bucket-lifecycle-configuration \
    --bucket monitoring-backups \
    --lifecycle-configuration file://lifecycle-policy.json

{
    "Rules": [
        {
            "Id": "TransitionToGlacier",
            "Status": "Enabled",
            "Prefix": "",
            "Transitions": [
                {
                    "Days": 30,
                    "StorageClass": "GLACIER"
                },
                {
                    "Days": 90,
                    "StorageClass": "DEEP_ARCHIVE"
                }
            ],
            "Expiration": {
                "Days": 365
            }
        }
    ]
}

MinIO Storage

#!/bin/bash
# upload-to-minio.sh

set -euo pipefail

MC_ALIAS="minio"
MINIO_ENDPOINT="http://minio:9000"
MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY}"
MINIO_SECRET_KEY="${MINIO_SECRET_KEY}"
BUCKET="monitoring-backups"
SOURCE_DIR="/backups"

# Configure mc client
mc alias set "${MC_ALIAS}" "${MINIO_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"

# Create bucket if not exists
mc mb -p "${MC_ALIAS}/${BUCKET}"

# Upload backups
mc mirror --overwrite "${SOURCE_DIR}" "${MC_ALIAS}/${BUCKET}"

# Set retention policy
mc retention set --default COMPLIANCE --days 30 "${MC_ALIAS}/${BUCKET}"

Encryption

#!/bin/bash
# encrypt-backup.sh
# Encrypt backups using GPG

set -euo pipefail

BACKUP_FILE="$1"
GPG_RECIPIENT="backup@example.com"

# Encrypt backup
gpg --encrypt --recipient "${GPG_RECIPIENT}" \
    --output "${BACKUP_FILE}.gpg" \
    "${BACKUP_FILE}"

# Verify encryption
if [ -f "${BACKUP_FILE}.gpg" ]; then
    echo "Backup encrypted: ${BACKUP_FILE}.gpg"
    rm "${BACKUP_FILE}"
else
    echo "Encryption failed" >&2
    exit 1
fi

Restore Procedures

Prometheus Restore

#!/bin/bash
# prometheus-restore.sh

set -euo pipefail

BACKUP_FILE="$1"
PROMETHEUS_DATA_DIR="/prometheus"

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}

Error()
{
    Log "ERROR: $*" >&2
    exit 1
}

if [ -z "${BACKUP_FILE}" ]; then
    Error "Usage: $0 <backup-file.tar.gz>"
fi

if [ ! -f "${BACKUP_FILE}" ]; then
    Error "Backup file not found: ${BACKUP_FILE}"
fi

Log "Starting Prometheus restore from ${BACKUP_FILE}"

# Stop Prometheus
Log "Stopping Prometheus..."
docker stop prometheus || systemctl stop prometheus

# Verify checksum if available
if [ -f "${BACKUP_FILE}.sha256" ]; then
    Log "Verifying checksum..."
    sha256sum -c "${BACKUP_FILE}.sha256" || Error "Checksum verification failed"
fi

# Backup current data
Log "Backing up current data..."
BACKUP_TIMESTAMP=$(date +%Y%m%d-%H%M%S)
if [ -d "${PROMETHEUS_DATA_DIR}" ]; then
    mv "${PROMETHEUS_DATA_DIR}" "${PROMETHEUS_DATA_DIR}.backup.${BACKUP_TIMESTAMP}"
fi

# Extract backup
Log "Extracting backup..."
mkdir -p "${PROMETHEUS_DATA_DIR}"
tar -xzf "${BACKUP_FILE}" -C /tmp/

# Find snapshot directory
SNAPSHOT_DIR=$(find /tmp -type d -name "prometheus-snapshot-*" | head -1)

if [ -z "${SNAPSHOT_DIR}" ]; then
    Error "No snapshot directory found in backup"
fi

# Move data to Prometheus data directory
Log "Restoring data..."
mv "${SNAPSHOT_DIR}"/* "${PROMETHEUS_DATA_DIR}/"

# Fix permissions
chown -R prometheus:prometheus "${PROMETHEUS_DATA_DIR}"

# Clean up
rm -rf /tmp/prometheus-snapshot-*

# Start Prometheus
Log "Starting Prometheus..."
docker start prometheus || systemctl start prometheus

# Wait for Prometheus to be ready
Log "Waiting for Prometheus to be ready..."
for i in {1..30}; do
    if curl -sf http://localhost:9090/-/ready > /dev/null 2>&1; then
        Log "Prometheus is ready"
        break
    fi
    sleep 2
done

Log "Restore completed successfully"

Testing Restore Procedures

#!/bin/bash
# test-restore.sh
# Automated restore testing

set -euo pipefail

TEST_ENV="restore-test"
BACKUP_FILE="$1"

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
}

Log "Starting restore test..."

# Create isolated test environment
Log "Creating test environment..."
docker network create "${TEST_ENV}" || true

# Deploy test Prometheus
docker run -d --name prometheus-test --network "${TEST_ENV}" \
    -v prometheus-test-data:/prometheus \
    prom/prometheus:v2.53.0

# Perform restore in test environment
Log "Performing restore..."
./prometheus-restore.sh "${BACKUP_FILE}"

# Verify restoration
Log "Verifying restored data..."
sleep 10

# Check Prometheus is running
if ! docker exec prometheus-test wget -q -O- http://localhost:9090/-/ready; then
    Log "ERROR: Prometheus not ready"
    exit 1
fi

# Verify metrics exist
METRIC_COUNT=$(docker exec prometheus-test wget -q -O- 'http://localhost:9090/api/v1/label/__name__/values' | jq '.data | length')

if [ "${METRIC_COUNT}" -eq 0 ]; then
    Log "ERROR: No metrics found"
    exit 1
fi

Log "Restore test successful: ${METRIC_COUNT} metrics found"

# Clean up test environment
docker stop prometheus-test
docker rm prometheus-test
docker volume rm prometheus-test-data
docker network rm "${TEST_ENV}"

Log "Test environment cleaned up"

Disaster Recovery Testing

DR Drill Script

#!/bin/bash
# dr-drill.sh
# Complete disaster recovery drill

set -euo pipefail

DR_DATE=$(date +%Y-%m-%d)
DR_LOG="/var/log/dr-drill-${DR_DATE}.log"

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | tee -a "${DR_LOG}"
}

Log "=== Starting Disaster Recovery Drill ==="

# Step 1: Verify backups exist
Log "Step 1: Verifying backups..."
LATEST_PROMETHEUS_BACKUP=$(find /backups/prometheus -name "*.tar.gz" -type f -mtime -1 | head -1)
LATEST_GRAFANA_BACKUP=$(find /backups/grafana -name "grafana-db-*.dump" -type f -mtime -1 | head -1)

if [ -z "${LATEST_PROMETHEUS_BACKUP}" ]; then
    Log "ERROR: No recent Prometheus backup found"
    exit 1
fi

if [ -z "${LATEST_GRAFANA_BACKUP}" ]; then
    Log "ERROR: No recent Grafana backup found"
    exit 1
fi

Log "Found backups:"
Log "  Prometheus: ${LATEST_PROMETHEUS_BACKUP}"
Log "  Grafana: ${LATEST_GRAFANA_BACKUP}"

# Step 2: Test Prometheus restore
Log "Step 2: Testing Prometheus restore..."
START_TIME=$(date +%s)
./test-restore.sh "${LATEST_PROMETHEUS_BACKUP}"
PROMETHEUS_RTO=$(($(date +%s) - START_TIME))
Log "Prometheus RTO: ${PROMETHEUS_RTO} seconds"

# Step 3: Test Grafana restore
Log "Step 3: Testing Grafana restore..."
START_TIME=$(date +%s)
./test-grafana-restore.sh "${LATEST_GRAFANA_BACKUP}"
GRAFANA_RTO=$(($(date +%s) - START_TIME))
Log "Grafana RTO: ${GRAFANA_RTO} seconds"

# Step 4: Verify data integrity
Log "Step 4: Verifying data integrity..."

# Check metric count
EXPECTED_METRICS=1000
ACTUAL_METRICS=$(curl -s 'http://prometheus-test:9090/api/v1/label/__name__/values' | jq '.data | length')

if [ "${ACTUAL_METRICS}" -lt "${EXPECTED_METRICS}" ]; then
    Log "WARNING: Metric count lower than expected (${ACTUAL_METRICS} < ${EXPECTED_METRICS})"
fi

# Check dashboard count
EXPECTED_DASHBOARDS=20
ACTUAL_DASHBOARDS=$(curl -s -H "Authorization: Bearer ${GRAFANA_API_KEY}" \
    "http://grafana-test:3000/api/search?type=dash-db" | jq '. | length')

if [ "${ACTUAL_DASHBOARDS}" -lt "${EXPECTED_DASHBOARDS}" ]; then
    Log "WARNING: Dashboard count lower than expected (${ACTUAL_DASHBOARDS} < ${EXPECTED_DASHBOARDS})"
fi

# Step 5: Generate report
Log "Step 5: Generating DR drill report..."

cat > "/var/log/dr-drill-report-${DR_DATE}.txt" <<EOF
Disaster Recovery Drill Report
Date: ${DR_DATE}

Backup Status:
- Prometheus: ✓ Found ($(date -r "${LATEST_PROMETHEUS_BACKUP}" '+%Y-%m-%d %H:%M:%S'))
- Grafana: ✓ Found ($(date -r "${LATEST_GRAFANA_BACKUP}" '+%Y-%m-%d %H:%M:%S'))

Recovery Time Objectives (RTO):
- Prometheus: ${PROMETHEUS_RTO} seconds (Target: <300s)
- Grafana: ${GRAFANA_RTO} seconds (Target: <120s)

Data Integrity:
- Metrics restored: ${ACTUAL_METRICS} (Expected: >${EXPECTED_METRICS})
- Dashboards restored: ${ACTUAL_DASHBOARDS} (Expected: >${EXPECTED_DASHBOARDS})

Overall Status: PASSED
EOF

Log "=== Disaster Recovery Drill Completed ==="
Log "Full log: ${DR_LOG}"
Log "Report: /var/log/dr-drill-report-${DR_DATE}.txt"

RTO/RPO Testing

#!/bin/bash
# rto-rpo-test.sh
# Measure actual RTO and RPO

set -euo pipefail

# Simulate failure and measure recovery time
echo "=== RTO/RPO Test ==="

# Record current time
FAILURE_TIME=$(date +%s)
echo "Simulated failure at: $(date -d @${FAILURE_TIME})"

# Stop services
docker stop prometheus grafana

# Wait for monitoring to detect failure
sleep 60

# Start recovery
RECOVERY_START=$(date +%s)
docker start prometheus grafana

# Wait for services to be ready
while true; do
    if curl -sf http://localhost:9090/-/ready > /dev/null 2>&1 && \
       curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
        break
    fi
    sleep 1
done

RECOVERY_END=$(date +%s)
RTO=$((RECOVERY_END - RECOVERY_START))

echo "Recovery Time Objective (RTO): ${RTO} seconds"

# Calculate RPO (data loss)
LAST_SCRAPE=$(curl -s 'http://localhost:9090/api/v1/query?query=up' | \
    jq -r '.data.result[0].value[0]')
RPO=$((FAILURE_TIME - LAST_SCRAPE))

echo "Recovery Point Objective (RPO): ${RPO} seconds of data loss"

Backup Monitoring

# backup-monitoring.yml
groups:
  - name: backup_monitoring
    interval: 5m
    rules:
      - alert: BackupFailed
        expr: time() - backup_last_success_timestamp_seconds > 86400
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: "Backup has not succeeded in 24 hours"
          description: "Last successful backup was {{ $value | humanizeDuration }} ago"

      - alert: BackupSizeAnomaly
        expr: abs(backup_size_bytes - avg_over_time(backup_size_bytes[7d])) / stddev_over_time(backup_size_bytes[7d]) > 3
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Backup size is anomalous"
          description: "Backup size {{ $value | humanize }}B deviates significantly from average"

      - alert: BackupStorageFull
        expr: (backup_storage_used_bytes / backup_storage_total_bytes) > 0.9
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Backup storage is {{ $value | humanizePercentage }} full"

      - alert: BackupRetentionViolation
        expr: count(backup_age_days > 365) > 0
        for: 1h
        labels:
          severity: info
        annotations:
          summary: "Backups older than retention policy found"

Automation Scripts

The combined automation below orchestrates the Prometheus, Grafana, and Alertmanager backup scripts for the entire monitoring stack.

Complete Backup Automation

#!/bin/bash
# backup-all.sh
# Complete backup automation with notifications

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_FILE="/var/log/backup-$(date +%Y%m%d).log"
HEALTHCHECK_URL="${HEALTHCHECK_URL:-}"

Log()
{
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}"
}

Error()
{
    Log "ERROR: $*"
    Send_notification "Backup Failed" "$*"
    exit 1
}

Send_notification()
{
    local title="$1"
    local message="$2"
    
    # Slack notification
    if [ -n "${SLACK_WEBHOOK_URL:-}" ]; then
        curl -X POST "${SLACK_WEBHOOK_URL}" \
            -H 'Content-Type: application/json' \
            -d "{\"text\": \"${title}: ${message}\"}"
    fi
    
    # Email notification
    if [ -n "${EMAIL_TO:-}" ]; then
        echo "${message}" | mail -s "${title}" "${EMAIL_TO}"
    fi
}

# Trap errors
trap 'Error "Backup script failed at line $LINENO"' ERR

Log "=== Starting Automated Backup ==="

# Backup Prometheus
Log "Backing up Prometheus..."
"${SCRIPT_DIR}/prometheus-backup.sh" || Error "Prometheus backup failed"

# Backup Grafana
Log "Backing up Grafana..."
"${SCRIPT_DIR}/grafana-full-backup.sh" || Error "Grafana backup failed"

# Backup Alertmanager
Log "Backing up Alertmanager..."
"${SCRIPT_DIR}/alertmanager-backup.sh" || Error "Alertmanager backup failed"

# Verify backups
Log "Verifying backups..."
PROMETHEUS_BACKUP=$(find /backups/prometheus -name "*.tar.gz" -type f -mmin -30 | head -1)
GRAFANA_BACKUP=$(find /backups/grafana -name "grafana-db-*.dump" -type f -mmin -30 | head -1)

if [ -z "${PROMETHEUS_BACKUP}" ] || [ -z "${GRAFANA_BACKUP}" ]; then
    Error "Backup verification failed - backup files not found"
fi

Log "Backups verified successfully"

# Ping healthcheck
if [ -n "${HEALTHCHECK_URL}" ]; then
    curl -m 10 --retry 5 "${HEALTHCHECK_URL}" || Log "WARNING: Failed to ping healthcheck"
fi

Send_notification "Backup Successful" "All backups completed successfully"

Log "=== Automated Backup Completed ==="

Cron Configuration

# /etc/cron.d/monitoring-backups

# Daily Prometheus backup at 2 AM
0 2 * * * root /opt/scripts/prometheus-backup.sh >> /var/log/prometheus-backup.log 2>&1

# Daily Grafana backup at 3 AM
0 3 * * * root /opt/scripts/grafana-full-backup.sh >> /var/log/grafana-backup.log 2>&1

# Hourly Alertmanager backup
0 * * * * root /opt/scripts/alertmanager-backup.sh >> /var/log/alertmanager-backup.log 2>&1

# Weekly DR drill (Sundays at 1 AM)
0 1 * * 0 root /opt/scripts/dr-drill.sh >> /var/log/dr-drill.log 2>&1

# Daily backup cleanup
0 4 * * * root /opt/scripts/prometheus-retention.sh >> /var/log/backup-retention.log 2>&1

Table of Contents