mirror of
https://github.com/ThmsRyn/homelab-proxmox-starter.git
synced 2026-11-20 21:23:38 +03:00
292 lines
8.6 KiB
Bash
292 lines
8.6 KiB
Bash
#!/usr/bin/env bash
|
|
# backup-job.sh — Verify Proxmox backup age and report stale or missing backups
|
|
#
|
|
# For each VM and LXC on this node, checks when the last backup was created.
|
|
# Exits with code 1 and prints an alert if any backup is older than --max-age-hours.
|
|
#
|
|
# Usage:
|
|
# bash backup-job.sh [OPTIONS]
|
|
#
|
|
# Options:
|
|
# --storage <id> Proxmox storage ID to check (e.g. pbs-01, backup-local)
|
|
# --max-age-hours <n> Alert if last backup is older than N hours (default: 26)
|
|
# --notify-email <addr> Send alert email via sendmail (optional)
|
|
# --exclude-vmid <id> Skip a VMID/CTID (repeat option as needed)
|
|
# --dry-run Only report, do not exit with error code
|
|
# --help Show this help message
|
|
#
|
|
# Exit codes:
|
|
# 0 All backups are within the max-age window
|
|
# 1 One or more backups are stale or missing
|
|
#
|
|
# Requirements:
|
|
# - Run on the Proxmox VE host as root
|
|
# - pvesh and pvesm must be available (they are on any PVE node)
|
|
# - sendmail optional (only required for --notify-email)
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Defaults
|
|
# ---------------------------------------------------------------------------
|
|
STORAGE=""
|
|
MAX_AGE_HOURS=26
|
|
NOTIFY_EMAIL=""
|
|
DRY_RUN=false
|
|
SCRIPT_NAME="$(basename "$0")"
|
|
NODE="$(hostname)"
|
|
ALERT_LINES=()
|
|
HAS_STALE=false
|
|
EXCLUDED_VMIDS=()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Color output
|
|
# ---------------------------------------------------------------------------
|
|
if [ -t 1 ]; then
|
|
RED='\033[0;31m'
|
|
YELLOW='\033[1;33m'
|
|
GREEN='\033[0;32m'
|
|
CYAN='\033[0;36m'
|
|
RESET='\033[0m'
|
|
else
|
|
RED='' YELLOW='' GREEN='' CYAN='' RESET=''
|
|
fi
|
|
|
|
log_info() { echo -e "${CYAN}[INFO]${RESET} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
|
log_ok() { echo -e "${GREEN}[OK]${RESET} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${RESET} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
|
log_alert() { echo -e "${RED}[ALERT]${RESET} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Usage
|
|
# ---------------------------------------------------------------------------
|
|
usage() {
|
|
sed -n '/^# Usage:/,/^# Requirements:/p' "$0" | sed 's/^# //'
|
|
exit 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--storage)
|
|
STORAGE="${2:?--storage requires a value}"
|
|
shift 2
|
|
;;
|
|
--max-age-hours)
|
|
MAX_AGE_HOURS="${2:?--max-age-hours requires a value}"
|
|
shift 2
|
|
;;
|
|
--notify-email)
|
|
NOTIFY_EMAIL="${2:?--notify-email requires a value}"
|
|
shift 2
|
|
;;
|
|
--exclude-vmid)
|
|
EXCLUDED_VMIDS+=("${2:?--exclude-vmid requires a value}")
|
|
shift 2
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
--help|-h)
|
|
usage
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1" >&2
|
|
usage
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
is_excluded_vmid() {
|
|
local vmid="$1"
|
|
local excluded
|
|
for excluded in "${EXCLUDED_VMIDS[@]}"; do
|
|
if [[ "$excluded" == "$vmid" ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
validate() {
|
|
if [[ -z "$STORAGE" ]]; then
|
|
echo "Error: --storage is required." >&2
|
|
usage
|
|
fi
|
|
|
|
if ! [[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]] || (( MAX_AGE_HOURS < 1 )); then
|
|
echo "Error: --max-age-hours must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ $EUID -ne 0 ]]; then
|
|
echo "Error: this script must run as root on the Proxmox host." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v pvesh &>/dev/null; then
|
|
echo "Error: pvesh not found. This script must run on a Proxmox VE node." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v pvesm &>/dev/null; then
|
|
echo "Error: pvesm not found. This script must run on a Proxmox VE node." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Verify storage exists
|
|
if ! pvesm status | awk '{print $1}' | grep -q "^${STORAGE}$"; then
|
|
echo "Error: storage '${STORAGE}' not found on this node." >&2
|
|
echo "Available storage:" >&2
|
|
pvesm status | awk 'NR>1 {print " " $1}' >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Get list of VMIDs (VMs and LXCs) on this node
|
|
# ---------------------------------------------------------------------------
|
|
get_all_vmids() {
|
|
{
|
|
pvesh get /nodes/"$NODE"/qemu --output-format json 2>/dev/null \
|
|
| python3 -c "import sys,json; [print(v['vmid']) for v in json.load(sys.stdin)]" 2>/dev/null || true
|
|
pvesh get /nodes/"$NODE"/lxc --output-format json 2>/dev/null \
|
|
| python3 -c "import sys,json; [print(v['vmid']) for v in json.load(sys.stdin)]" 2>/dev/null || true
|
|
} | sort -n
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Get the timestamp of the latest backup for a given VMID in the storage
|
|
# Returns epoch seconds or empty string if no backup found
|
|
# ---------------------------------------------------------------------------
|
|
get_latest_backup_epoch() {
|
|
local vmid="$1"
|
|
# pvesm list returns lines like:
|
|
# pbs-01:backup/vm/101/2026-04-01T03:00:00Z ... ctime=1743476400
|
|
pvesm list "$STORAGE" --vmid "$vmid" 2>/dev/null \
|
|
| awk 'NR>1 {print $NF}' \
|
|
| grep -oP 'ctime=\K[0-9]+' \
|
|
| sort -n \
|
|
| tail -1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Send email alert via sendmail
|
|
# ---------------------------------------------------------------------------
|
|
send_email_alert() {
|
|
if [[ -z "$NOTIFY_EMAIL" ]]; then
|
|
return
|
|
fi
|
|
|
|
if ! command -v sendmail &>/dev/null; then
|
|
log_warn "sendmail not found — cannot send email alert."
|
|
return
|
|
fi
|
|
|
|
local subject="[Proxmox Backup Alert] $NODE — stale or missing backups"
|
|
local body
|
|
body=$(printf "Backup alert from %s\n\nThe following VMs/LXCs have stale or missing backups:\n\n%s\n\nMax age configured: %d hours\nTimestamp: %s\n" \
|
|
"$NODE" \
|
|
"$(printf '%s\n' "${ALERT_LINES[@]}")" \
|
|
"$MAX_AGE_HOURS" \
|
|
"$(date)")
|
|
|
|
{
|
|
echo "To: $NOTIFY_EMAIL"
|
|
echo "Subject: $subject"
|
|
echo "Content-Type: text/plain; charset=utf-8"
|
|
echo ""
|
|
echo "$body"
|
|
} | sendmail "$NOTIFY_EMAIL"
|
|
|
|
log_info "Alert email sent to $NOTIFY_EMAIL."
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
main() {
|
|
parse_args "$@"
|
|
validate
|
|
|
|
log_info "Proxmox backup verification"
|
|
log_info "Node : $NODE"
|
|
log_info "Storage : $STORAGE"
|
|
log_info "Max age : ${MAX_AGE_HOURS}h"
|
|
log_info "Excluded : ${EXCLUDED_VMIDS[*]:-(none)}"
|
|
log_info "Dry-run : $DRY_RUN"
|
|
echo
|
|
|
|
local now
|
|
now=$(date +%s)
|
|
local max_age_seconds=$(( MAX_AGE_HOURS * 3600 ))
|
|
|
|
mapfile -t vmids < <(get_all_vmids)
|
|
|
|
if [[ ${#vmids[@]} -eq 0 ]]; then
|
|
log_warn "No VMs or LXCs found on node '$NODE'."
|
|
exit 0
|
|
fi
|
|
|
|
log_info "Found ${#vmids[@]} VMs/LXCs to check."
|
|
echo
|
|
|
|
for vmid in "${vmids[@]}"; do
|
|
if is_excluded_vmid "$vmid"; then
|
|
log_info "VMID $vmid skipped (--exclude-vmid)"
|
|
continue
|
|
fi
|
|
|
|
local latest_epoch
|
|
latest_epoch=$(get_latest_backup_epoch "$vmid")
|
|
|
|
if [[ -z "$latest_epoch" ]]; then
|
|
local msg="VMID $vmid — NO BACKUP FOUND in storage '$STORAGE'"
|
|
log_alert "$msg"
|
|
ALERT_LINES+=("$msg")
|
|
HAS_STALE=true
|
|
continue
|
|
fi
|
|
|
|
local age_seconds=$(( now - latest_epoch ))
|
|
local age_hours=$(( age_seconds / 3600 ))
|
|
local last_backup_str
|
|
last_backup_str=$(date -d "@$latest_epoch" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$latest_epoch" '+%Y-%m-%d %H:%M:%S')
|
|
|
|
if (( age_seconds > max_age_seconds )); then
|
|
local msg="VMID $vmid — last backup: $last_backup_str (${age_hours}h ago — STALE, max ${MAX_AGE_HOURS}h)"
|
|
log_alert "$msg"
|
|
ALERT_LINES+=("$msg")
|
|
HAS_STALE=true
|
|
else
|
|
log_ok "VMID $vmid — last backup: $last_backup_str (${age_hours}h ago)"
|
|
fi
|
|
done
|
|
|
|
echo
|
|
|
|
if [[ "$HAS_STALE" == true ]]; then
|
|
log_alert "RESULT: ${#ALERT_LINES[@]} VM(s) with stale or missing backups."
|
|
send_email_alert
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
exit 1
|
|
else
|
|
log_info "Dry-run mode: exiting with 0 despite alerts."
|
|
exit 0
|
|
fi
|
|
else
|
|
log_ok "RESULT: All ${#vmids[@]} VM(s) have backups within the ${MAX_AGE_HOURS}h window."
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
main "$@"
|