proxmox-config/backup.sh

#!/usr/bin/env bash
# =============================================================================
# Proxmox Config Backup — runs on PVE1 or PVE2
# Backs up all critical configs to GitHub weekly
# Install: /usr/local/bin/proxmox-backup
# Cron:    0 3 * * 0 /usr/local/bin/proxmox-backup >> /var/log/proxmox-backup.log 2>&1
# =============================================================================
set -euo pipefail

PAT="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"
REPO_URL="https://${PAT}@github.com/myronblair/proxmox-config.git"
REPO_DIR="/opt/proxmox-config"
NODE=$(hostname)
LOG_PREFIX="[$(date '+%Y-%m-%d %H:%M:%S')] [$NODE]"

log()  { echo "$LOG_PREFIX $*"; }
die()  { echo "$LOG_PREFIX ERROR: $*" >&2; exit 1; }

# ---------------------------------------------------------------------------
# 1. Clone or update local repo
# ---------------------------------------------------------------------------
if [[ -d "$REPO_DIR/.git" ]]; then
    log "Pulling latest from GitHub"
    cd "$REPO_DIR"
    git config user.email "proxmox-backup@orbishosting.com"
    git config user.name  "Proxmox Backup"
    git pull --rebase origin main -q || { log "Pull failed — continuing with local state"; true; }
else
    log "Cloning repo to $REPO_DIR"
    git clone "$REPO_URL" "$REPO_DIR"
    cd "$REPO_DIR"
    git config user.email "proxmox-backup@orbishosting.com"
    git config user.name  "Proxmox Backup"
fi

cd "$REPO_DIR"

# ---------------------------------------------------------------------------
# 2. Node-specific directories
# ---------------------------------------------------------------------------
NODE_DIR="$REPO_DIR/$NODE"
mkdir -p \
    "$NODE_DIR/network" \
    "$NODE_DIR/cron" \
    "$NODE_DIR/scripts" \
    "$NODE_DIR/systemd" \
    "$NODE_DIR/jarvis-agent"

# ---------------------------------------------------------------------------
# 3. Shared PVE cluster configs (only back up from primary node to avoid
#    conflicts — PVE1/pve is the primary; pve2 skips this section)
# ---------------------------------------------------------------------------
if [[ "$NODE" == "pve" ]]; then
    log "Backing up shared PVE cluster configs"
    SHARED_DIR="$REPO_DIR/shared"
    mkdir -p \
        "$SHARED_DIR/firewall" \
        "$SHARED_DIR/ha" \
        "$SHARED_DIR/sdn" \
        "$SHARED_DIR/nodes/pve/qemu-server" \
        "$SHARED_DIR/nodes/pve/lxc" \
        "$SHARED_DIR/nodes/pve2/qemu-server" \
        "$SHARED_DIR/nodes/pve2/lxc"

    # Top-level cluster configs
    for f in storage.cfg datacenter.cfg user.cfg corosync.conf vzdump.cron jobs.cfg ceph.conf replication.cfg; do
        [[ -f "/etc/pve/$f" ]] && cp "/etc/pve/$f" "$SHARED_DIR/$f" || true
    done

    # Firewall
    [[ -d /etc/pve/firewall ]] && rsync -a --delete /etc/pve/firewall/ "$SHARED_DIR/firewall/" 2>/dev/null || true

    # HA
    [[ -d /etc/pve/ha ]] && rsync -a --delete /etc/pve/ha/ "$SHARED_DIR/ha/" 2>/dev/null || true

    # SDN
    [[ -d /etc/pve/sdn ]] && rsync -a --delete /etc/pve/sdn/ "$SHARED_DIR/sdn/" 2>/dev/null || true

    # VM/LXC configs (per-node)
    for PVENODE in pve pve2; do
        [[ -d "/etc/pve/nodes/$PVENODE/qemu-server" ]] && \
            rsync -a --delete /etc/pve/nodes/$PVENODE/qemu-server/ "$SHARED_DIR/nodes/$PVENODE/qemu-server/" 2>/dev/null || true
        [[ -d "/etc/pve/nodes/$PVENODE/lxc" ]] && \
            rsync -a --delete /etc/pve/nodes/$PVENODE/lxc/ "$SHARED_DIR/nodes/$PVENODE/lxc/" 2>/dev/null || true
    done

    # PBS fingerprint (needed for restore)
    grep -A5 "pbs: ProxBackups" /etc/pve/storage.cfg > "$SHARED_DIR/pbs_fingerprint.txt" 2>/dev/null || true
fi

# ---------------------------------------------------------------------------
# 4. Network config
# ---------------------------------------------------------------------------
log "Backing up network config"
cp /etc/network/interfaces "$NODE_DIR/network/interfaces"
[[ -d /etc/network/interfaces.d ]] && \
    rsync -a --delete /etc/network/interfaces.d/ "$NODE_DIR/network/interfaces.d/" 2>/dev/null || true
cp /etc/hosts    "$NODE_DIR/network/hosts"
cp /etc/hostname "$NODE_DIR/network/hostname"
[[ -f /etc/resolv.conf ]] && cp /etc/resolv.conf "$NODE_DIR/network/resolv.conf" || true

# ---------------------------------------------------------------------------
# 5. Root crontab
# ---------------------------------------------------------------------------
log "Backing up crontab"
crontab -l 2>/dev/null > "$NODE_DIR/cron/root" || echo "# no crontab" > "$NODE_DIR/cron/root"
[[ -f /etc/vzdump.conf ]] && cp /etc/vzdump.conf "$NODE_DIR/vzdump.conf" || true

# ---------------------------------------------------------------------------
# 6. Custom scripts (text only — skip large binaries)
# ---------------------------------------------------------------------------
log "Backing up custom scripts"
for f in /usr/local/bin/*.sh /usr/local/bin/*.py /usr/local/bin/*.pl; do
    [[ -f "$f" ]] || continue
    size=$(stat -c%s "$f" 2>/dev/null || echo 0)
    if [[ $size -lt 524288 ]]; then  # skip files > 512KB (binaries)
        cp "$f" "$NODE_DIR/scripts/"
    fi
done

# ---------------------------------------------------------------------------
# 7. Custom systemd units (only non-stock units)
# ---------------------------------------------------------------------------
log "Backing up custom systemd units"
STOCK_UNITS="chronyd.service iscsi.service pve-manager.service smartd.service sshd.service zed.service"
for f in /etc/systemd/system/*.service; do
    [[ -f "$f" ]] || continue
    bname=$(basename "$f")
    if ! echo "$STOCK_UNITS" | grep -qw "$bname"; then
        cp "$f" "$NODE_DIR/systemd/"
    fi
done

# ---------------------------------------------------------------------------
# 8. JARVIS agent config
# ---------------------------------------------------------------------------
log "Backing up JARVIS agent config"
[[ -f /opt/jarvis-agent/config.json ]] && \
    cp /opt/jarvis-agent/config.json "$NODE_DIR/jarvis-agent/config.json" || true

# ---------------------------------------------------------------------------
# 9. SSH authorized_keys (for access restoration — no private keys)
# ---------------------------------------------------------------------------
mkdir -p "$NODE_DIR/ssh"
[[ -f /root/.ssh/authorized_keys ]] && cp /root/.ssh/authorized_keys "$NODE_DIR/ssh/authorized_keys" || true
[[ -f /root/.ssh/id_rsa.pub ]] && cp /root/.ssh/id_rsa.pub "$NODE_DIR/ssh/id_rsa.pub" || true

# ---------------------------------------------------------------------------
# 10. Commit and push
# ---------------------------------------------------------------------------
log "Committing changes"
git add -A
if git diff --cached --quiet; then
    log "No changes to commit"
else
    CHANGES=$(git diff --cached --stat | tail -1)
    git commit -m "[$NODE] Weekly backup $(date '+%Y-%m-%d') — $CHANGES"
    log "Pushing to GitHub"
    git push origin main
    log "Backup complete"
fi