fix: deploy locking, safe eval replacement, first-boot error handling, script hardening

- S4: Add Bitcoin readiness gate and container tracking with final summary - S5: Replace eval "$DB_PASSWORDS" with safe case-based variable parsing - S6: Add deploy locking with stale lock detection (30min timeout) - S7: Deploy rollback already implemented — verified existing mechanism - S8: Switch trust-archipelago-cert.sh to SSH key auth, sshpass as fallback - S9: Pipe MariaDB SQL via stdin to avoid password in ps output - S17: Add disk space pre-flight check (abort if >85% full) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-21 01:39:22 +00:00
parent 3b7d541224
commit 2f60ef44ea
3 changed files with 142 additions and 24 deletions
--- a/scripts/deploy-to-target.sh
+++ b/scripts/deploy-to-target.sh
@@ -75,6 +75,36 @@ if [ -n "$TAILSCALE_NODE" ]; then
    exec "$SCRIPT_DIR/deploy-tailscale.sh" "$TAILSCALE_NODE"
 fi

+# Deploy locking — prevent concurrent deploys to the same target
+TARGET_IP_FOR_LOCK="$(echo "$TARGET_HOST" | cut -d@ -f2)"
+LOCK_DIR="/tmp/archipelago-deploy-${TARGET_IP_FOR_LOCK}.lock"
+# Check for stale lock (older than 30 minutes)
+if [ -d "$LOCK_DIR" ]; then
+    LOCK_STAMP="$LOCK_DIR/pid"
+    if [ -f "$LOCK_STAMP" ]; then
+        # macOS uses stat -f %m, Linux uses stat -c %Y
+        if stat -c %Y "$LOCK_STAMP" >/dev/null 2>&1; then
+            LOCK_MTIME=$(stat -c %Y "$LOCK_STAMP")
+        else
+            LOCK_MTIME=$(stat -f %m "$LOCK_STAMP")
+        fi
+        LOCK_AGE=$(( $(date +%s) - ${LOCK_MTIME:-0} ))
+        if [ "$LOCK_AGE" -gt 1800 ]; then
+            echo "$(timestamp) WARNING: Removing stale lock (${LOCK_AGE}s old)"
+            rm -rf "$LOCK_DIR"
+        fi
+    fi
+fi
+# mkdir is atomic — fails if directory already exists
+if ! mkdir "$LOCK_DIR" 2>/dev/null; then
+    echo "ERROR: Deploy already in progress for $TARGET_HOST (lock: $LOCK_DIR)"
+    exit 1
+fi
+echo $$ > "$LOCK_DIR/pid"
+# Clean up lock on exit (normal, error, or signal)
+cleanup_lock() { rm -rf "$LOCK_DIR"; }
+trap cleanup_lock EXIT
+
 # Dry run mode: show what would be deployed without executing
 if [[ "$DRY_RUN" == "true" ]]; then
    echo "═══ DRY RUN MODE — no changes will be made ═══"
@@ -168,6 +198,13 @@ if ! ssh $SSH_OPTS -o ConnectTimeout=5 "$TARGET_HOST" "echo ok" >/dev/null 2>&1;
 fi
 echo "   Connected."

+# Disk space pre-flight — abort if target is dangerously full
+DISK_PCT=$(ssh $SSH_OPTS $TARGET_HOST "df / | tail -1 | awk '{print \$(NF-1)}' | tr -d '%'" 2>/dev/null)
+if [ -n "$DISK_PCT" ] && [ "$DISK_PCT" -gt 85 ] 2>/dev/null; then
+  echo "ERROR: Target disk at ${DISK_PCT}% — need <85% for safe deploy. Free space and retry."
+  exit 1
+fi
+
 # Install prerequisites if missing (rsync for code sync, python3 for Claude API proxy)
 progress "Checking prerequisites"
 ssh $SSH_OPTS "$TARGET_HOST" '
@@ -940,7 +977,21 @@ MANIFEST_EOF
          echo "FEDI_HASH=$(sudo cat "$SECRETS_DIR/fedimint-gateway-hash")"
      fi
    ' 2>/dev/null)
-    eval "$DB_PASSWORDS"
+    # Safe variable parsing — never eval untrusted SSH output
+    while IFS='=' read -r key value; do
+        # Skip empty lines
+        [ -z "$key" ] && continue
+        # Only allow expected variable names
+        case "$key" in
+            MEMPOOL_DB_PASS)  MEMPOOL_DB_PASS="$value" ;;
+            BTCPAY_DB_PASS)   BTCPAY_DB_PASS="$value" ;;
+            IMMICH_DB_PASS)   IMMICH_DB_PASS="$value" ;;
+            PENPOT_DB_PASS)   PENPOT_DB_PASS="$value" ;;
+            MYSQL_ROOT_PASS)  MYSQL_ROOT_PASS="$value" ;;
+            FEDI_HASH)        FEDI_HASH="$value" ;;
+            *)  echo "   WARNING: Ignoring unexpected variable from server: $key" ;;
+        esac
+    done <<< "$DB_PASSWORDS"
    # Fallback if hash not available
    if [ -z "${FEDI_HASH:-}" ]; then
        FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC'
--- a/scripts/first-boot-containers.sh
+++ b/scripts/first-boot-containers.sh
@@ -115,6 +115,24 @@ else
 fi
 log "Fedimint gateway password stored in $SECRETS_DIR/fedimint-gateway-password"

+BITCOIN_READY=false
+TOTAL=0
+SUCCESS=0
+FAILED_LIST=""
+
+# Track container start result — call after each container creation attempt
+track_container() {
+  local name="$1"
+  TOTAL=$((TOTAL + 1))
+  if $DOCKER ps --filter "name=^${name}$" --format '{{.Names}}' 2>/dev/null | grep -q "^${name}$"; then
+    SUCCESS=$((SUCCESS + 1))
+    log "  [OK] $name is running"
+  else
+    FAILED_LIST="$FAILED_LIST $name"
+    log "  [FAIL] $name is NOT running"
+  fi
+}
+
 log "First-boot container creation starting (host=$TARGET_IP)"

 # Create swap file if not present (50% of RAM, min 2GB, max 8GB)
@@ -269,7 +287,14 @@ else
  log "Bitcoin Knots already running"
 fi
 # Wait for Bitcoin Knots RPC to be responsive
-wait_for_container "Bitcoin Knots RPC" "$DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS getblockchaininfo" 60
+if wait_for_container "Bitcoin Knots RPC" "$DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS getblockchaininfo" 60; then
+  BITCOIN_READY=true
+  log "Bitcoin Knots is ready — dependent containers will proceed"
+else
+  BITCOIN_READY=false
+  log "WARNING: Bitcoin Knots NOT ready — skipping dependent containers (electrumx, lnd, mempool, btcpay, fedimint)"
+fi
+track_container "bitcoin-knots"

 # Ensure wallet exists (Bitcoin Knots no longer auto-creates a default wallet)
 if ! $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS listwallets 2>/dev/null | grep -q "archipelago"; then
@@ -278,7 +303,8 @@ if ! $DOCKER exec bitcoin-knots bitcoin-cli -rpcuser=$BITCOIN_RPC_USER -rpcpassw
  log "Bitcoin Knots wallet 'archipelago' created/loaded"
 fi

-# 2. Mempool stack (matches deploy)
+# 2. Mempool stack (matches deploy) — depends on Bitcoin
+if [ "$BITCOIN_READY" = "true" ]; then
 if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-db|mysql-mempool'; then
  log "Creating mysql-mempool..."
  mkdir -p /var/lib/archipelago/mysql-mempool
@@ -289,11 +315,12 @@ if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-d
    -e MYSQL_DATABASE=mempool -e MYSQL_USER=mempool -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS \
    -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
    docker.io/mariadb:10.11 2>>"$LOG" || true
-  wait_for_container "Mempool MariaDB" "$DOCKER exec archy-mempool-db mariadb -uroot -p$MYSQL_ROOT_PASS -e 'SELECT 1'" 30
+  wait_for_container "Mempool MariaDB" "echo 'SELECT 1' | $DOCKER exec -i archy-mempool-db mariadb -uroot --password=\"$MYSQL_ROOT_PASS\"" 30
 fi
 MYSQL_CNT=$($DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
 MYSQL_CNT=${MYSQL_CNT:-archy-mempool-db}
 $DOCKER network connect archy-net "$MYSQL_CNT" 2>/dev/null || true
+track_container "archy-mempool-db"

 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
  if $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
@@ -311,6 +338,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
      docker.io/lukechilds/electrumx:v1.18.0 2>>"$LOG" || true
  fi
 fi
+track_container "electrumx"

 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
  log "Creating mempool-api..."
@@ -326,6 +354,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
    -e DATABASE_USERNAME=mempool -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
    docker.io/mempool/backend:v2.5.0 2>>"$LOG" || true
 fi
+track_container "mempool-api"

 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-web|mempool-web'; then
  log "Creating mempool frontend..."
@@ -335,6 +364,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-mempool-web|
    -p 4080:8080 -e FRONTEND_HTTP_PORT=8080 -e BACKEND_MAINNET_HTTP_HOST=mempool-api \
    docker.io/mempool/frontend:v2.5.0 2>>"$LOG" || true
 fi
+track_container "archy-mempool-web"

 # 2b. ElectrumX UI (status dashboard on port 50002, host network for backend access)
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrs-ui; then
@@ -357,7 +387,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrs-ui; then
  fi
 fi

-# 3. BTCPay stack (matches deploy)
+# 3. BTCPay stack (matches deploy) — depends on Bitcoin
 if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
  log "Creating PostgreSQL for BTCPay..."
  mkdir -p /var/lib/archipelago/postgres-btcpay
@@ -369,6 +399,7 @@ if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db
    docker.io/postgres:15-alpine 2>>"$LOG" || true
  wait_for_container "BTCPay PostgreSQL" "$DOCKER exec archy-btcpay-db pg_isready -U postgres" 30
 fi
+track_container "archy-btcpay-db"
 # Create nbxplorer DB only if postgres is running
 if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
  $DOCKER exec archy-btcpay-db psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='nbxplorer'" 2>/dev/null | grep -q 1 || \
@@ -392,6 +423,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; the
      docker.io/nicolasdorier/nbxplorer:2.6.0 2>>"$LOG" && sleep 5 || true
  fi
 fi
+track_container "archy-nbxplorer"

 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
  log "Creating BTCPay Server..."
@@ -410,12 +442,13 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
    -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
    docker.io/btcpayserver/btcpayserver:1.13.5 2>>"$LOG" || true
 fi
+track_container "btcpay-server"

 # ── Tier 2: Core Services ─────────────────────────────────────────────────
 log "=== Tier 2: Core Services ==="
 sleep 5  # Let databases stabilize

-# 4. LND
+# 4. LND — depends on Bitcoin
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE '^lnd$'; then
  log "Creating LND..."
  mkdir -p /var/lib/archipelago/lnd
@@ -457,8 +490,9 @@ LNDCONF
    -v /var/lib/archipelago/lnd:/root/.lnd \
    docker.io/lightninglabs/lnd:v0.18.4-beta 2>>"$LOG" || true
 fi
+track_container "lnd"

-# 5. Fedimint
+# 5. Fedimint — depends on Bitcoin
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint; then
  log "Creating Fedimint..."
  mkdir -p /var/lib/archipelago/fedimint
@@ -476,6 +510,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint; then
    -e FM_BITCOIND_URL=http://"$TARGET_IP":8332 \
    docker.io/fedimint/fedimintd:v0.10.0 2>>"$LOG" || true
 fi
+track_container "fedimint"

 # 5b. Fedimint Gateway (companion to fedimint)
 # Auto-detect LND: if running with credentials, use lnd mode; otherwise use ldk (built-in)
@@ -518,8 +553,13 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; th
      ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway 2>>"$LOG" || true
  fi
 fi
+track_container "fedimint-gateway"

-# ── Tier 3: Applications ──────────────────────────────────────────────────
+else
+  log "SKIPPED: mempool stack, electrumx, btcpay stack, lnd, fedimint (Bitcoin not ready)"
+fi  # end BITCOIN_READY
+
+# ── Tier 3: Applications (independent — always attempt) ───────────────────
 log "=== Tier 3: Applications ==="
 sleep 5  # Let core services stabilize

@@ -536,6 +576,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'homeassistant|home
    -e TZ=UTC \
    docker.io/homeassistant/home-assistant:2024.1 2>>"$LOG" || true
 fi
+track_container "homeassistant"

 # 7. Single-container apps (Grafana, Uptime Kuma, Jellyfin, PhotoPrism, Ollama, Vaultwarden)
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q grafana; then
@@ -552,6 +593,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q grafana; then
    -e GF_PATHS_DATA=/var/lib/grafana -e GF_USERS_ALLOW_SIGN_UP=false \
    docker.io/grafana/grafana:10.2.0 2>>"$LOG" || true
 fi
+track_container "grafana"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q uptime-kuma; then
  log "Creating Uptime Kuma..."
  mkdir -p /var/lib/archipelago/uptime-kuma
@@ -564,6 +606,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q uptime-kuma; then
    -e TZ=UTC \
    docker.io/louislam/uptime-kuma:1 2>>"$LOG" || true
 fi
+track_container "uptime-kuma"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q jellyfin; then
  log "Creating Jellyfin..."
  mkdir -p /var/lib/archipelago/jellyfin/config /var/lib/archipelago/jellyfin/cache
@@ -576,6 +619,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q jellyfin; then
    -v /var/lib/archipelago/jellyfin/cache:/cache \
    docker.io/jellyfin/jellyfin:10.8.13 2>>"$LOG" || true
 fi
+track_container "jellyfin"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q photoprism; then
  log "Creating PhotoPrism..."
  mkdir -p /var/lib/archipelago/photoprism
@@ -588,6 +632,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q photoprism; then
    -e PHOTOPRISM_ADMIN_PASSWORD=archipelago -e PHOTOPRISM_DEFAULT_LOCALE=en \
    "${PHOTOPRISM_IMAGE:-docker.io/photoprism/photoprism:240915}" 2>>"$LOG" || true
 fi
+track_container "photoprism"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q ollama; then
  log "Creating Ollama..."
  mkdir -p /var/lib/archipelago/ollama
@@ -599,6 +644,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q ollama; then
    -p 11434:11434 -v /var/lib/archipelago/ollama:/root/.ollama \
    "${OLLAMA_IMAGE:-docker.io/ollama/ollama:0.5.4}" 2>>"$LOG" || true
 fi
+track_container "ollama"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q vaultwarden; then
  log "Creating Vaultwarden..."
  mkdir -p /var/lib/archipelago/vaultwarden
@@ -610,6 +656,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q vaultwarden; then
    -p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \
    docker.io/vaultwarden/server:1.30.0-alpine 2>>"$LOG" || true
 fi
+track_container "vaultwarden"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nextcloud; then
  log "Creating Nextcloud..."
  mkdir -p /var/lib/archipelago/nextcloud
@@ -621,6 +668,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nextcloud; then
    -p 8085:80 -v /var/lib/archipelago/nextcloud:/var/www/html \
    docker.io/library/nextcloud:28 2>>"$LOG" || true
 fi
+track_container "nextcloud"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q searxng; then
  log "Creating SearXNG..."
  $DOCKER run -d --name searxng --restart unless-stopped \
@@ -631,6 +679,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q searxng; then
    -p 8888:8080 \
    "${SEARXNG_IMAGE:-docker.io/searxng/searxng:2024.11.17}" 2>>"$LOG" || true
 fi
+track_container "searxng"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q onlyoffice; then
  log "Creating OnlyOffice..."
  $DOCKER run -d --name onlyoffice --restart unless-stopped \
@@ -641,6 +690,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q onlyoffice; then
    -p 9980:80 \
    docker.io/onlyoffice/documentserver:7.5.1 2>>"$LOG" || true
 fi
+track_container "onlyoffice"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q filebrowser; then
  log "Creating File Browser..."
  mkdir -p /var/lib/archipelago/filebrowser /var/lib/archipelago/filebrowser-db
@@ -650,6 +700,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q filebrowser; then
    -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv \
    docker.io/filebrowser/filebrowser:v2.27.0 2>>"$LOG" || true
 fi
+track_container "filebrowser"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nginx-proxy-manager; then
  log "Creating Nginx Proxy Manager..."
  mkdir -p /var/lib/archipelago/nginx-proxy-manager/data /var/lib/archipelago/nginx-proxy-manager/letsencrypt
@@ -663,6 +714,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nginx-proxy-manager;
    -v /var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt \
    "${NPM_IMAGE:-docker.io/jc21/nginx-proxy-manager:2}" 2>>"$LOG" || true
 fi
+track_container "nginx-proxy-manager"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q portainer; then
  log "Creating Portainer..."
  mkdir -p /var/lib/archipelago/portainer
@@ -676,6 +728,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q portainer; then
    -v /var/run/podman/podman.sock:/var/run/docker.sock \
    docker.io/portainer/portainer-ce:2.19.4 2>>"$LOG" || true
 fi
+track_container "portainer"
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q tailscale; then
  log "Creating Tailscale..."
  mkdir -p /var/lib/archipelago/tailscale
@@ -695,6 +748,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q tailscale; then
    docker.io/tailscale/tailscale:stable \
    sh -c 'tailscale web --listen 0.0.0.0:8240 & exec tailscaled' 2>>"$LOG" || true
 fi
+track_container "tailscale"

 # Immich stack (postgres + redis + server - ML optional)
 # Remove old single-container 'immich' if present (wrong port 2283:3001, conflicts with immich_server)
@@ -739,6 +793,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
      ghcr.io/immich-app/immich-server:release 2>>"$LOG" || true
  fi
 fi
+track_container "immich_server"

 # Penpot stack (postgres + valkey + backend + exporter + frontend)
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
@@ -798,6 +853,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; the
      "${PENPOT_FRONTEND_IMAGE:-docker.io/penpotapp/frontend:2.4.2}" 2>>"$LOG" || true
  fi
 fi
+track_container "penpot-frontend"

 # 8. Nostr relays (optional - only if images were loaded; deploy does not create these on first boot)
 # nostr-rs-relay and strfry are in ISO image bundle; create if image exists
@@ -912,18 +968,7 @@ mkdir -p /var/lib/archipelago/identities
 # Ensure archipelago user can write to these directories
 chown -R 1000:1000 /var/lib/archipelago/tor-config /var/lib/archipelago/identity /var/lib/archipelago/identities 2>/dev/null || true

-# 11. Post-boot validation
-log "Validating container creation..."
-TOTAL=0; RUNNING=0
-for c in bitcoin-knots lnd btcpay-server fedimint homeassistant grafana uptime-kuma; do
-  TOTAL=$((TOTAL + 1))
-  if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q "$c"; then
-    RUNNING=$((RUNNING + 1))
-  fi
-done
-log "Post-boot validation: $RUNNING/$TOTAL core containers running"
-
-# 12. Run container doctor for any remaining issues
+# 11. Run container doctor for any remaining issues
 log "Running container doctor..."
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 if [ -x "$SCRIPT_DIR/container-doctor.sh" ]; then
@@ -932,4 +977,19 @@ elif [ -x "/opt/archipelago/scripts/container-doctor.sh" ]; then
    bash "/opt/archipelago/scripts/container-doctor.sh" --local 2>&1 | tee -a "$LOG"
 fi

+# 12. Final summary
+FAILED=$((TOTAL - SUCCESS))
+log "============================================="
+log "  FIRST-BOOT CONTAINER SUMMARY"
+log "============================================="
+log "  Total tracked:  $TOTAL"
+log "  Running:        $SUCCESS"
+log "  Failed:         $FAILED"
+if [ "$BITCOIN_READY" != "true" ]; then
+  log "  Bitcoin:        NOT READY (dependent containers skipped)"
+fi
+if [ -n "$FAILED_LIST" ]; then
+  log "  Failed list:   $FAILED_LIST"
+fi
+log "============================================="
 log "First-boot container creation complete"
--- a/scripts/trust-archipelago-cert.sh
+++ b/scripts/trust-archipelago-cert.sh
@@ -18,13 +18,20 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_DIR="$(dirname "$SCRIPT_DIR")"

 # Try to fetch cert from server via SSH (most reliable)
-if [ -f "$SCRIPT_DIR/deploy-config.sh" ]; then
+SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}"
+echo "Fetching certificate from server..."
+if [ -f "$SSH_KEY" ]; then
+  ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" archipelago@${HOST} \
+    'sudo -n cat /etc/archipelago/ssl/archipelago.crt' > "$CERT_FILE" 2>/dev/null || true
+elif [ -f "$SCRIPT_DIR/deploy-config.sh" ]; then
+  # Last-resort fallback: password auth (leaks credentials to process list)
  . "$SCRIPT_DIR/deploy-config.sh"
-  SSH_OPTS="-o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no"
+  echo "WARNING: SSH key not found at $SSH_KEY — falling back to password auth"
  if command -v sshpass >/dev/null 2>&1; then
-    echo "Fetching certificate from server..."
-    sshpass -p "$ARCHIPELAGO_PASSWORD" ssh $SSH_OPTS archipelago@${HOST} \
+    sshpass -p "$ARCHIPELAGO_PASSWORD" ssh -o StrictHostKeyChecking=no archipelago@${HOST} \
      'sudo -n cat /etc/archipelago/ssl/archipelago.crt' > "$CERT_FILE" 2>/dev/null || true
+  else
+    echo "WARNING: No SSH key and sshpass not installed — skipping SSH fetch"
  fi
 fi