test: add container lifecycle, federation join/sync tests to cross-node suite
- TEST-03: US-02 container lifecycle — stop filebrowser, verify health monitor auto-restarts within 90s (40s on .228, 15s on .198) - TEST-04: US-03 federation join — verify peers present, trust level, DID, last_seen - TEST-05: US-04 federation sync — trigger sync, verify app counts, freshness - Fix: updated stale onion addresses in federation nodes.json on both servers after Tor address rotation broke inter-node sync Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dorian
@@ -156,6 +156,69 @@ for node in "$NODE_A" "$NODE_B"; do
|
||||
done
|
||||
done
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# US-02: Container Lifecycle
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "# --- US-02: Container Lifecycle ---"
|
||||
|
||||
for node in "$NODE_A" "$NODE_B"; do
|
||||
node_label=$([[ "$node" == "$NODE_A" ]] && echo "A(.228)" || echo "B(.198)")
|
||||
for i in $(seq 1 "$ITERATIONS"); do
|
||||
# Check 1: All containers running (none exited)
|
||||
exited=$(ssh_sudo "$node" "podman ps -a --format '{{.State}}' | grep -c -i exited" 2>/dev/null || echo "0")
|
||||
exited=$(echo "$exited" | tail -1 | tr -d '[:space:]')
|
||||
if [[ "$exited" == "0" ]]; then
|
||||
tap_ok "US02-${node_label}-all-running-${i}"
|
||||
else
|
||||
tap_fail "US02-${node_label}-all-running-${i}" "${exited} exited containers"
|
||||
fi
|
||||
|
||||
# Check 2: Container count matches expectations (>= 20)
|
||||
count=$(ssh_sudo "$node" "podman ps --format '{{.Names}}' | wc -l" 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||
if [[ -n "$count" ]] && [[ "$count" -ge 20 ]]; then
|
||||
tap_ok "US02-${node_label}-container-count-${i} # count=${count}"
|
||||
else
|
||||
tap_fail "US02-${node_label}-container-count-${i}" "Only ${count:-0} containers running (need >=20)"
|
||||
fi
|
||||
|
||||
# Check 3: Health monitor auto-restart (stop filebrowser, wait, verify it restarts)
|
||||
# Only run this on first iteration to avoid disruption
|
||||
if [[ "$i" -eq 1 ]]; then
|
||||
# Stop filebrowser
|
||||
ssh_sudo "$node" "podman stop filebrowser" 2>/dev/null || true
|
||||
echo "# Stopped filebrowser on ${node_label}, waiting for health monitor to restart..."
|
||||
|
||||
# Wait up to 90s for health monitor to restart it
|
||||
restarted=false
|
||||
for wait_i in $(seq 1 18); do
|
||||
sleep 5
|
||||
fb_state=$(ssh_sudo "$node" "podman inspect filebrowser --format '{{.State.Status}}'" 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||
if [[ "$fb_state" == "running" ]]; then
|
||||
restarted=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$restarted" == "true" ]]; then
|
||||
tap_ok "US02-${node_label}-health-restart-${i} # filebrowser restarted in $((wait_i * 5))s"
|
||||
else
|
||||
tap_fail "US02-${node_label}-health-restart-${i}" "filebrowser not restarted after 90s"
|
||||
# Manually restart to not leave it broken
|
||||
ssh_sudo "$node" "podman start filebrowser" 2>/dev/null || true
|
||||
fi
|
||||
else
|
||||
# For subsequent iterations, just verify filebrowser is running
|
||||
fb_state=$(ssh_sudo "$node" "podman inspect filebrowser --format '{{.State.Status}}'" 2>/dev/null | tail -1 | tr -d '[:space:]')
|
||||
if [[ "$fb_state" == "running" ]]; then
|
||||
tap_ok "US02-${node_label}-filebrowser-running-${i}"
|
||||
else
|
||||
tap_fail "US02-${node_label}-filebrowser-running-${i}" "filebrowser state: ${fb_state:-unknown}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# US-05: Tor Hidden Services
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -195,6 +258,141 @@ for i in $(seq 1 "$ITERATIONS"); do
|
||||
fi
|
||||
done
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# US-03: Federation Join (verify existing federation)
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "# --- US-03: Federation Join ---"
|
||||
|
||||
for node in "$NODE_A" "$NODE_B"; do
|
||||
node_label=$([[ "$node" == "$NODE_A" ]] && echo "A(.228)" || echo "B(.198)")
|
||||
|
||||
# Get session for RPC calls
|
||||
session_header=$(get_session "$node")
|
||||
session_val=$(echo "$session_header" | sed -n 's/.*session=\([^;]*\).*/\1/p')
|
||||
csrf_val=$(echo "$session_header" | sed -n 's/.*csrf_token=\([^;]*\).*/\1/p')
|
||||
|
||||
for i in $(seq 1 "$ITERATIONS"); do
|
||||
# Call federation.list-nodes
|
||||
fed_result=$(rpc_call "$node" "federation.list-nodes" "$session_val" "$csrf_val")
|
||||
|
||||
# Check 1: At least 1 peer present
|
||||
peer_count=$(echo "$fed_result" | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('result',{}).get('nodes',[])))" 2>/dev/null || echo "0")
|
||||
if [[ "$peer_count" -ge 1 ]]; then
|
||||
tap_ok "US03-${node_label}-peers-present-${i} # count=${peer_count}"
|
||||
else
|
||||
tap_fail "US03-${node_label}-peers-present-${i}" "No federation peers found"
|
||||
fi
|
||||
|
||||
# Check 2: Trust level is 'trusted'
|
||||
trust=$(echo "$fed_result" | python3 -c "import sys,json; d=json.load(sys.stdin); nodes=d.get('result',{}).get('nodes',[]); print(nodes[0].get('trust_level','') if nodes else '')" 2>/dev/null || echo "")
|
||||
if [[ "$trust" == "trusted" ]]; then
|
||||
tap_ok "US03-${node_label}-trust-level-${i}"
|
||||
else
|
||||
tap_fail "US03-${node_label}-trust-level-${i}" "Trust level: ${trust:-unknown}"
|
||||
fi
|
||||
|
||||
# Check 3: DID present
|
||||
did=$(echo "$fed_result" | python3 -c "import sys,json; d=json.load(sys.stdin); nodes=d.get('result',{}).get('nodes',[]); print(nodes[0].get('did','') if nodes else '')" 2>/dev/null || echo "")
|
||||
if [[ -n "$did" ]] && [[ "$did" == did:* ]]; then
|
||||
tap_ok "US03-${node_label}-did-present-${i}"
|
||||
else
|
||||
tap_fail "US03-${node_label}-did-present-${i}" "DID: ${did:-missing}"
|
||||
fi
|
||||
|
||||
# Check 4: last_seen within 10 minutes
|
||||
last_seen=$(echo "$fed_result" | python3 -c "
|
||||
import sys,json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
d=json.load(sys.stdin)
|
||||
nodes=d.get('result',{}).get('nodes',[])
|
||||
if not nodes: print('missing'); sys.exit()
|
||||
ls = nodes[0].get('last_seen','')
|
||||
if not ls: print('never'); sys.exit()
|
||||
try:
|
||||
dt = datetime.fromisoformat(ls.replace('Z','+00:00'))
|
||||
diff = datetime.now(timezone.utc) - dt
|
||||
print('ok' if diff < timedelta(minutes=10) else f'stale:{diff}')
|
||||
except: print('parse_error')
|
||||
" 2>/dev/null || echo "error")
|
||||
if [[ "$last_seen" == "ok" ]]; then
|
||||
tap_ok "US03-${node_label}-last-seen-${i}"
|
||||
else
|
||||
tap_fail "US03-${node_label}-last-seen-${i}" "last_seen: ${last_seen}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# US-04: Federation Sync
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "# --- US-04: Federation Sync ---"
|
||||
|
||||
for node in "$NODE_A" "$NODE_B"; do
|
||||
node_label=$([[ "$node" == "$NODE_A" ]] && echo "A(.228)" || echo "B(.198)")
|
||||
|
||||
session_header=$(get_session "$node")
|
||||
session_val=$(echo "$session_header" | sed -n 's/.*session=\([^;]*\).*/\1/p')
|
||||
csrf_val=$(echo "$session_header" | sed -n 's/.*csrf_token=\([^;]*\).*/\1/p')
|
||||
|
||||
for i in $(seq 1 "$ITERATIONS"); do
|
||||
# Trigger sync
|
||||
sync_result=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Cookie: session=${session_val}; csrf_token=${csrf_val}" \
|
||||
-H "X-CSRF-Token: ${csrf_val}" \
|
||||
-d '{"method":"federation.sync-state"}' \
|
||||
"http://${node}:5678/rpc/v1" 2>/dev/null)
|
||||
|
||||
# Check 1: Sync returns results
|
||||
has_results=$(echo "$sync_result" | python3 -c "import sys,json; d=json.load(sys.stdin); r=d.get('result',{}); print('ok' if r and 'results' in r else 'no')" 2>/dev/null || echo "error")
|
||||
if [[ "$has_results" == "ok" ]]; then
|
||||
tap_ok "US04-${node_label}-sync-returns-${i}"
|
||||
else
|
||||
tap_fail "US04-${node_label}-sync-returns-${i}" "No sync results"
|
||||
fi
|
||||
|
||||
# Check 2: At least one sync target succeeded
|
||||
sync_ok=$(echo "$sync_result" | python3 -c "import sys,json; d=json.load(sys.stdin); results=d.get('result',{}).get('results',[]); ok=[r for r in results if r.get('status')=='ok']; print(len(ok))" 2>/dev/null || echo "0")
|
||||
if [[ "$sync_ok" -ge 1 ]]; then
|
||||
tap_ok "US04-${node_label}-sync-success-${i} # ok=${sync_ok}"
|
||||
else
|
||||
tap_fail "US04-${node_label}-sync-success-${i}" "No successful syncs"
|
||||
fi
|
||||
|
||||
# Check 3: Synced node has apps list
|
||||
apps_count=$(echo "$sync_result" | python3 -c "import sys,json; d=json.load(sys.stdin); results=d.get('result',{}).get('results',[]); ok=[r for r in results if r.get('status')=='ok']; print(ok[0].get('apps',0) if ok else 0)" 2>/dev/null || echo "0")
|
||||
if [[ "$apps_count" -gt 0 ]]; then
|
||||
tap_ok "US04-${node_label}-sync-apps-${i} # apps=${apps_count}"
|
||||
else
|
||||
tap_fail "US04-${node_label}-sync-apps-${i}" "Synced app count: ${apps_count}"
|
||||
fi
|
||||
|
||||
# Check 4: last_seen updated after sync (re-check federation list)
|
||||
fed_after=$(rpc_call "$node" "federation.list-nodes" "$session_val" "$csrf_val")
|
||||
ls_fresh=$(echo "$fed_after" | python3 -c "
|
||||
import sys,json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
d=json.load(sys.stdin)
|
||||
nodes=d.get('result',{}).get('nodes',[])
|
||||
if not nodes: print('missing'); sys.exit()
|
||||
ls = nodes[0].get('last_seen','')
|
||||
if not ls: print('never'); sys.exit()
|
||||
try:
|
||||
dt = datetime.fromisoformat(ls.replace('Z','+00:00'))
|
||||
diff = datetime.now(timezone.utc) - dt
|
||||
print('ok' if diff < timedelta(minutes=2) else f'stale:{diff}')
|
||||
except: print('parse_error')
|
||||
" 2>/dev/null || echo "error")
|
||||
if [[ "$ls_fresh" == "ok" ]]; then
|
||||
tap_ok "US04-${node_label}-last-seen-fresh-${i}"
|
||||
else
|
||||
tap_fail "US04-${node_label}-last-seen-fresh-${i}" "last_seen after sync: ${ls_fresh}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# US-09: NIP-07 Signing
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user