test coverage improvements; mTLS enforcement; agent broke :(

scripts/autofill_evidence_templates.sh

@@ -0,0 +1,305 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ARG="${1:-}"
+
+if [ -n "${RUN_ARG}" ]; then
+  if [ -d "${RUN_ARG}" ]; then
+    RUN_DIR="${RUN_ARG}"
+  else
+    RUN_DIR="${ROOT_DIR}/evidence/runs/${RUN_ARG}"
+  fi
+else
+  RUN_DIR="$(ls -1dt "${ROOT_DIR}"/evidence/runs/* 2>/dev/null | head -n 1)"
+fi
+
+if [ -z "${RUN_DIR:-}" ] || [ ! -d "${RUN_DIR}" ]; then
+  echo "Run directory not found." >&2
+  exit 1
+fi
+
+RUN_ID="$(basename "${RUN_DIR}")"
+TEMPLATE_DIR="${RUN_DIR}/templates"
+mkdir -p "${TEMPLATE_DIR}"
+
+SUMMARY_FILE="${RUN_DIR}/SUMMARY.md"
+TEST_FILE="${RUN_DIR}/tests/django-tests.txt"
+FAIL_SUMMARY_FILE="${RUN_DIR}/tests/django-tests-failures-summary.txt"
+COVERAGE_FILE="${RUN_DIR}/tests/coverage-report.txt"
+LOG_COUNTS_FILE="${RUN_DIR}/logs/agent-endpoint-status-counts.txt"
+COMMIT_FILE="${RUN_DIR}/meta/git-head.txt"
+SCREENSHOT_DIR="${RUN_DIR}/screenshots"
+
+utc_time="$(grep -m1 '^- UTC Time:' "${SUMMARY_FILE}" 2>/dev/null | sed -E 's/.*`([^`]*)`.*/\1/')"
+if [ -z "${utc_time}" ]; then
+  utc_time="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+fi
+
+commit_full="$(head -n 1 "${COMMIT_FILE}" 2>/dev/null || true)"
+if [ -z "${commit_full}" ]; then
+  commit_full="unknown"
+fi
+commit_short="$(printf "%s" "${commit_full}" | cut -c1-12)"
+
+env_domain="$(grep -m1 '^KEYWARDEN_DOMAIN=' "${ROOT_DIR}/.env" 2>/dev/null | cut -d'=' -f2-)"
+if [ -z "${env_domain}" ]; then
+  env_domain="$(grep -m1 '^KEYWARDEN_DOMAIN=' "${ROOT_DIR}/.env.example" 2>/dev/null | cut -d'=' -f2-)"
+fi
+if [ -z "${env_domain}" ]; then
+  env_url="(set manually)"
+elif printf "%s" "${env_domain}" | grep -qE '^https?://'; then
+  env_url="${env_domain}"
+else
+  env_url="https://${env_domain}"
+fi
+
+tests_line="$(grep -E '^Ran [0-9]+ tests? in' "${TEST_FILE}" 2>/dev/null | tail -n 1)"
+tests_status="$(grep -E '^(OK|FAILED.*)$' "${TEST_FILE}" 2>/dev/null | tail -n 1)"
+if [ -z "${tests_line}" ]; then
+  tests_line="Ran ? tests in ?s"
+fi
+if [ -z "${tests_status}" ]; then
+  tests_status="Unknown"
+fi
+
+failure_summary="$(cat "${FAIL_SUMMARY_FILE}" 2>/dev/null || true)"
+if [ -z "${failure_summary}" ]; then
+  failure_summary="No failure summary captured."
+fi
+
+coverage_total_line="$(grep -E '^TOTAL[[:space:]]+' "${COVERAGE_FILE}" 2>/dev/null | tail -n 1)"
+if [ -n "${coverage_total_line}" ]; then
+  coverage_stmts="$(printf "%s\n" "${coverage_total_line}" | awk '{print $2}')"
+  coverage_miss="$(printf "%s\n" "${coverage_total_line}" | awk '{print $3}')"
+  coverage_pct="$(printf "%s\n" "${coverage_total_line}" | awk '{print $4}')"
+else
+  coverage_stmts="?"
+  coverage_miss="?"
+  coverage_pct="N/A"
+fi
+
+host_key_status="Manual"
+host_key_note="Check shell SSH options in source."
+if grep -q 'StrictHostKeyChecking=no' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null; then
+  host_key_status="FAIL (auto)"
+  host_key_note="Strict host key checking is disabled in shell command builder."
+elif grep -q 'UserKnownHostsFile=/dev/null' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null \
+  || grep -q 'GlobalKnownHostsFile=/dev/null' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null; then
+  host_key_status="FAIL (auto)"
+  host_key_note="Known-hosts verification is bypassed via /dev/null host-key files."
+elif grep -q 'StrictHostKeyChecking=' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null \
+  && grep -q 'UpdateHostKeys=yes' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null; then
+  host_key_status="PASS (auto)"
+  host_key_note="Host-key checking is enabled and host key updates are requested."
+fi
+
+test_result() {
+  local test_name="$1"
+  if grep -q "${test_name}.*ok" "${TEST_FILE}" 2>/dev/null; then
+    printf "PASS (auto)"
+    return
+  fi
+  if grep -q "${test_name}.*FAIL" "${TEST_FILE}" 2>/dev/null; then
+    printf "FAIL (auto)"
+    return
+  fi
+  if grep -q "${test_name}" "${TEST_FILE}" 2>/dev/null; then
+    printf "Observed (review)"
+    return
+  fi
+  printf "Not covered"
+}
+
+sec03_status="$(test_result "test_ingest_logs_rejects_mtls_cert_for_different_server")"
+sec03_actual="Agent mTLS certificate/server binding mismatch test present in Django suite."
+sec04_status="$(test_result "test_detail_view_denied_when_users_scope_not_granted")"
+sec04_actual="Permission-boundary denial test present in Django suite."
+sec08_status="Manual"
+sec08_actual="No dedicated WebSocket auth negative test was auto-detected."
+sec05_status="$(test_result "test_admin_dashboard_requires_admin")"
+sec05_actual="Unauthorized UI route redirects to disguised target (dashboard)."
+
+append_path() {
+  local current="$1"
+  local next="$2"
+  if [ -z "${next}" ]; then
+    printf "%s" "${current}"
+    return
+  fi
+  if [ -z "${current}" ]; then
+    printf "%s" "${next}"
+    return
+  fi
+  printf "%s, %s" "${current}" "${next}"
+}
+
+screenshot_count="0"
+if [ -d "${SCREENSHOT_DIR}" ]; then
+  screenshot_count="$(find "${SCREENSHOT_DIR}" -maxdepth 1 -type f -name '*.png' | wc -l | tr -d ' ')"
+fi
+
+e_login=""
+[ -f "${SCREENSHOT_DIR}/01-login-entry.png" ] && e_login="$(append_path "${e_login}" "screenshots/01-login-entry.png")"
+[ -f "${SCREENSHOT_DIR}/02-login-native.png" ] && e_login="$(append_path "${e_login}" "screenshots/02-login-native.png")"
+e_dashboard=""
+[ -f "${SCREENSHOT_DIR}/03-dashboard.png" ] && e_dashboard="screenshots/03-dashboard.png"
+e_key_upload=""
+[ -f "${SCREENSHOT_DIR}/04-profile.png" ] && e_key_upload="screenshots/04-profile.png"
+e_server_reg=""
+[ -f "${SCREENSHOT_DIR}/07-admin-server-reg.png" ] && e_server_reg="screenshots/07-admin-server-reg.png"
+e_access_queue=""
+[ -f "${SCREENSHOT_DIR}/06-admin-dashboard.png" ] && e_access_queue="screenshots/06-admin-dashboard.png"
+e_audit=""
+[ -f "${SCREENSHOT_DIR}/09-server-audit.png" ] && e_audit="screenshots/09-server-audit.png"
+e_server_detail=""
+[ -f "${SCREENSHOT_DIR}/08-server-detail.png" ] && e_server_detail="screenshots/08-server-detail.png"
+e_shell=""
+[ -f "${SCREENSHOT_DIR}/11-server-shell.png" ] && e_shell="screenshots/11-server-shell.png"
+
+cat > "${TEMPLATE_DIR}/browser-matrix.md" <<EOF
+# Browser Test Matrix
+
+## Autofill Summary
+
+- Run ID: \`${RUN_ID}\`
+- Date (UTC): \`${utc_time}\`
+- Environment URL: \`${env_url}\`
+- Commit: \`${commit_full}\`
+- Automated tests: ${tests_line} (${tests_status})
+- Coverage: ${coverage_pct} (${coverage_stmts} stmts, ${coverage_miss} missed)
+- Screenshot files detected: ${screenshot_count}
+
+## How To Complete
+
+1. Use one browser at a time and run every row end-to-end.
+2. In each browser column, enter one of: \`PASS\`, \`FAIL\`, \`PARTIAL\`, or \`N/A\`.
+3. Put a short failure symptom in \`Notes\` (for example: "OIDC callback loop").
+4. Put concrete evidence file paths in \`Evidence Path\` (screenshots, HAR, console logs).
+5. Keep timestamps in filenames so evidence can be traced to this run.
+
+## Matrix
+
+| Area | Chrome | Firefox | Safari | Edge | Notes | Evidence Path |
+|---|---|---|---|---|---|---|
+| Login / OIDC |  |  |  |  |  | ${e_login} |
+| Dashboard |  |  |  |  |  | ${e_dashboard} |
+| Key upload |  |  |  |  |  | ${e_key_upload} |
+| Server registration |  |  |  |  |  | ${e_server_reg} |
+| Access approve/revoke |  |  |  |  |  | ${e_access_queue} |
+| Audit logs |  |  |  |  |  | ${e_audit} |
+| Server detail + heartbeat |  |  |  |  |  | ${e_server_detail} |
+| Browser shell page render |  |  |  |  |  | ${e_shell} |
+| Browser shell interactive session |  |  |  |  |  |  |
+
+EOF
+
+cat > "${TEMPLATE_DIR}/security-test-report.md" <<EOF
+# Security Test Report
+
+## Autofill Summary
+
+- Run ID: \`${RUN_ID}\`
+- Date (UTC): \`${utc_time}\`
+- Environment URL: \`${env_url}\`
+- Commit: \`${commit_full}\`
+- Django suite summary: ${tests_line} (${tests_status})
+- Failure summary source: \`tests/django-tests-failures-summary.txt\`
+- Coverage source: \`tests/coverage-report.txt\`
+
+## How To Complete
+
+1. Keep autofilled values unless the underlying test evidence changes.
+2. For each row, set \`Pass/Fail\` to \`PASS\`, \`FAIL\`, \`PARTIAL\`, or \`N/A\`.
+3. Replace generic \`Actual\` text with concrete observations (status codes, error text).
+4. Add exact artifact paths for every completed row.
+5. Write findings as evidence-backed statements only.
+
+## Scope
+
+- Authentication and session handling
+- Authorization boundaries
+- Token handling
+- Input validation
+- File/command safety
+- WebSocket controls
+
+## Test Cases
+
+| ID | Area | Test | Expected | Actual | Pass/Fail | Evidence Path |
+|---|---|---|---|---|---|---|
+| SEC-01 | Input validation | Submit malformed SSH key payload | 4xx with validation error | Not auto-detected in current suite | Manual |  |
+| SEC-02 | Auth bypass | Call admin API as non-admin | 403 | Not auto-detected in current suite | Manual |  |
+| SEC-03 | mTLS misuse | Use mismatched agent client certificate | 401/403 | ${sec03_actual} | ${sec03_status} | tests/django-tests.txt |
+| SEC-04 | Object permission | Access server not granted to user | denied response | ${sec04_actual} | ${sec04_status} | tests/django-tests.txt |
+| SEC-05 | Session/CSRF | Unsafe method without CSRF/session validity | blocked or redirected | ${sec05_actual} | ${sec05_status} | tests/django-tests.txt |
+| SEC-06 | File handling | Upload invalid file/path content | Rejected safely | Not auto-detected in current suite | Manual |  |
+| SEC-07 | Command injection | Shell input escaping checks | No command injection path | Partial automated shell command tests exist | Manual | tests/django-tests.txt |
+| SEC-08 | WebSocket shell auth | Open shell WS without scope | Rejected | ${sec08_actual} | ${sec08_status} |  |
+| SEC-09 | Host key verification | Shell verifies host identity | Verified | ${host_key_note} | ${host_key_status} | app/apps/servers/consumers.py |
+
+## Findings
+
+- High:
+- Medium:
+- Low:
+
+## Follow-up Actions
+
+- 
+
+EOF
+
+cat > "${TEMPLATE_DIR}/standards-checklist.md" <<EOF
+# Standards Checklist
+
+## Autofill Summary
+
+- Run ID: \`${RUN_ID}\`
+- Date (UTC): \`${utc_time}\`
+- Environment URL: \`${env_url}\`
+- Commit: \`${commit_full}\`
+- Test summary: ${tests_line} (${tests_status})
+- Coverage summary: ${coverage_pct}
+
+## How To Complete
+
+1. Treat autofilled \`Status\` values as initial evidence flags, not final compliance.
+2. Replace status with one of: \`Meets\`, \`Partially Meets\`, \`Does Not Meet\`, \`Not Evaluated\`.
+3. Keep evidence paths concrete and local to this run where possible.
+4. In \`Notes\`, explain why each control is or is not met.
+5. Keep \`Gap Summary\` short and directly tied to failed/partial controls.
+
+## OWASP ASVS (selected)
+
+| Control | Requirement | Status | Evidence Path | Notes |
+|---|---|---|---|---|
+| V1 | Architecture/design documented | Auto: Evidence present | diagrams/listing.txt | Diagram sources copied into run artifacts |
+| V2 | Authentication controls | Auto: Evidence present | tests/django-tests.txt | Auth/account tests are present; final compliance needs manual judgment |
+| V3 | Session management | Auto: Partial evidence | tests/django-tests.txt | Access denial behavior covered; full session-hardening review still manual |
+| V4 | Access control enforcement | Auto: Evidence present | tests/django-tests.txt | Scope and permission boundary tests present |
+| V5 | Input validation and encoding | Auto: Partial evidence | tests/django-tests.txt | Some validation paths covered; broad input fuzzing not automated |
+| V7 | Error handling and logging | Auto: Evidence present | logs/gunicorn-access-tail-2000.log | Runtime logs are captured in every run |
+| V8 | Data protection at rest/in transit | Auto: Partial evidence | runtime/docker-compose-ps.txt | TLS/database setup exists; key-material risk analysis remains manual |
+| V10 | Malicious code / injection resistance | Auto: Partial evidence | tests/django-tests.txt | Some shell-related tests present; full security testing remains manual |
+
+## NIST SP 800-53 / 800-63-aligned checks (selected)
+
+| Control Family | Requirement | Status | Evidence Path | Notes |
+|---|---|---|---|---|
+| AC | Least privilege / access enforcement | Auto: Evidence present | tests/django-tests.txt | Access scope and object-permission tests present |
+| AU | Auditable events and retention | Auto: Evidence present | logs/agent-access-tail-2000.log | Audit and runtime logs captured |
+| IA | Identity proofing and authentication federation | Auto: Partial evidence | tests/django-tests.txt | Auth flows tested; federation assurance still manual |
+| SC | Cryptographic protections and key handling | Auto: Partial evidence | app/apps/servers/consumers.py | Host-key verification gap still open |
+| CM | Configuration and change management | Auto: Evidence present | meta/git-head.txt | Commit and repo state captured per run |
+| CP | Resilience / single point of failure acknowledgement | Auto: Partial evidence | SUMMARY.md | Availability risks identified but not fully mitigated |
+
+## Gap Summary
+
+- Host key verification remains a known shell security gap.
+- Full WebSocket shell security testing is not yet automated.
+- Standards mapping still requires manual final classification per control.
+
+EOF
+
+echo "Autofilled templates for run: ${RUN_ID}"

scripts/capture_ui_screenshots.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ARG="${1:-}"
+
+if [ -n "${RUN_ARG}" ]; then
+  if [ -d "${RUN_ARG}" ]; then
+    RUN_DIR="${RUN_ARG}"
+    RUN_ID="$(basename "${RUN_DIR}")"
+  else
+    RUN_ID="${RUN_ARG}"
+    RUN_DIR="${ROOT_DIR}/evidence/runs/${RUN_ID}"
+  fi
+else
+  RUN_DIR="$(ls -1dt "${ROOT_DIR}"/evidence/runs/* 2>/dev/null | head -n 1)"
+  RUN_ID="$(basename "${RUN_DIR}")"
+fi
+
+if [ -z "${RUN_DIR:-}" ] || [ ! -d "${RUN_DIR}" ]; then
+  echo "Run directory not found." >&2
+  exit 1
+fi
+
+BASE_URL="${KEYWARDEN_SCREENSHOT_BASE_URL:-}"
+ADMIN_USER="${KEYWARDEN_ADMIN_USERNAME:-}"
+ADMIN_EMAIL="${KEYWARDEN_ADMIN_EMAIL:-}"
+ADMIN_PASS="${KEYWARDEN_ADMIN_PASSWORD:-}"
+SERVER_ID="${KEYWARDEN_SCREENSHOT_SERVER_ID:-}"
+
+if [ -f "${ROOT_DIR}/.env" ]; then
+  if [ -z "${BASE_URL}" ]; then
+    DOMAIN_LINE="$(grep -m1 '^KEYWARDEN_DOMAIN=' "${ROOT_DIR}/.env" || true)"
+    DOMAIN_VALUE="${DOMAIN_LINE#KEYWARDEN_DOMAIN=}"
+    if [ -n "${DOMAIN_VALUE}" ]; then
+      if printf "%s" "${DOMAIN_VALUE}" | grep -qE '^https?://'; then
+        BASE_URL="${DOMAIN_VALUE}"
+      else
+        BASE_URL="https://${DOMAIN_VALUE}"
+      fi
+    fi
+  fi
+  if [ -z "${ADMIN_USER}" ]; then
+    USER_LINE="$(grep -m1 '^KEYWARDEN_ADMIN_USERNAME=' "${ROOT_DIR}/.env" || true)"
+    ADMIN_USER="${USER_LINE#KEYWARDEN_ADMIN_USERNAME=}"
+  fi
+  if [ -z "${ADMIN_EMAIL}" ]; then
+    EMAIL_LINE="$(grep -m1 '^KEYWARDEN_ADMIN_EMAIL=' "${ROOT_DIR}/.env" || true)"
+    ADMIN_EMAIL="${EMAIL_LINE#KEYWARDEN_ADMIN_EMAIL=}"
+  fi
+  if [ -z "${ADMIN_PASS}" ]; then
+    PASS_LINE="$(grep -m1 '^KEYWARDEN_ADMIN_PASSWORD=' "${ROOT_DIR}/.env" || true)"
+    ADMIN_PASS="${PASS_LINE#KEYWARDEN_ADMIN_PASSWORD=}"
+  fi
+fi
+
+if [ -z "${BASE_URL}" ]; then
+  BASE_URL="https://localhost"
+fi
+if [ -z "${ADMIN_USER}" ]; then
+  ADMIN_USER="admin"
+fi
+if [ -z "${ADMIN_EMAIL}" ]; then
+  ADMIN_EMAIL="admin@example.com"
+fi
+if [ -z "${ADMIN_PASS}" ]; then
+  echo "Missing KEYWARDEN_ADMIN_PASSWORD (env or .env)." >&2
+  exit 1
+fi
+if [ -z "${SERVER_ID}" ] && [ -f "${RUN_DIR}/logs/agent-endpoint-status-counts.txt" ]; then
+  SERVER_ID="$(
+    sed -n 's#.*servers/\([0-9][0-9]*\)/.*#\1#p' "${RUN_DIR}/logs/agent-endpoint-status-counts.txt" \
+      | head -n 1
+  )"
+fi
+
+CONTAINER_TOOL_DIR="/tmp/keywarden-evidence-tools"
+CONTAINER_OUT_DIR="/app/.evidence-screens/${RUN_ID}"
+HOST_TMP_OUT_DIR="${ROOT_DIR}/app/.evidence-screens/${RUN_ID}"
+HOST_FINAL_OUT_DIR="${RUN_DIR}/screenshots"
+
+docker compose exec -T \
+  -e KEYWARDEN_SCREENSHOT_BASE_URL="${BASE_URL}" \
+  -e KEYWARDEN_ADMIN_USERNAME="${ADMIN_USER}" \
+  -e KEYWARDEN_ADMIN_EMAIL="${ADMIN_EMAIL}" \
+  -e KEYWARDEN_ADMIN_PASSWORD="${ADMIN_PASS}" \
+  -e KEYWARDEN_SCREENSHOT_SERVER_ID="${SERVER_ID}" \
+  keywarden sh -lc "
+    set -eu
+    mkdir -p '${CONTAINER_TOOL_DIR}' '${CONTAINER_OUT_DIR}'
+    if [ ! -f '${CONTAINER_TOOL_DIR}/package.json' ]; then
+      cd '${CONTAINER_TOOL_DIR}'
+      npm init -y >/dev/null 2>&1
+      npm pkg set private=true >/dev/null 2>&1
+      npm pkg set type=module >/dev/null 2>&1
+    fi
+    if [ ! -d '${CONTAINER_TOOL_DIR}/node_modules/playwright' ]; then
+      cd '${CONTAINER_TOOL_DIR}'
+      npm install --silent playwright@1.53.0
+    fi
+    ln -sfn '${CONTAINER_TOOL_DIR}/node_modules' /app/scripts/node_modules
+    cd '${CONTAINER_TOOL_DIR}'
+    if [ ! -f '${CONTAINER_TOOL_DIR}/.deps-ok' ]; then
+      if ! npx playwright install-deps chromium >/dev/null 2>&1; then
+        apt-get update >/dev/null
+        apt-get install -y --no-install-recommends \
+          libglib2.0-0 \
+          libdbus-1-3 \
+          libatk1.0-0 \
+          libatk-bridge2.0-0 \
+          libatspi2.0-0 \
+          libcups2 \
+          libxcb1 \
+          libxkbcommon0 \
+          libx11-6 \
+          libxcomposite1 \
+          libxdamage1 \
+          libxext6 \
+          libxfixes3 \
+          libxrandr2 \
+          libgbm1 \
+          libcairo2 \
+          libpango-1.0-0 \
+          libasound2 >/dev/null
+      fi
+      touch '${CONTAINER_TOOL_DIR}/.deps-ok'
+    fi
+    npx playwright install chromium >/dev/null
+    cd /app
+    python manage.py shell -c \"import os; from django.contrib.auth import get_user_model; U=get_user_model(); username=os.environ['KEYWARDEN_ADMIN_USERNAME']; email=os.environ.get('KEYWARDEN_ADMIN_EMAIL','admin@example.com'); password=os.environ['KEYWARDEN_ADMIN_PASSWORD']; user, created = U.objects.get_or_create(username=username, defaults={'email': email}); user.email = ((user.email or email) if created else user.email); user.is_active=True; user.is_staff=True; user.is_superuser=True; user.set_password(password); user.save(); print('ensured-screenshot-admin', user.username, 'created=' + str(created))\"
+    node /app/scripts/capture_ui_screenshots.mjs '${CONTAINER_OUT_DIR}'
+  "
+
+rm -rf "${HOST_FINAL_OUT_DIR}"
+mkdir -p "${HOST_FINAL_OUT_DIR}"
+cp -R "${HOST_TMP_OUT_DIR}/." "${HOST_FINAL_OUT_DIR}/"
+ls -1 "${HOST_FINAL_OUT_DIR}" > "${HOST_FINAL_OUT_DIR}/listing.txt"
+
+echo "Screenshots captured for run ${RUN_ID}: ${HOST_FINAL_OUT_DIR}"

scripts/collect_evidence.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ID="${1:-$(date -u +%Y%m%dT%H%M%SZ)}"
+OUT_DIR="${ROOT_DIR}/evidence/runs/${RUN_ID}"
+SUMMARY_FILE="${OUT_DIR}/SUMMARY.md"
+
+mkdir -p "${OUT_DIR}"/{meta,runtime,tests,logs,docs}
+
+write_summary() {
+  printf "%s\n" "${1-}" >> "${SUMMARY_FILE}"
+}
+
+run_capture() {
+  local rel_path="$1"
+  shift
+  local target="${OUT_DIR}/${rel_path}"
+  local target_dir
+  target_dir="$(dirname "${target}")"
+  mkdir -p "${target_dir}"
+  if "$@" > "${target}" 2>&1; then
+    write_summary "- OK: \`${rel_path}\`"
+    return 0
+  else
+    local rc=$?
+    write_summary "- FAIL(${rc}): \`${rel_path}\`"
+    return "${rc}"
+  fi
+}
+
+write_summary "# Evidence Run"
+write_summary
+write_summary "- Run ID: \`${RUN_ID}\`"
+write_summary "- UTC Time: \`$(date -u +"%Y-%m-%dT%H:%M:%SZ")\`"
+write_summary "- Host: \`$(hostname)\`"
+write_summary "- Root: \`${ROOT_DIR}\`"
+write_summary
+write_summary "## Capture Status"
+
+run_capture "meta/git-head.txt" git -C "${ROOT_DIR}" rev-parse HEAD
+run_capture "meta/git-status.txt" git -C "${ROOT_DIR}" status --short
+run_capture "meta/git-log-20.txt" git -C "${ROOT_DIR}" log -n 20 --date=iso --pretty=oneline
+run_capture "meta/env-date.txt" date -u
+
+if command -v docker >/dev/null 2>&1; then
+  run_capture "runtime/docker-compose-ps.txt" docker compose -f "${ROOT_DIR}/docker-compose.yml" ps
+  run_capture "runtime/keywarden-python-version.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -V"
+  run_capture "runtime/keywarden-pip-freeze.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m pip freeze"
+else
+  write_summary "- SKIP: docker not available in PATH"
+fi
+
+TEST_TARGETS="apps.accounts.tests.test_auth apps.audit.tests.test_api_audit_middleware apps.servers.tests.test_access_scopes apps.servers.tests.test_audit_pipeline apps.servers.tests.test_consumers apps.servers.tests.test_keys_router"
+
+if command -v docker >/dev/null 2>&1; then
+  run_capture "tests/django-tests.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python manage.py test ${TEST_TARGETS} --verbosity 2"
+  run_capture "tests/django-tests-failures-summary.txt" \
+    sh -lc "grep -E '^(FAIL|ERROR):|^FAILED \\(|^Ran [0-9]+ tests? in' '${OUT_DIR}/tests/django-tests.txt' || true"
+  if ! run_capture "tests/coverage-version.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage --version"; then
+    if run_capture "tests/coverage-install.txt" \
+      docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m pip install coverage"; then
+      run_capture "tests/coverage-version.txt" \
+        docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage --version"
+    fi
+  fi
+  if [ -f "${OUT_DIR}/tests/coverage-version.txt" ] && grep -qi "Coverage.py" "${OUT_DIR}/tests/coverage-version.txt"; then
+    run_capture "tests/coverage-report.txt" \
+      docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage erase && python -m coverage run manage.py test ${TEST_TARGETS} --verbosity 1; test_rc=\$?; python -m coverage report -m; report_rc=\$?; python -m coverage xml -o /tmp/keywarden-coverage.xml; xml_rc=\$?; exit \$(( test_rc != 0 ? test_rc : ( report_rc != 0 ? report_rc : xml_rc ) ))"
+    run_capture "tests/coverage-xml.xml" \
+      docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cat /tmp/keywarden-coverage.xml"
+  else
+    write_summary "- SKIP: coverage module not installed in keywarden container"
+  fi
+fi
+
+run_capture "docs/rbac-claims.txt" \
+  sh -lc "grep -RIn 'operator\\|auditor\\|administrator\\|ROLE_' '${ROOT_DIR}/app/keywarden/api/routers/users.py' '${ROOT_DIR}/app/apps/core/rbac.py' '${ROOT_DIR}/API_DOCS.md'"
+
+if [ -f "${ROOT_DIR}/nginx/logs/agent-access.log" ]; then
+  run_capture "logs/agent-access-tail-2000.log" tail -n 2000 "${ROOT_DIR}/nginx/logs/agent-access.log"
+  run_capture "logs/agent-endpoint-status-counts.txt" \
+    sh -lc "awk -F'\"' '{split(\$3, a, \" \"); if (\$2 != \"\") print \$2 \" \" a[2]}' '${ROOT_DIR}/nginx/logs/agent-access.log' | sort | uniq -c | sort -nr"
+fi
+
+if [ -f "${ROOT_DIR}/app/logs/gunicorn-access.log" ]; then
+  run_capture "logs/gunicorn-access-tail-2000.log" tail -n 2000 "${ROOT_DIR}/app/logs/gunicorn-access.log"
+  run_capture "logs/gunicorn-endpoint-status-counts.txt" \
+    sh -lc "awk -F'\"' '{split(\$3, a, \" \"); if (\$2 != \"\") print \$2 \" \" a[2]}' '${ROOT_DIR}/app/logs/gunicorn-access.log' | sort | uniq -c | sort -nr"
+fi
+
+if [ -f "${ROOT_DIR}/nginx/logs/error.log" ]; then
+  run_capture "logs/nginx-error-tail-1000.log" tail -n 1000 "${ROOT_DIR}/nginx/logs/error.log"
+fi
+
+if [ -d "${ROOT_DIR}/evidence/diagrams" ]; then
+  mkdir -p "${OUT_DIR}/diagrams"
+  run_capture "diagrams/listing.txt" sh -lc "cp '${ROOT_DIR}'/evidence/diagrams/*.mmd '${OUT_DIR}/diagrams/' && ls -1 '${OUT_DIR}/diagrams'"
+fi
+
+if [ -d "${ROOT_DIR}/evidence/templates" ]; then
+  mkdir -p "${OUT_DIR}/templates"
+  run_capture "templates/listing.txt" sh -lc "cp '${ROOT_DIR}'/evidence/templates/*.md '${OUT_DIR}/templates/' && ls -1 '${OUT_DIR}/templates'"
+fi
+
+if [ "${KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS:-0}" = "1" ] && [ -x "${ROOT_DIR}/scripts/capture_ui_screenshots.sh" ]; then
+  run_capture "screenshots-capture.log" \
+    bash "${ROOT_DIR}/scripts/capture_ui_screenshots.sh" "${RUN_ID}"
+fi
+
+if [ -x "${ROOT_DIR}/scripts/autofill_evidence_templates.sh" ]; then
+  run_capture "templates/autofill.log" \
+    bash "${ROOT_DIR}/scripts/autofill_evidence_templates.sh" "${RUN_ID}"
+fi
+
+if [ -x "${ROOT_DIR}/scripts/summarize_operational_metrics.sh" ]; then
+  run_capture "evaluation/generation.log" \
+    bash "${ROOT_DIR}/scripts/summarize_operational_metrics.sh" "${RUN_ID}"
+fi
+
+write_summary
+write_summary "## Notes"
+write_summary "- This run captures operational evidence and test output that can be re-generated."
+write_summary "- Screenshot capture can be automated by setting \`KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS=1\`."
+
+printf "Evidence run complete: %s\n" "${OUT_DIR}"

scripts/summarize_operational_metrics.sh

@@ -0,0 +1,268 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ARG="${1:-}"
+
+if [ -n "${RUN_ARG}" ]; then
+  if [ -d "${RUN_ARG}" ]; then
+    RUN_DIR="${RUN_ARG}"
+  else
+    RUN_DIR="${ROOT_DIR}/evidence/runs/${RUN_ARG}"
+  fi
+else
+  RUN_DIR="$(ls -1dt "${ROOT_DIR}"/evidence/runs/* 2>/dev/null | head -n 1)"
+fi
+
+if [ -z "${RUN_DIR:-}" ] || [ ! -d "${RUN_DIR}" ]; then
+  echo "Run directory not found." >&2
+  exit 1
+fi
+
+RUN_ID="$(basename "${RUN_DIR}")"
+OUT_DIR="${RUN_DIR}/evaluation"
+mkdir -p "${OUT_DIR}"
+
+AGENT_LOG="${RUN_DIR}/logs/agent-access-tail-2000.log"
+NGINX_ERROR_LOG="${RUN_DIR}/logs/nginx-error-tail-1000.log"
+SUMMARY_MD="${OUT_DIR}/operational-summary.md"
+PARSED_CSV="${OUT_DIR}/agent-access-parsed.csv"
+HEARTBEAT_EVENTS="${OUT_DIR}/heartbeat-events.csv"
+HEARTBEAT_INTERVALS="${OUT_DIR}/heartbeat-intervals.csv"
+HEARTBEAT_SUMMARY="${OUT_DIR}/heartbeat-summary.csv"
+HEARTBEAT_RECOVERY="${OUT_DIR}/heartbeat-error-recovery.csv"
+ENDPOINT_STATUS_COUNTS="${OUT_DIR}/endpoint-status-by-server.csv"
+
+if [ ! -f "${AGENT_LOG}" ]; then
+  cat > "${SUMMARY_MD}" <<EOF
+# Operational Metrics Summary
+
+- Run ID: \`${RUN_ID}\`
+- Status: No \`logs/agent-access-tail-2000.log\` found for this run.
+
+EOF
+  echo "Operational metrics summary written: ${SUMMARY_MD}"
+  exit 0
+fi
+
+awk '
+BEGIN {
+  OFS = ","
+  print "raw_timestamp,server_id,method,path,status,bytes,client_ip"
+}
+{
+  client_ip = $1
+  left_bracket = index($0, "[")
+  right_bracket = index($0, "]")
+  if (left_bracket == 0 || right_bracket <= left_bracket) {
+    next
+  }
+  raw_ts = substr($0, left_bracket + 1, right_bracket - left_bracket - 1)
+  quote_count = split($0, quoted, "\"")
+  if (quote_count < 3) {
+    next
+  }
+  split(quoted[2], req_parts, " ")
+  method = req_parts[1]
+  path = req_parts[2]
+  if (path == "") {
+    next
+  }
+  split(path, segments, "/")
+  server_id = ""
+  for (i = 1; i <= length(segments); i++) {
+    if (segments[i] == "servers" && (i + 1) <= length(segments)) {
+      server_id = segments[i + 1]
+      break
+    }
+  }
+  if (server_id == "" || server_id !~ /^[0-9]+$/) {
+    next
+  }
+  split(quoted[3], tail_parts)
+  status = tail_parts[1]
+  bytes = tail_parts[2]
+  print raw_ts, server_id, method, path, status, bytes, client_ip
+}
+' "${AGENT_LOG}" > "${PARSED_CSV}"
+
+echo "server_id,epoch,raw_timestamp,status" > "${HEARTBEAT_EVENTS}"
+rm -f "${OUT_DIR}/heartbeat-events-raw.tmp"
+tail -n +2 "${PARSED_CSV}" | while IFS=',' read -r raw_ts server_id method path status bytes client_ip; do
+  case "${path}" in
+    */heartbeat)
+      normalized_ts="$(printf "%s" "${raw_ts}" | sed -E 's#^([0-9]{2})/([A-Za-z]{3})/([0-9]{4}):([0-9]{2}:[0-9]{2}:[0-9]{2}) ([+-][0-9]{4})$#\1 \2 \3 \4 \5#')"
+      epoch="$(date -u -d "${normalized_ts}" +%s 2>/dev/null || true)"
+      if [ -n "${epoch}" ]; then
+        printf "%s,%s,%s,%s\n" "${server_id}" "${epoch}" "${raw_ts}" "${status}" >> "${OUT_DIR}/heartbeat-events-raw.tmp"
+      fi
+      ;;
+  esac
+done
+if [ -f "${OUT_DIR}/heartbeat-events-raw.tmp" ]; then
+  sort -t, -k1,1n -k2,2n "${OUT_DIR}/heartbeat-events-raw.tmp" >> "${HEARTBEAT_EVENTS}"
+else
+  : > "${OUT_DIR}/heartbeat-events-raw.tmp"
+fi
+
+awk -F',' -v intervals_file="${HEARTBEAT_INTERVALS}" -v summary_file="${HEARTBEAT_SUMMARY}" '
+BEGIN {
+  print "server_id,from_timestamp,to_timestamp,interval_seconds,from_status,to_status" > intervals_file
+}
+NR == 1 {
+  next
+}
+{
+  server_id = $1
+  epoch = $2 + 0
+  raw_ts = $3
+  status = $4 + 0
+  event_count[server_id]++
+  if (status >= 500) {
+    error_count[server_id]++
+  }
+  if (server_id in prev_epoch) {
+    interval = epoch - prev_epoch[server_id]
+    if (interval >= 0) {
+      print server_id "," prev_ts[server_id] "," raw_ts "," interval "," prev_status[server_id] "," status >> intervals_file
+      interval_count[server_id]++
+      interval_sum[server_id] += interval
+      if (!(server_id in interval_min) || interval < interval_min[server_id]) {
+        interval_min[server_id] = interval
+      }
+      if (!(server_id in interval_max) || interval > interval_max[server_id]) {
+        interval_max[server_id] = interval
+      }
+    }
+  }
+  prev_epoch[server_id] = epoch
+  prev_ts[server_id] = raw_ts
+  prev_status[server_id] = status
+}
+END {
+  print "server_id,heartbeat_events,interval_samples,avg_interval_s,min_interval_s,max_interval_s,heartbeat_5xx" > summary_file
+  for (server_id in event_count) {
+    avg = interval_count[server_id] ? interval_sum[server_id] / interval_count[server_id] : 0
+    min_v = interval_count[server_id] ? interval_min[server_id] : 0
+    max_v = interval_count[server_id] ? interval_max[server_id] : 0
+    err_v = error_count[server_id] + 0
+    printf "%s,%d,%d,%.2f,%d,%d,%d\n", server_id, event_count[server_id], interval_count[server_id] + 0, avg, min_v, max_v, err_v >> summary_file
+  }
+}
+' "${HEARTBEAT_EVENTS}"
+
+{
+  head -n 1 "${HEARTBEAT_SUMMARY}"
+  tail -n +2 "${HEARTBEAT_SUMMARY}" | sort -t, -k1,1n
+} > "${OUT_DIR}/heartbeat-summary.tmp"
+mv "${OUT_DIR}/heartbeat-summary.tmp" "${HEARTBEAT_SUMMARY}"
+
+awk -F',' '
+NR == 1 {
+  next
+}
+{
+  key = $2 "," $4 "," $5
+  counts[key]++
+}
+END {
+  for (key in counts) {
+    split(key, parts, ",")
+    printf "%s,%s,%s,%d\n", parts[1], parts[2], parts[3], counts[key]
+  }
+}
+' "${PARSED_CSV}" | sort -t, -k4,4nr -k1,1n > "${OUT_DIR}/endpoint-status-by-server-raw.tmp"
+
+{
+  echo "server_id,path,status,count"
+  cat "${OUT_DIR}/endpoint-status-by-server-raw.tmp"
+} > "${ENDPOINT_STATUS_COUNTS}"
+
+awk -F',' '
+BEGIN {
+  print "server_id,error_timestamp,error_status,recovery_timestamp,recovery_status,recovery_seconds"
+}
+NR == 1 {
+  next
+}
+{
+  server_id = $1
+  epoch = $2 + 0
+  raw_ts = $3
+  status = $4 + 0
+  if (status >= 500 && !(server_id in pending_epoch)) {
+    pending_epoch[server_id] = epoch
+    pending_ts[server_id] = raw_ts
+    pending_status[server_id] = status
+  } else if (status >= 200 && status < 300 && (server_id in pending_epoch)) {
+    recovery = epoch - pending_epoch[server_id]
+    printf "%s,%s,%d,%s,%d,%d\n", server_id, pending_ts[server_id], pending_status[server_id], raw_ts, status, recovery
+    delete pending_epoch[server_id]
+    delete pending_ts[server_id]
+    delete pending_status[server_id]
+  }
+}
+' "${HEARTBEAT_EVENTS}" > "${HEARTBEAT_RECOVERY}"
+
+total_agent_events="$(awk -F',' 'NR>1{count++} END{print count+0}' "${PARSED_CSV}")"
+unique_servers="$(awk -F',' 'NR>1{seen[$2]=1} END{count=0; for (s in seen) count++; print count+0}' "${PARSED_CSV}")"
+heartbeat_events_total="$(awk -F',' 'NR>1{count++} END{print count+0}' "${HEARTBEAT_EVENTS}")"
+heartbeat_errors_total="$(awk -F',' 'NR>1 && $4+0>=500{count++} END{print count+0}' "${HEARTBEAT_EVENTS}")"
+nginx_upstream_refused="0"
+if [ -f "${NGINX_ERROR_LOG}" ]; then
+  nginx_upstream_refused="$(grep -c "connect() failed" "${NGINX_ERROR_LOG}" || true)"
+fi
+
+{
+  echo "# Operational Metrics Summary"
+  echo
+  echo "- Run ID: \`${RUN_ID}\`"
+  echo "- Parsed agent access events: \`${total_agent_events}\`"
+  echo "- Servers observed in agent traffic: \`${unique_servers}\`"
+  echo "- Heartbeat events observed: \`${heartbeat_events_total}\`"
+  echo "- Heartbeat 5xx responses observed: \`${heartbeat_errors_total}\`"
+  echo "- Nginx upstream connection errors (tail window): \`${nginx_upstream_refused}\`"
+  echo
+  echo "## Heartbeat Interval Summary"
+  echo
+  echo "| Server ID | Heartbeat Events | Interval Samples | Avg Interval (s) | Min (s) | Max (s) | Heartbeat 5xx |"
+  echo "|---|---:|---:|---:|---:|---:|---:|"
+  tail -n +2 "${HEARTBEAT_SUMMARY}" | while IFS=',' read -r sid ev samples avg minv maxv errv; do
+    printf "| %s | %s | %s | %s | %s | %s | %s |\n" "${sid}" "${ev}" "${samples}" "${avg}" "${minv}" "${maxv}" "${errv}"
+  done
+  echo
+  echo "## Heartbeat Error Recovery Events"
+  echo
+  if [ "$(awk -F',' 'NR>1{count++} END{print count+0}' "${HEARTBEAT_RECOVERY}")" -eq 0 ]; then
+    echo "- No heartbeat error->recovery transitions were found in this log window."
+  else
+    echo "| Server ID | Error Timestamp | Error Status | Recovery Timestamp | Recovery Status | Recovery Seconds |"
+    echo "|---|---|---:|---|---:|---:|"
+    tail -n +2 "${HEARTBEAT_RECOVERY}" | while IFS=',' read -r sid ets ests rts rsts sec; do
+      printf "| %s | %s | %s | %s | %s | %s |\n" "${sid}" "${ets}" "${ests}" "${rts}" "${rsts}" "${sec}"
+    done
+  fi
+  echo
+  echo "## Top Endpoint/Status Counts"
+  echo
+  echo "| Rank | Server ID | Path | Status | Count |"
+  echo "|---:|---:|---|---:|---:|"
+  tail -n +2 "${ENDPOINT_STATUS_COUNTS}" | head -n 15 | nl -ba -w1 -s',' | while IFS=',' read -r rank sid path status count; do
+    printf "| %s | %s | \`%s\` | %s | %s |\n" "${rank}" "${sid}" "${path}" "${status}" "${count}"
+  done
+  echo
+  echo "## Generated Artifacts"
+  echo
+  echo "- \`evaluation/agent-access-parsed.csv\`"
+  echo "- \`evaluation/heartbeat-events.csv\`"
+  echo "- \`evaluation/heartbeat-intervals.csv\`"
+  echo "- \`evaluation/heartbeat-summary.csv\`"
+  echo "- \`evaluation/heartbeat-error-recovery.csv\`"
+  echo "- \`evaluation/endpoint-status-by-server.csv\`"
+} > "${SUMMARY_MD}"
+
+rm -f \
+  "${OUT_DIR}/heartbeat-events-raw.tmp" \
+  "${OUT_DIR}/endpoint-status-by-server-raw.tmp"
+
+echo "Operational metrics summary written: ${SUMMARY_MD}"