131 lines
6.2 KiB
Bash
Executable File
131 lines
6.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -u -o pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
RUN_ID="${1:-$(date -u +%Y%m%dT%H%M%SZ)}"
|
|
OUT_DIR="${ROOT_DIR}/evidence/runs/${RUN_ID}"
|
|
SUMMARY_FILE="${OUT_DIR}/SUMMARY.md"
|
|
|
|
mkdir -p "${OUT_DIR}"/{meta,runtime,tests,logs,docs}
|
|
|
|
write_summary() {
|
|
printf "%s\n" "${1-}" >> "${SUMMARY_FILE}"
|
|
}
|
|
|
|
run_capture() {
|
|
local rel_path="$1"
|
|
shift
|
|
local target="${OUT_DIR}/${rel_path}"
|
|
local target_dir
|
|
target_dir="$(dirname "${target}")"
|
|
mkdir -p "${target_dir}"
|
|
if "$@" > "${target}" 2>&1; then
|
|
write_summary "- OK: \`${rel_path}\`"
|
|
return 0
|
|
else
|
|
local rc=$?
|
|
write_summary "- FAIL(${rc}): \`${rel_path}\`"
|
|
return "${rc}"
|
|
fi
|
|
}
|
|
|
|
write_summary "# Evidence Run"
|
|
write_summary
|
|
write_summary "- Run ID: \`${RUN_ID}\`"
|
|
write_summary "- UTC Time: \`$(date -u +"%Y-%m-%dT%H:%M:%SZ")\`"
|
|
write_summary "- Host: \`$(hostname)\`"
|
|
write_summary "- Root: \`${ROOT_DIR}\`"
|
|
write_summary
|
|
write_summary "## Capture Status"
|
|
|
|
run_capture "meta/git-head.txt" git -C "${ROOT_DIR}" rev-parse HEAD
|
|
run_capture "meta/git-status.txt" git -C "${ROOT_DIR}" status --short
|
|
run_capture "meta/git-log-20.txt" git -C "${ROOT_DIR}" log -n 20 --date=iso --pretty=oneline
|
|
run_capture "meta/env-date.txt" date -u
|
|
|
|
if command -v docker >/dev/null 2>&1; then
|
|
run_capture "runtime/docker-compose-ps.txt" docker compose -f "${ROOT_DIR}/docker-compose.yml" ps
|
|
run_capture "runtime/keywarden-python-version.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -V"
|
|
run_capture "runtime/keywarden-pip-freeze.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m pip freeze"
|
|
else
|
|
write_summary "- SKIP: docker not available in PATH"
|
|
fi
|
|
|
|
TEST_TARGETS="apps.accounts.tests.test_auth apps.audit.tests.test_api_audit_middleware apps.servers.tests.test_access_scopes apps.servers.tests.test_audit_pipeline apps.servers.tests.test_consumers apps.servers.tests.test_keys_router"
|
|
|
|
if command -v docker >/dev/null 2>&1; then
|
|
run_capture "tests/django-tests.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python manage.py test ${TEST_TARGETS} --verbosity 2"
|
|
run_capture "tests/django-tests-failures-summary.txt" \
|
|
sh -lc "grep -E '^(FAIL|ERROR):|^FAILED \\(|^Ran [0-9]+ tests? in' '${OUT_DIR}/tests/django-tests.txt' || true"
|
|
if ! run_capture "tests/coverage-version.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage --version"; then
|
|
if run_capture "tests/coverage-install.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m pip install coverage"; then
|
|
run_capture "tests/coverage-version.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage --version"
|
|
fi
|
|
fi
|
|
if [ -f "${OUT_DIR}/tests/coverage-version.txt" ] && grep -qi "Coverage.py" "${OUT_DIR}/tests/coverage-version.txt"; then
|
|
run_capture "tests/coverage-report.txt" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage erase && python -m coverage run manage.py test ${TEST_TARGETS} --verbosity 1; test_rc=\$?; python -m coverage report -m; report_rc=\$?; python -m coverage xml -o /tmp/keywarden-coverage.xml; xml_rc=\$?; exit \$(( test_rc != 0 ? test_rc : ( report_rc != 0 ? report_rc : xml_rc ) ))"
|
|
run_capture "tests/coverage-xml.xml" \
|
|
docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cat /tmp/keywarden-coverage.xml"
|
|
else
|
|
write_summary "- SKIP: coverage module not installed in keywarden container"
|
|
fi
|
|
fi
|
|
|
|
run_capture "docs/rbac-claims.txt" \
|
|
sh -lc "grep -RIn 'operator\\|auditor\\|administrator\\|ROLE_' '${ROOT_DIR}/app/keywarden/api/routers/users.py' '${ROOT_DIR}/app/apps/core/rbac.py' '${ROOT_DIR}/API_DOCS.md'"
|
|
|
|
if [ -f "${ROOT_DIR}/nginx/logs/agent-access.log" ]; then
|
|
run_capture "logs/agent-access-tail-2000.log" tail -n 2000 "${ROOT_DIR}/nginx/logs/agent-access.log"
|
|
run_capture "logs/agent-endpoint-status-counts.txt" \
|
|
sh -lc "awk -F'\"' '{split(\$3, a, \" \"); if (\$2 != \"\") print \$2 \" \" a[2]}' '${ROOT_DIR}/nginx/logs/agent-access.log' | sort | uniq -c | sort -nr"
|
|
fi
|
|
|
|
if [ -f "${ROOT_DIR}/app/logs/gunicorn-access.log" ]; then
|
|
run_capture "logs/gunicorn-access-tail-2000.log" tail -n 2000 "${ROOT_DIR}/app/logs/gunicorn-access.log"
|
|
run_capture "logs/gunicorn-endpoint-status-counts.txt" \
|
|
sh -lc "awk -F'\"' '{split(\$3, a, \" \"); if (\$2 != \"\") print \$2 \" \" a[2]}' '${ROOT_DIR}/app/logs/gunicorn-access.log' | sort | uniq -c | sort -nr"
|
|
fi
|
|
|
|
if [ -f "${ROOT_DIR}/nginx/logs/error.log" ]; then
|
|
run_capture "logs/nginx-error-tail-1000.log" tail -n 1000 "${ROOT_DIR}/nginx/logs/error.log"
|
|
fi
|
|
|
|
if [ -d "${ROOT_DIR}/evidence/diagrams" ]; then
|
|
mkdir -p "${OUT_DIR}/diagrams"
|
|
run_capture "diagrams/listing.txt" sh -lc "cp '${ROOT_DIR}'/evidence/diagrams/*.mmd '${OUT_DIR}/diagrams/' && ls -1 '${OUT_DIR}/diagrams'"
|
|
fi
|
|
|
|
if [ -d "${ROOT_DIR}/evidence/templates" ]; then
|
|
mkdir -p "${OUT_DIR}/templates"
|
|
run_capture "templates/listing.txt" sh -lc "cp '${ROOT_DIR}'/evidence/templates/*.md '${OUT_DIR}/templates/' && ls -1 '${OUT_DIR}/templates'"
|
|
fi
|
|
|
|
if [ "${KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS:-0}" = "1" ] && [ -x "${ROOT_DIR}/scripts/capture_ui_screenshots.sh" ]; then
|
|
run_capture "screenshots-capture.log" \
|
|
bash "${ROOT_DIR}/scripts/capture_ui_screenshots.sh" "${RUN_ID}"
|
|
fi
|
|
|
|
if [ -x "${ROOT_DIR}/scripts/autofill_evidence_templates.sh" ]; then
|
|
run_capture "templates/autofill.log" \
|
|
bash "${ROOT_DIR}/scripts/autofill_evidence_templates.sh" "${RUN_ID}"
|
|
fi
|
|
|
|
if [ -x "${ROOT_DIR}/scripts/summarize_operational_metrics.sh" ]; then
|
|
run_capture "evaluation/generation.log" \
|
|
bash "${ROOT_DIR}/scripts/summarize_operational_metrics.sh" "${RUN_ID}"
|
|
fi
|
|
|
|
write_summary
|
|
write_summary "## Notes"
|
|
write_summary "- This run captures operational evidence and test output that can be re-generated."
|
|
write_summary "- Screenshot capture can be automated by setting \`KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS=1\`."
|
|
|
|
printf "Evidence run complete: %s\n" "${OUT_DIR}"
|